You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/09/11 15:23:02 UTC
(opendal) branch main updated: feat(services/hf): reuse resolved downloads across readers (#8280)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 43d87279b feat(services/hf): reuse resolved downloads across readers (#8280)
43d87279b is described below
commit 43d87279b8af7af96402fd975c40344493c6497b
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Sep 11 23:22:56 2026 +0800
feat(services/hf): reuse resolved downloads across readers (#8280)
* feat(services/hf): reuse resolved downloads across readers
* fix(services/hf): synchronize configuration and CI checks
* fix(ci): honor configured behavior test concurrency
---
.github/actions/test_behavior_core/action.yaml | 3 +-
.github/services/hf/hf_bucket/action.yml | 2 +
.github/services/hf/hf_bucket_http/action.yml | 2 +
.github/services/hf/hf_dataset/action.yml | 2 +
.../ServiceConfig/HuggingfaceServiceConfig.cs | 8 +
.../java/org/apache/opendal/ServiceConfig.java | 9 +
bindings/python/python/opendal/config.py | 2 +
core/core/src/raw/http_util/error.rs | 9 +
core/core/src/types/http_transport/mod.rs | 32 ++-
core/core/src/types/http_transport/uri.rs | 166 +++++++++++++
core/core/src/types/mod.rs | 2 +
core/http-transports/reqwest/src/lib.rs | 50 +++-
core/services/hf/src/backend.rs | 29 ++-
core/services/hf/src/config.rs | 21 ++
core/services/hf/src/core.rs | 269 +++++++++++++++++++--
core/services/hf/src/docs.md | 30 +++
core/services/hf/src/reader.rs | 86 +------
website/data/services.json | 17 +-
18 files changed, 626 insertions(+), 113 deletions(-)
diff --git a/.github/actions/test_behavior_core/action.yaml b/.github/actions/test_behavior_core/action.yaml
index b25d637c0..a26cfb3b4 100644
--- a/.github/actions/test_behavior_core/action.yaml
+++ b/.github/actions/test_behavior_core/action.yaml
@@ -71,7 +71,8 @@ runs:
if: always() && steps.setup_$index.outcome == 'success'
shell: bash
working-directory: core
- run: cargo test behavior --features tests,$TEST_FEATURE
+ # libtest-mimic requires an argument for the configured thread limit.
+ run: cargo test behavior --features tests,$TEST_FEATURE -- \${RUST_TEST_THREADS:+--test-threads="\$RUST_TEST_THREADS"}
env:
OPENDAL_TEST: $TEST_SERVICE
- name: Cleanup Test Core - $setup
diff --git a/.github/services/hf/hf_bucket/action.yml b/.github/services/hf/hf_bucket/action.yml
index edb4f6698..e0d24b0c3 100644
--- a/.github/services/hf/hf_bucket/action.yml
+++ b/.github/services/hf/hf_bucket/action.yml
@@ -36,6 +36,8 @@ runs:
- name: Setup env
shell: bash
run: |
+ # Behavior tests overwrite paths and must observe the updated content.
cat << EOF >> $GITHUB_ENV
OPENDAL_HF_REPO_TYPE=bucket
+ OPENDAL_HF_FORCE_RESOLVE=true
EOF
diff --git a/.github/services/hf/hf_bucket_http/action.yml b/.github/services/hf/hf_bucket_http/action.yml
index ecc4db57d..89c80b848 100644
--- a/.github/services/hf/hf_bucket_http/action.yml
+++ b/.github/services/hf/hf_bucket_http/action.yml
@@ -36,7 +36,9 @@ runs:
- name: Setup env
shell: bash
run: |
+ # Behavior tests overwrite paths and must observe the updated content.
cat << EOF >> $GITHUB_ENV
OPENDAL_HF_REPO_TYPE=bucket
OPENDAL_HF_DOWNLOAD_MODE=http
+ OPENDAL_HF_FORCE_RESOLVE=true
EOF
diff --git a/.github/services/hf/hf_dataset/action.yml b/.github/services/hf/hf_dataset/action.yml
index 3439b33ae..da7290fad 100644
--- a/.github/services/hf/hf_dataset/action.yml
+++ b/.github/services/hf/hf_dataset/action.yml
@@ -36,7 +36,9 @@ runs:
- name: Setup env
shell: bash
run: |
+ # Behavior tests overwrite paths and must observe the updated content.
cat << EOF >> $GITHUB_ENV
OPENDAL_HF_REPO_TYPE=dataset
+ OPENDAL_HF_FORCE_RESOLVE=true
RUST_TEST_THREADS=1
EOF
diff --git a/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs b/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs
index 3fa7aa9c4..685efe912 100644
--- a/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs
+++ b/bindings/dotnet/OpenDAL/ServiceConfig/HuggingfaceServiceConfig.cs
@@ -37,6 +37,10 @@ namespace OpenDAL.ServiceConfig
/// </summary>
public string? Endpoint { get; init; }
/// <summary>
+ /// Resolve every range through the Hugging Face Hub. Defaults to `false`. Set to `true` to resolve every range through the Hub. See [`HfBuilder::force_resolve`] for cache and freshness semantics.
+ /// </summary>
+ public bool? ForceResolve { get; init; }
+ /// <summary>
/// Repo id of this backend. This is required.
/// </summary>
public string? RepoId { get; init; }
@@ -70,6 +74,10 @@ namespace OpenDAL.ServiceConfig
{
map["endpoint"] = Utilities.ToOptionString(Endpoint);
}
+ if (ForceResolve is not null)
+ {
+ map["force_resolve"] = Utilities.ToOptionString(ForceResolve);
+ }
if (RepoId is not null)
{
map["repo_id"] = Utilities.ToOptionString(RepoId);
diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
index be58bfea7..70287125d 100644
--- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
+++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
@@ -1686,6 +1686,12 @@ public interface ServiceConfig {
* <p>The default is <code>https://huggingface.co</code>.</p>
*/
public final String endpoint;
+ /**
+ * <p>Resolve every range through the Hugging Face Hub. Defaults to <code>false</code>.</p>
+ * <p>Set to <code>true</code> to resolve every range through the Hub. See
+ * [<code>HfBuilder::force_resolve</code>] for cache and freshness semantics.</p>
+ */
+ public final Boolean forceResolve;
/**
* <p>Repo id of this backend.</p>
* <p>This is required.</p>
@@ -1725,6 +1731,9 @@ public interface ServiceConfig {
if (endpoint != null) {
map.put("endpoint", endpoint);
}
+ if (forceResolve != null) {
+ map.put("force_resolve", String.valueOf(forceResolve));
+ }
if (repoId != null) {
map.put("repo_id", repoId);
}
diff --git a/bindings/python/python/opendal/config.py b/bindings/python/python/opendal/config.py
index 9129b156d..a8eb04f0e 100644
--- a/bindings/python/python/opendal/config.py
+++ b/bindings/python/python/opendal/config.py
@@ -412,6 +412,8 @@ class HfConfig(TypedDict):
"""Download mode. Either `xet` (default) or `http`. When unset, the mode is resolved from the `HF_HUB_DISABLE_XET` environment variable: a non-empty value forces `http`, otherwise it defaults to `xet`. An explicit value here takes precedence. See <https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubdisablexet>."""
endpoint: NotRequired[str]
"""Endpoint of the Hugging Face Hub. The default is `https://huggingface.co`."""
+ force_resolve: NotRequired[bool]
+ """Resolve every range through the Hugging Face Hub. Defaults to `false`. Set to `true` to resolve every range through the Hub. See [`HfBuilder::force_resolve`] for cache and freshness semantics."""
repo_id: NotRequired[str]
"""Repo id of this backend. This is required."""
repo_type: NotRequired[str]
diff --git a/core/core/src/raw/http_util/error.rs b/core/core/src/raw/http_util/error.rs
index 418e25bb6..cefb72b77 100644
--- a/core/core/src/raw/http_util/error.rs
+++ b/core/core/src/raw/http_util/error.rs
@@ -62,6 +62,15 @@ pub fn with_error_response_context(mut err: Error, mut parts: Parts) -> Error {
parts.headers.remove("WWW-Authenticate");
parts.headers.remove("Proxy-Authenticate");
+ if parts.headers.contains_key(http::header::LOCATION) {
+ let location = crate::HttpUri::from_response_location(&mut parts)
+ .map(|uri| uri.redacted_uri())
+ .unwrap_or("<invalid URL>")
+ .parse()
+ .unwrap_or_else(|_| http::HeaderValue::from_static("<invalid URL>"));
+ parts.headers.insert(http::header::LOCATION, location);
+ }
+
err = err.with_context("response", format!("{parts:?}"));
err
diff --git a/core/core/src/types/http_transport/mod.rs b/core/core/src/types/http_transport/mod.rs
index d97a57ed7..6ea1758de 100644
--- a/core/core/src/types/http_transport/mod.rs
+++ b/core/core/src/types/http_transport/mod.rs
@@ -36,12 +36,16 @@ use crate::types::Result;
mod body;
pub use body::HttpBody;
+mod uri;
+pub use uri::{HttpRedirect, HttpUri};
+
static DEFAULT_HTTP_TRANSPORTER: OnceLock<HttpTransporter> = OnceLock::new();
/// HTTP transport used by OpenDAL services.
///
/// Implement this trait to provide a custom HTTP backend. A transport must
/// support 3xx redirection.
+/// It can support [`HttpRedirect`] to let services reuse redirect destinations.
pub trait HttpTransport: Send + Sync + Unpin + 'static {
/// Fetch a request and return a streamable [`HttpBody`].
#[cfg(not(target_arch = "wasm32"))]
@@ -114,8 +118,32 @@ impl HttpTransporter {
}
/// Fetch a request and return a streamable [`HttpBody`].
- pub async fn fetch(&self, req: Request<Buffer>) -> Result<Response<HttpBody>> {
- self.inner.fetch_dyn(req).await
+ ///
+ /// Mark credential headers as sensitive for diagnostics without changing
+ /// their values.
+ pub async fn fetch(&self, mut req: Request<Buffer>) -> Result<Response<HttpBody>> {
+ // Mark credentials at the shared boundary before any transport wrapper
+ // formats the request. Values on the wire remain unchanged.
+ for (name, value) in req.headers_mut() {
+ if matches!(
+ name.as_str(),
+ "authorization" | "proxy-authorization" | "cookie" | "cookie2"
+ ) {
+ value.set_sensitive(true);
+ }
+ }
+ let mut resp = self.inner.fetch_dyn(req).await?;
+ for (name, value) in resp.headers_mut() {
+ if matches!(
+ name.as_str(),
+ "set-cookie" | "www-authenticate" | "proxy-authenticate"
+ ) {
+ value.set_sensitive(true);
+ }
+ }
+ let (mut parts, body) = resp.into_parts();
+ HttpUri::from_response_location(&mut parts);
+ Ok(Response::from_parts(parts, body))
}
}
diff --git a/core/core/src/types/http_transport/uri.rs b/core/core/src/types/http_transport/uri.rs
new file mode 100644
index 000000000..feeb6a08d
--- /dev/null
+++ b/core/core/src/types/http_transport/uri.rs
@@ -0,0 +1,166 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::{Debug, Formatter};
+use std::sync::Arc;
+
+use percent_encoding::percent_decode_str;
+
+/// An HTTP URI or relative location with a precomputed diagnostic representation.
+///
+/// Construction preserves the original spelling and masks user information and
+/// recognized credential query values in a separate representation. It does not
+/// validate, resolve, or normalize the URI. Ordinary query values remain intact.
+/// Clones share both representations; formatting does not repeat redaction.
+///
+/// Recognized query names are `Signature`, `Policy`, `token`, `access_token`,
+/// `X-Amz-Signature`, and `X-Amz-Security-Token`, matched case-insensitively after
+/// decoding the name. Arbitrary application parameters are not classified.
+#[derive(Clone)]
+pub struct HttpUri {
+ original: Arc<str>,
+ redacted: Arc<str>,
+}
+
+impl HttpUri {
+ /// Retain the original URI and compute its diagnostic representation once.
+ pub fn new(uri: impl Into<Arc<str>>) -> Self {
+ let original = uri.into();
+ let redacted = redact_uri(&original).into();
+ Self { original, redacted }
+ }
+
+ /// Return the original URI, including any credentials it contains.
+ pub fn original_uri(&self) -> &str {
+ &self.original
+ }
+
+ /// Return the URI with recognized credentials masked for diagnostics.
+ pub fn redacted_uri(&self) -> &str {
+ &self.redacted
+ }
+
+ /// Get or create the diagnostic value for a response's `Location` header.
+ ///
+ /// Store it in response extensions so service errors and shared response
+ /// diagnostics reuse the same value. If a wrapper changes the header,
+ /// replace the cached value. Return `None` for missing or non-text headers.
+ /// The original header is not changed.
+ pub fn from_response_location(parts: &mut http::response::Parts) -> Option<&Self> {
+ let location = parts.headers.get(http::header::LOCATION)?.to_str().ok()?;
+ if parts
+ .extensions
+ .get::<Self>()
+ .is_none_or(|uri| uri.original_uri() != location)
+ {
+ let uri = Self::new(location);
+ parts.extensions.insert(uri);
+ }
+ parts.extensions.get::<Self>()
+ }
+}
+
+impl Debug for HttpUri {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_tuple("HttpUri")
+ .field(&self.redacted_uri())
+ .finish()
+ }
+}
+
+/// A reusable HTTP redirect handled by the transport.
+///
+/// A transport that supports this extension may return it with a response after
+/// following redirects. Services can copy it into later request extensions to
+/// reuse that destination while keeping the original request URI for diagnostics.
+/// The transport must send to the destination and apply its redirect credential
+/// policy, including credentials supplied by client defaults.
+///
+/// The destination retains its original and redacted representations in
+/// [`HttpUri`]. Only the transport should use the original as a request URI.
+/// Services may inspect it to determine whether reuse is valid, but must use
+/// the redacted representation for diagnostics. This extension alone does not
+/// establish that the original path still selects the same object. Transports
+/// that do not support reusable redirects must not return it.
+#[derive(Clone)]
+pub struct HttpRedirect(HttpUri);
+
+impl HttpRedirect {
+ /// Record a destination that this transport can reuse.
+ pub fn new(uri: http::Uri) -> Self {
+ Self(HttpUri::new(uri.to_string()))
+ }
+
+ /// Return the destination's original and diagnostic representations.
+ pub fn uri(&self) -> &HttpUri {
+ &self.0
+ }
+}
+
+impl Debug for HttpRedirect {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_tuple("HttpRedirect")
+ .field(&self.0.redacted_uri())
+ .finish()
+ }
+}
+
+fn redact_uri(value: &str) -> String {
+ let mut value = value.to_string();
+ let authority_start = value
+ .split_once("://")
+ .filter(|(scheme, _)| {
+ scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https")
+ })
+ .map(|(scheme, _)| scheme.len() + 3)
+ .or_else(|| value.starts_with("//").then_some(2));
+ if let Some(start) = authority_start {
+ let end = value[start..]
+ .find(['/', '?', '#'])
+ .map_or(value.len(), |offset| start + offset);
+ if let Some(offset) = value[start..end].rfind('@') {
+ value.replace_range(start..start + offset, "[REDACTED]");
+ }
+ }
+
+ let end = value.find('#').unwrap_or(value.len());
+ if let Some(start) = value[..end].find('?') {
+ let query = value[start + 1..end]
+ .split('&')
+ .map(|pair| {
+ let Some((key, _)) = pair.split_once('=') else {
+ return pair.to_string();
+ };
+ let name = percent_decode_str(key).decode_utf8_lossy();
+ match name.to_ascii_lowercase().as_str() {
+ "signature"
+ | "policy"
+ | "token"
+ | "access_token"
+ | "x-amz-signature"
+ | "x-amz-security-token" => {
+ format!("{key}=[REDACTED]")
+ }
+ _ => pair.to_string(),
+ }
+ })
+ .collect::<Vec<_>>()
+ .join("&");
+ value.replace_range(start + 1..end, &query);
+ }
+ value
+}
diff --git a/core/core/src/types/mod.rs b/core/core/src/types/mod.rs
index a09861b73..dc2ea2c8e 100644
--- a/core/core/src/types/mod.rs
+++ b/core/core/src/types/mod.rs
@@ -25,8 +25,10 @@ pub(crate) mod compact;
mod http_transport;
pub use http_transport::HttpBody;
+pub use http_transport::HttpRedirect;
pub use http_transport::HttpTransport;
pub use http_transport::HttpTransporter;
+pub use http_transport::HttpUri;
mod entry;
pub use entry::Entry;
diff --git a/core/http-transports/reqwest/src/lib.rs b/core/http-transports/reqwest/src/lib.rs
index 1d211ca04..4e8bb667b 100644
--- a/core/http-transports/reqwest/src/lib.rs
+++ b/core/http-transports/reqwest/src/lib.rs
@@ -32,6 +32,7 @@ use opendal_core::Buffer;
use opendal_core::Error;
use opendal_core::ErrorKind;
use opendal_core::HttpBody;
+use opendal_core::HttpRedirect;
use opendal_core::HttpTransport;
use opendal_core::HttpTransporter;
use opendal_core::Result;
@@ -79,15 +80,43 @@ impl HttpTransport for ReqwestTransport {
let uri = req.uri().clone();
let is_head = req.method() == http::Method::HEAD;
- let (parts, body) = req.into_parts();
+ let (mut parts, body) = req.into_parts();
+ let original = uri.to_string();
+ let target = parts
+ .extensions
+ .get::<HttpRedirect>()
+ .map(|redirect| redirect.uri().original_uri())
+ .unwrap_or(&original);
- let url = reqwest::Url::parse(&uri.to_string()).map_err(|err| {
+ let url = reqwest::Url::parse(target).map_err(|err| {
Error::new(ErrorKind::Unexpected, "request url is invalid")
.with_operation("reqwest::fetch")
.with_context("url", uri.to_string())
.set_source(err)
})?;
+ if parts.extensions.get::<HttpRedirect>().is_some() {
+ let original = reqwest::Url::parse(&uri.to_string()).map_err(|err| {
+ Error::new(ErrorKind::Unexpected, "original request url is invalid").set_source(err)
+ })?;
+ if original.origin() != url.origin() {
+ // A reused redirect must not gain credentials from client
+ // defaults or cookie storage that a normal redirect strips.
+ for name in [
+ "authorization",
+ "proxy-authorization",
+ "cookie",
+ "cookie2",
+ "www-authenticate",
+ ] {
+ parts.headers.insert(
+ http::header::HeaderName::from_static(name),
+ http::HeaderValue::from_static(""),
+ );
+ }
+ }
+ }
+
let mut req_builder = self
.client
.request(parts.method, url)
@@ -116,7 +145,7 @@ impl HttpTransport for ReqwestTransport {
.with_operation("reqwest::send")
.with_context("url", uri.to_string())
.with_temporary(is_temporary_error(&err))
- .set_source(err)
+ .set_source(err.without_url())
})?;
// Get content length from header so that we can check it.
@@ -135,6 +164,19 @@ impl HttpTransport for ReqwestTransport {
// it later.
.extension(uri.clone());
+ // Optional metadata must not turn a successful fetch into an error.
+ if let Ok(target) = resp.url().as_str().parse::<http::Uri>()
+ && target != uri
+ {
+ let redirect = parts
+ .extensions
+ .get::<HttpRedirect>()
+ .filter(|redirect| redirect.uri().original_uri() == resp.url().as_str())
+ .cloned()
+ .unwrap_or_else(|| HttpRedirect::new(target));
+ hr = hr.extension(redirect);
+ }
+
// Response builder under wasm doesn't support set version.
#[cfg(not(target_arch = "wasm32"))]
{
@@ -153,7 +195,7 @@ impl HttpTransport for ReqwestTransport {
.with_operation("reqwest::fetch")
.with_context("url", uri.to_string())
.with_temporary(is_temporary_error(&err))
- .set_source(err)
+ .set_source(err.without_url())
}),
content_length,
);
diff --git a/core/services/hf/src/backend.rs b/core/services/hf/src/backend.rs
index 696d9499c..78de27333 100644
--- a/core/services/hf/src/backend.rs
+++ b/core/services/hf/src/backend.rs
@@ -133,6 +133,23 @@ impl HfBuilder {
self
}
+ /// Resolve every range through the Hugging Face Hub.
+ ///
+ /// Defaults to `false`: readers share resolved HTTP download addresses and
+ /// XET file metadata on the same backend. HTTP addresses refresh near expiry.
+ /// Set to `true` to resolve every range through the Hub in either mode.
+ /// This bypasses cached HTTP addresses and XET file metadata.
+ ///
+ /// Enable this when paths can be overwritten, deleted and reused, or moved
+ /// to different content by another client or a floating repository revision.
+ /// With reuse enabled, an issued download URL can remain usable until expiry
+ /// even after Hub permissions change. Separate authorization identities must
+ /// use separately constructed backends.
+ pub fn force_resolve(mut self, enabled: bool) -> Self {
+ self.config.force_resolve = enabled;
+ self
+ }
+
/// Set the Hub base URL.
///
/// Configure this when your organization uses a
@@ -266,16 +283,10 @@ impl Builder for HfBuilder {
let repo = HfRepo::new(repo_type, repo_id, Some(revision.clone()));
debug!("backend repo uri: {:?}", repo.uri(&root, ""));
+ let mut core = HfCore::build(info, capability, repo, root, token, endpoint, download_mode)?;
+ core.force_resolve = self.config.force_resolve;
Ok(HfBackend {
- core: Arc::new(HfCore::build(
- info,
- capability,
- repo,
- root,
- token,
- endpoint,
- download_mode,
- )?),
+ core: Arc::new(core),
})
}
}
diff --git a/core/services/hf/src/config.rs b/core/services/hf/src/config.rs
index ea7db3f96..44b13cbcd 100644
--- a/core/services/hf/src/config.rs
+++ b/core/services/hf/src/config.rs
@@ -60,6 +60,11 @@ pub struct HfConfig {
///
/// See <https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubdisablexet>.
pub download_mode: Option<HfDownloadMode>,
+ /// Resolve every range through the Hugging Face Hub. Defaults to `false`.
+ ///
+ /// Set to `true` to resolve every range through the Hub. See
+ /// [`HfBuilder::force_resolve`] for cache and freshness semantics.
+ pub force_resolve: bool,
}
impl Debug for HfConfig {
@@ -73,6 +78,7 @@ impl Debug for HfConfig {
.field("revision", &self.revision)
.field("root", &self.root)
.field("download_mode", &self.download_mode)
+ .field("force_resolve", &self.force_resolve)
.finish_non_exhaustive()
}
}
@@ -106,6 +112,19 @@ impl opendal_core::Configurator for HfConfig {
.map(|s| HfDownloadMode::parse(s))
.transpose()?;
+ let force_resolve = opts
+ .get("force_resolve")
+ .map(|value| value.parse::<bool>())
+ .transpose()
+ .map_err(|err| {
+ opendal_core::Error::new(
+ opendal_core::ErrorKind::ConfigInvalid,
+ "force_resolve must be true or false",
+ )
+ .set_source(err)
+ })?
+ .unwrap_or_default();
+
if !path.is_empty() {
// Full URI like "hf://datasets/user/repo@rev/path"
let parsed = HfUri::parse(&path)?;
@@ -117,6 +136,7 @@ impl opendal_core::Configurator for HfConfig {
token: opts.get("token").cloned(),
endpoint: opts.get("endpoint").cloned(),
download_mode,
+ force_resolve,
})
} else {
// Bare scheme from via_iter, all config is in options.
@@ -138,6 +158,7 @@ impl opendal_core::Configurator for HfConfig {
token: opts.get("token").cloned(),
endpoint: opts.get("endpoint").cloned(),
download_mode,
+ force_resolve,
})
}
}
diff --git a/core/services/hf/src/core.rs b/core/services/hf/src/core.rs
index 56f2c5979..0dc37f654 100644
--- a/core/services/hf/src/core.rs
+++ b/core/services/hf/src/core.rs
@@ -24,9 +24,11 @@ use http::StatusCode;
use http::header;
use log::debug;
use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
use std::fmt::Debug;
use std::sync::Arc;
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::sync::Mutex as StdMutex;
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use xet::xet_session::{XetDownloadStreamGroup, XetSession, XetSessionBuilder, XetUploadCommit};
@@ -295,6 +297,123 @@ pub(super) struct LastCommit {
pub date: String,
}
+pub(super) enum HfReadResponse {
+ Http(Response<HttpBody>),
+ Xet(XetFileResponse),
+}
+
+impl HfReadResponse {
+ async fn from_response(resp: Response<HttpBody>, mode: HfDownloadMode) -> Result<Self> {
+ if mode != HfDownloadMode::Xet || !resp.headers().contains_key("x-xet-hash") {
+ return Ok(Self::Http(resp));
+ }
+ let (_, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ let info = serde_json::from_reader(buf.reader()).map_err(new_json_deserialize_error)?;
+ Ok(Self::Xet(info))
+ }
+}
+
+// Bound entries, paths, and HTTP URIs. Busy entries are never evicted:
+// recreating their slots would allow two concurrent resolutions for one path.
+const RESOLVED_FILE_MAX_ENTRIES: usize = 512;
+const RESOLVED_FILE_MAX_PATH_BYTES: usize = 4096;
+const HTTP_DOWNLOAD_MAX_URI_BYTES: usize = 8192;
+const HTTP_DOWNLOAD_EXPIRY_MARGIN: Duration = Duration::from_secs(30);
+
+#[derive(Default)]
+struct ResolvedFiles {
+ entries: StdMutex<HashMap<String, ResolvedFileEntry>>,
+}
+
+struct ResolvedFileEntry {
+ accessed: Instant,
+ state: Arc<Mutex<ResolvedFile>>,
+}
+
+#[derive(Default)]
+enum ResolvedFile {
+ #[default]
+ Empty,
+ Http(Arc<HttpDownload>),
+ Xet(XetFileResponse),
+ // The path is not XET-backed and has no reusable HTTP destination.
+ NotXet,
+ // Wake existing waiters onto the uncached path when a response cannot be
+ // admitted. Its slot leaves the table so later reads can try again.
+ Bypass,
+}
+
+struct HttpDownload {
+ redirect: HttpRedirect,
+ valid_until: Instant,
+}
+
+impl ResolvedFiles {
+ fn entry(&self, path: &str) -> Option<Arc<Mutex<ResolvedFile>>> {
+ if path.len() > RESOLVED_FILE_MAX_PATH_BYTES {
+ return None;
+ }
+
+ let mut entries = self
+ .entries
+ .lock()
+ .expect("resolved file cache lock poisoned");
+ if entries.len() == RESOLVED_FILE_MAX_ENTRIES && !entries.contains_key(path) {
+ let oldest = entries
+ .iter()
+ .filter(|(_, entry)| Arc::strong_count(&entry.state) == 1)
+ .min_by_key(|(_, entry)| entry.accessed)
+ .map(|(path, _)| path.clone())?;
+ entries.remove(&oldest);
+ }
+ let entry = entries
+ .entry(path.to_string())
+ .or_insert_with(|| ResolvedFileEntry {
+ accessed: Instant::now(),
+ state: Arc::default(),
+ });
+ entry.accessed = Instant::now();
+ Some(entry.state.clone())
+ }
+}
+
+impl HttpDownload {
+ fn from_response(resp: &Response<HttpBody>) -> Option<Self> {
+ let redirect = resp.extensions().get::<HttpRedirect>()?;
+ let uri = redirect.uri().original_uri().parse::<http::Uri>().ok()?;
+ if uri.scheme_str() != Some("https")
+ || uri.authority()?.as_str().contains('@')
+ || redirect.uri().original_uri().len() > HTTP_DOWNLOAD_MAX_URI_BYTES
+ {
+ return None;
+ }
+
+ // HF's signed CDN destinations advertise their authorization deadline
+ // as Expires. Unknown signature formats fall back to fresh resolution.
+ let mut expirations = uri.query()?.split('&').filter_map(|pair| {
+ let (key, value) = pair.split_once('=')?;
+ (key == "Expires").then_some(value)
+ });
+ let expires = expirations.next()?.parse::<u64>().ok()?;
+ if expirations.next().is_some() {
+ return None;
+ }
+ let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?;
+ let lifetime = Duration::from_secs(expires)
+ .checked_sub(now)?
+ .checked_sub(HTTP_DOWNLOAD_EXPIRY_MARGIN)?;
+ if lifetime.is_zero() {
+ return None;
+ }
+
+ Some(Self {
+ redirect: redirect.clone(),
+ valid_until: Instant::now().checked_add(lifetime)?,
+ })
+ }
+}
+
// Core HuggingFace client that manages API interactions, authentication
// and shared logic for reader/writer/lister.
@@ -308,6 +427,9 @@ pub struct HfCore {
pub endpoint: String,
pub xet_session: XetSession,
pub download_mode: HfDownloadMode,
+ pub force_resolve: bool,
+ /// Shared file resolutions, keyed by paths relative to this core's root.
+ resolved_files: Arc<ResolvedFiles>,
/// Cached CAS read token, shared by every `XetDownloadStreamGroup` this
/// core creates, so at most one `xet-read-token` request happens per
/// token lifetime instead of one per group (one per file read).
@@ -348,6 +470,8 @@ impl HfCore {
endpoint,
xet_session,
download_mode,
+ force_resolve: false,
+ resolved_files: Arc::default(),
xet_read_token: Arc::new(Mutex::new(None)),
xet_write_token: Arc::new(Mutex::new(None)),
}
@@ -522,14 +646,14 @@ impl HfCore {
op: Operation,
service_operation: &'static str,
) -> Result<http::request::Builder> {
- // Every outbound HF/CAS request passes through here -- enable with
- // `RUST_LOG=opendal_service_hf::core=debug` to see what's queried.
+ let url = HttpUri::new(url);
debug!(
- "hf request: service_operation={service_operation} operation={op} method={method} url={url}"
+ "hf request: service_operation={service_operation} operation={op} method={method} url={}",
+ url.redacted_uri()
);
let mut req = Request::builder()
.method(method)
- .uri(url)
+ .uri(url.original_uri())
.extension(op)
.extension(ServiceOperation(service_operation));
match &self.token {
@@ -596,11 +720,13 @@ impl HfCore {
format!("{}{path}", self.endpoint)
};
- debug!("hf request redirected: url={} -> {target}", retry.uri());
+ let target = HttpUri::new(target);
+ debug!("hf request redirected: url={}", target.redacted_uri());
let (mut parts, body) = retry.into_parts();
parts.uri = target
+ .original_uri()
.parse()
- .map_err(|err| new_http_uri_invalid_error(err).with_context("location", &target))?;
+ .map_err(new_http_uri_invalid_error)?;
ctx.http_transport()
.fetch(Request::from_parts(parts, body))
.await
@@ -645,6 +771,125 @@ impl HfCore {
Ok(files.remove(0))
}
+ pub(super) async fn read(
+ &self,
+ ctx: &OperationContext,
+ path: &str,
+ range: BytesRange,
+ ) -> Result<HfReadResponse> {
+ let entry = if self.force_resolve {
+ None
+ } else {
+ self.resolved_files.entry(path)
+ };
+ let Some(entry) = entry else {
+ let resp = self.resolve(ctx, path, range, self.download_mode).await?;
+ return HfReadResponse::from_response(resp, self.download_mode).await;
+ };
+
+ let mut rejected: Option<Arc<HttpDownload>> = None;
+ loop {
+ let destination = {
+ let mut state = entry.lock().await;
+ match &*state {
+ ResolvedFile::Xet(info) => return Ok(HfReadResponse::Xet(info.clone())),
+ ResolvedFile::NotXet | ResolvedFile::Bypass => {
+ drop(state);
+ let resp = self.resolve(ctx, path, range, HfDownloadMode::Http).await?;
+ return Ok(HfReadResponse::Http(resp));
+ }
+ ResolvedFile::Http(current)
+ if current.valid_until > Instant::now()
+ && !rejected
+ .as_ref()
+ .is_some_and(|old| Arc::ptr_eq(old, current)) =>
+ {
+ current.clone()
+ }
+ _ => {
+ let mode = if matches!(&*state, ResolvedFile::Http(_)) {
+ HfDownloadMode::Http
+ } else {
+ self.download_mode
+ };
+ // Serialize resolution and refresh. XET metadata is
+ // independent of the range; a one-byte probe bounds the
+ // body discarded when the file turns out to use HTTP.
+ *state = ResolvedFile::Empty;
+ let resolve_range = if mode == HfDownloadMode::Xet {
+ BytesRange::new(0, Some(1))
+ } else {
+ range
+ };
+ let resp = self.resolve(ctx, path, resolve_range, mode).await?;
+ let resp = match HfReadResponse::from_response(resp, mode).await? {
+ HfReadResponse::Xet(info) => {
+ *state = ResolvedFile::Xet(info.clone());
+ return Ok(HfReadResponse::Xet(info));
+ }
+ HfReadResponse::Http(resp) => resp,
+ };
+ *state = match HttpDownload::from_response(&resp) {
+ Some(download) => ResolvedFile::Http(Arc::new(download)),
+ None if self.download_mode == HfDownloadMode::Xet => {
+ ResolvedFile::NotXet
+ }
+ None => {
+ // This live slot cannot have been evicted or
+ // replaced while we hold an Arc to it.
+ self.resolved_files
+ .entries
+ .lock()
+ .expect("resolved file cache lock poisoned")
+ .remove(path);
+ ResolvedFile::Bypass
+ }
+ };
+ if mode == HfDownloadMode::Http {
+ // The response already contains the requested range.
+ return Ok(HfReadResponse::Http(resp));
+ }
+ // The XET probe returned one byte of HTTP content.
+ // Reuse its destination to fetch the caller's range.
+ continue;
+ }
+ }
+ };
+
+ let url = self
+ .repo
+ .uri(&self.root, path)
+ .resolve_url(&self.endpoint, self.repo.revision());
+ let mut req = self
+ .request(http::Method::GET, &url, Operation::Read, "Download")?
+ .extension(destination.redirect.clone());
+ if !range.is_full() {
+ req = req.header(header::RANGE, range.to_header());
+ }
+ let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+ let resp = ctx.http_transport().fetch(req).await?;
+ if resp.status().is_success() {
+ return Ok(HfReadResponse::Http(resp));
+ }
+ if rejected.is_none()
+ && matches!(
+ resp.status(),
+ StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
+ )
+ {
+ // A late rejection must not discard another caller's refreshed
+ // destination. The next iteration compares the Arc generation.
+ rejected = Some(destination);
+ continue;
+ }
+ let (parts, _) = resp.into_parts();
+ return Err(parse_error(
+ ErrorContext::new(ServiceOperation("Download")),
+ parts,
+ ));
+ }
+ }
+
/// Send `GET /resolve` and return the raw streaming response.
///
/// In `Xet` mode adds `Accept: application/vnd.xet-fileinfo+json` so the
@@ -1618,7 +1863,8 @@ impl ErrorContext {
}
/// Parse an error response using its service request context.
-pub(crate) fn parse_error(ctx: ErrorContext, parts: http::response::Parts) -> Error {
+pub(crate) fn parse_error(ctx: ErrorContext, mut parts: http::response::Parts) -> Error {
+ let location = HttpUri::from_response_location(&mut parts).cloned();
// HF sets x-error-message on every error response with a short human-readable
// description. Using the header avoids reading the response body, which can be
// a large HTML error page (e.g. 52 KB on 404s from the /resolve/ endpoint).
@@ -1626,15 +1872,10 @@ pub(crate) fn parse_error(ctx: ErrorContext, parts: http::response::Parts) -> Er
.headers
.get("x-error-message")
.and_then(|v| v.to_str().ok());
- let location = parts
- .headers
- .get(header::LOCATION)
- .and_then(|v| v.to_str().ok());
let message = match (error_message, location) {
(Some(message), _) => message.to_string(),
- // A 3xx only reaches here when `HfCore::send` refused to follow it,
- // so name the destination instead of "unknown error".
(None, Some(location)) if parts.status.is_redirection() => {
+ let location = location.redacted_uri();
format!("redirect to {location} not followed")
}
_ => "unknown error".to_string(),
diff --git a/core/services/hf/src/docs.md b/core/services/hf/src/docs.md
index 4493bbab7..9cc121794 100644
--- a/core/services/hf/src/docs.md
+++ b/core/services/hf/src/docs.md
@@ -32,6 +32,36 @@ Use [`crate::HfConfig`] for serializable configuration and this builder's
methods for direct construction. The field and method documentation defines
accepted values, defaults, and environment interaction.
+### Reading files that can change
+
+OpenDAL assumes that previously written files are not modified. Each backend
+shares resolved HTTP download addresses and XET file metadata across readers and
+batches. If files can change, enable
+[`force_resolve`](crate::Hf::force_resolve) to resolve every
+range through the Hub:
+
+```rust,no_run
+let builder = opendal_service_hf::Hf::default()
+ .repo_type("bucket")
+ .repo_id("username/my-bucket")
+ .force_resolve(true);
+```
+
+The option controls read freshness, not permission to write. It also applies to
+paths changed by other clients or a floating repository revision. With the
+option disabled, changed files can remain invisible while a resolved address or
+XET file metadata is reused. Issued signed URLs can also remain usable after Hub
+permissions change.
+
+The shared cache retains bounded in-memory metadata. HTTP addresses refresh
+30 seconds before their signed expiry, on the next read, or once after a cached
+request returns 401 or 403. In HTTP mode, the first resolve returns its body
+directly. Responses without supported redirect metadata or expiry continue to
+resolve normally.
+The reqwest transport supports the optional
+[`HttpRedirect`](opendal_core::HttpRedirect) extension; custom transports may omit it.
+Separate authorization identities require separately constructed backends.
+
## Examples
### Via Builder (Git-based dataset)
diff --git a/core/services/hf/src/reader.rs b/core/services/hf/src/reader.rs
index 7ea4afbc7..7b0379f80 100644
--- a/core/services/hf/src/reader.rs
+++ b/core/services/hf/src/reader.rs
@@ -16,9 +16,10 @@
// under the License.
use super::backend::*;
-use super::core::{HfCore, HfDownloadMode, XetFileResponse};
+#[cfg(test)]
+use super::core::HfDownloadMode;
+use super::core::{HfCore, HfReadResponse};
use asyncband::once::OnceCell;
-use bytes::Buf;
use http::Response;
use opendal_core::raw::*;
use opendal_core::*;
@@ -45,8 +46,7 @@ fn xet_range(range: BytesRange) -> Option<Range<u64>> {
impl HfReadStream {
/// Build the stream directly from an already-known XET hash + size,
- /// whether that came from a resolve response just parsed in [`HfReader::open`]
- /// or was reused from its cache.
+ /// resolved or reused by [`HfCore`].
async fn new_xet(
group: &XetDownloadStreamGroup,
hash: &str,
@@ -79,22 +79,6 @@ impl HfReadStream {
let metadata = parse_into_metadata(path, resp.headers())?;
Ok((RpRead::new(metadata), Self::Http(resp.into_body())))
}
-
- /// Retrieve a resolve response's XET metadata, or hand the response
- /// back untouched if it doesn't carry any -- its body is then the
- /// actual range content, still there for [`Self::new_http`].
- async fn maybe_xet(
- resp: Response<HttpBody>,
- ) -> Result<std::result::Result<XetFileResponse, Response<HttpBody>>> {
- if !resp.headers().contains_key("x-xet-hash") {
- return Ok(Err(resp));
- }
- let (_, mut body) = resp.into_parts();
- let buf = body.to_buffer().await?;
- let info: XetFileResponse =
- serde_json::from_reader(buf.reader()).map_err(new_json_deserialize_error)?;
- Ok(Ok(info))
- }
}
fn map_session_error(e: SessionError) -> Error {
@@ -119,16 +103,6 @@ pub struct HfReader {
backend: HfBackend,
ctx: OperationContext,
path: String,
- // `Some((hash, size))`/`None` once this path is known to be (or not be)
- // XET-backed; only consulted in `HfDownloadMode::Xet`. Single-flighted via
- // `get_or_try_init`, so concurrent cold opens share one probe and every
- // later range reuses it. If not XET-backed, every open falls back to its
- // own fresh resolve -- one wasted request per reader, not per range.
- //
- // Never re-validated for this reader's life, so a file changed mid-read
- // (e.g. a new commit on a floating `main`) can serve stale content --
- // scoped to one reader, not the `Operator`, to bound that window.
- resolved: OnceCell<Option<(String, u64)>>,
// Built once per reader and reused for every later XET range, seeded
// from `HfCore`'s cached CAS token. Reuse across concurrent ranges
// relies on the `xet` crate's documented (not type-enforced) support
@@ -142,7 +116,6 @@ impl HfReader {
backend,
ctx,
path: path.to_string(),
- resolved: OnceCell::new(),
xet_group: OnceCell::new(),
}
}
@@ -169,54 +142,11 @@ impl oio::StreamRead for HfReader {
let core = &self.backend.core;
let path = self.path.as_str();
- if core.download_mode != HfDownloadMode::Xet {
- // Http mode: resolve() is itself the byte-fetching request, done
- // fresh per range -- there is no metadata step to cache. HF puts
- // `x-xet-hash` on the 302 it answers with, not on the CDN
- // response the transport follows it to, so there is nothing to
- // classify here either.
- let resp = core
- .resolve(&self.ctx, path, range, core.download_mode)
- .await?;
- let (rp, stream) = HfReadStream::new_http(path, resp)?;
- return Ok((rp, Box::new(stream) as Box<dyn oio::ReadStreamDyn>));
- }
-
- // Xet mode: classification is single-flighted, see `resolved`'s
- // doc comment for why that's free for the XET outcome and cheap
- // (one extra request, once per reader) for the NotXet one.
- //
- // The probe asks for one byte rather than the caller's range. HF
- // returns whole-file XET metadata regardless of range, so one shared
- // response is correct for every range; and when the path turns out
- // not to be XET-backed the discarded body is a byte instead of the
- // caller's range.
- let classification = self
- .resolved
- .get_or_try_init(|| async {
- let resp = core
- .resolve(
- &self.ctx,
- path,
- BytesRange::new(0, Some(1)),
- HfDownloadMode::Xet,
- )
- .await?;
- match HfReadStream::maybe_xet(resp).await? {
- Ok(info) => Ok(Some((info.hash, info.size))),
- Err(_) => Ok(None),
- }
- })
- .await?;
-
- let (rp, stream) = match classification {
- Some((hash, size)) => self.xet_stream(core, hash, *size, range).await?,
- None => {
- let resp = core
- .resolve(&self.ctx, path, range, HfDownloadMode::Http)
- .await?;
- HfReadStream::new_http(path, resp)?
+ let (rp, stream) = match core.read(&self.ctx, path, range).await? {
+ HfReadResponse::Xet(info) => {
+ self.xet_stream(core, &info.hash, info.size, range).await?
}
+ HfReadResponse::Http(resp) => HfReadStream::new_http(path, resp)?,
};
Ok((rp, Box::new(stream) as Box<dyn oio::ReadStreamDyn>))
diff --git a/website/data/services.json b/website/data/services.json
index 696aff916..fb4d2ab93 100644
--- a/website/data/services.json
+++ b/website/data/services.json
@@ -2308,6 +2308,13 @@
"required": false,
"group": "General",
"comments": "Download mode. Either `xet` (default) or `http`.\n\nWhen unset, the mode is resolved from the `HF_HUB_DISABLE_XET`\nenvironment variable: a non-empty value forces `http`, otherwise it\ndefaults to `xet`. An explicit value here takes precedence.\n\nSee <https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubdisablexet>."
+ },
+ {
+ "name": "force_resolve",
+ "type": "bool",
+ "required": false,
+ "group": "General",
+ "comments": "Resolve every range through the Hugging Face Hub. Defaults to `false`.\n\nSet to `true` to resolve every range through the Hub. See\n[`HfBuilder::force_resolve`] for cache and freshness semantics."
}
],
"examples": [
@@ -2315,31 +2322,31 @@
"binding": "rust",
"language": "rust",
"minimal": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"hf\", [\n])?;",
- "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"hf\", [\n // Repo type of this backend. Required.\n // (\"repo_type\".to_string(), \"...\".to_string()),\n\n // Repo id of this backend.\n //\n // This is required.\n // (\"repo_id\".to_string(), \"...\".to_string()),\n\n // Revision of this backend.\n //\n // Default is main.\n // (\"revision\".to_string(), \"...\".to_string()),\n\n // Root of this backend. Can be \"/path/to [...]
+ "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"hf\", [\n // Repo type of this backend. Required.\n // (\"repo_type\".to_string(), \"...\".to_string()),\n\n // Repo id of this backend.\n //\n // This is required.\n // (\"repo_id\".to_string(), \"...\".to_string()),\n\n // Revision of this backend.\n //\n // Default is main.\n // (\"revision\".to_string(), \"...\".to_string()),\n\n // Root of this backend. Can be \"/path/to [...]
},
{
"binding": "java",
"language": "java",
"minimal": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap<String, String> config = new HashMap<>();\nOperator operator = Operator.of(\"hf\", config);",
- "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap<String, String> config = new HashMap<>();\n// Repo type of this backend. Required.\n// config.put(\"repo_type\", \"...\");\n\n// Repo id of this backend.\n//\n// This is required.\n// config.put(\"repo_id\", \"...\");\n\n// Revision of this backend.\n//\n// Default is main.\n// config.put(\"revision\", \"...\");\n\n// Root of this backend. Can be \"/path/to/dir\".\n//\n// Defau [...]
+ "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap<String, String> config = new HashMap<>();\n// Repo type of this backend. Required.\n// config.put(\"repo_type\", \"...\");\n\n// Repo id of this backend.\n//\n// This is required.\n// config.put(\"repo_id\", \"...\");\n\n// Revision of this backend.\n//\n// Default is main.\n// config.put(\"revision\", \"...\");\n\n// Root of this backend. Can be \"/path/to/dir\".\n//\n// Defau [...]
},
{
"binding": "python",
"language": "python",
"minimal": "import opendal\n\noperator = opendal.Operator(\n \"hf\",\n)",
- "full": "import opendal\n\noperator = opendal.Operator(\n \"hf\",\n # Repo type of this backend. Required.\n # repo_type=\"...\",\n\n # Repo id of this backend.\n #\n # This is required.\n # repo_id=\"...\",\n\n # Revision of this backend.\n #\n # Default is main.\n # revision=\"...\",\n\n # Root of this backend. Can be \"/path/to/dir\".\n #\n # Default is \"/\".\n # root=\"...\",\n\n # Token of this backend.\n #\n # This is [...]
+ "full": "import opendal\n\noperator = opendal.Operator(\n \"hf\",\n # Repo type of this backend. Required.\n # repo_type=\"...\",\n\n # Repo id of this backend.\n #\n # This is required.\n # repo_id=\"...\",\n\n # Revision of this backend.\n #\n # Default is main.\n # revision=\"...\",\n\n # Root of this backend. Can be \"/path/to/dir\".\n #\n # Default is \"/\".\n # root=\"...\",\n\n # Token of this backend.\n #\n # This is [...]
},
{
"binding": "nodejs",
"language": "javascript",
"minimal": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"hf\", {\n});",
- "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"hf\", {\n // Repo type of this backend. Required.\n // repo_type: \"...\",\n\n // Repo id of this backend.\n //\n // This is required.\n // repo_id: \"...\",\n\n // Revision of this backend.\n //\n // Default is main.\n // revision: \"...\",\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // root: \"...\",\n\n // Token of this backend.\n //\n // [...]
+ "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"hf\", {\n // Repo type of this backend. Required.\n // repo_type: \"...\",\n\n // Repo id of this backend.\n //\n // This is required.\n // repo_id: \"...\",\n\n // Revision of this backend.\n //\n // Default is main.\n // revision: \"...\",\n\n // Root of this backend. Can be \"/path/to/dir\".\n //\n // Default is \"/\".\n // root: \"...\",\n\n // Token of this backend.\n //\n // [...]
},
{
"binding": "go",
"language": "go",
"minimal": "import (\n\t\"github.com/apache/opendal-go-services/hf\"\n\topendal \"github.com/apache/opendal/bindings/go\"\n)\n\noperator, err := opendal.NewOperator(hf.Scheme, opendal.OperatorOptions{\n})",
- "full": "import (\n\t\"github.com/apache/opendal-go-services/hf\"\n\topendal \"github.com/apache/opendal/bindings/go\"\n)\n\noperator, err := opendal.NewOperator(hf.Scheme, opendal.OperatorOptions{\n\t// Repo type of this backend. Required.\n\t// \"repo_type\": \"...\",\n\n\t// Repo id of this backend.\n\t//\n\t// This is required.\n\t// \"repo_id\": \"...\",\n\n\t// Revision of this backend.\n\t//\n\t// Default is main.\n\t// \"revision\": \"...\",\n\n\t// Root of this backend [...]
+ "full": "import (\n\t\"github.com/apache/opendal-go-services/hf\"\n\topendal \"github.com/apache/opendal/bindings/go\"\n)\n\noperator, err := opendal.NewOperator(hf.Scheme, opendal.OperatorOptions{\n\t// Repo type of this backend. Required.\n\t// \"repo_type\": \"...\",\n\n\t// Repo id of this backend.\n\t//\n\t// This is required.\n\t// \"repo_id\": \"...\",\n\n\t// Revision of this backend.\n\t//\n\t// Default is main.\n\t// \"revision\": \"...\",\n\n\t// Root of this backend [...]
}
]
},