You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/10/14 08:13:32 UTC
(opendal) 01/04: feat: Add from_uri support for all object storage services
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch registry-more
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 505ca44d8345a1158010607cf9b42adfb3ef8fcf
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Oct 1 17:00:18 2025 +0800
feat: Add from_uri support for all object storage services
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/azblob/backend.rs | 90 ++++++++++++++++++++++++++++++++++++-
core/src/services/azblob/mod.rs | 2 +-
core/src/services/b2/backend.rs | 81 ++++++++++++++++++++++++++++++++-
core/src/services/b2/mod.rs | 2 +-
core/src/services/cos/backend.rs | 78 +++++++++++++++++++++++++++++++-
core/src/services/cos/mod.rs | 2 +-
core/src/services/fs/backend.rs | 15 +++++++
core/src/services/gcs/backend.rs | 79 +++++++++++++++++++++++++++++++-
core/src/services/gcs/mod.rs | 2 +-
core/src/services/memory/backend.rs | 10 +++++
core/src/services/obs/backend.rs | 77 ++++++++++++++++++++++++++++++-
core/src/services/obs/mod.rs | 2 +-
core/src/services/oss/backend.rs | 78 +++++++++++++++++++++++++++++++-
core/src/services/oss/mod.rs | 2 +-
core/src/services/s3/backend.rs | 11 +++++
core/src/services/upyun/backend.rs | 79 +++++++++++++++++++++++++++++++-
core/src/services/upyun/mod.rs | 2 +-
core/src/types/operator/registry.rs | 14 ++++++
18 files changed, 605 insertions(+), 21 deletions(-)
diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs
index 780caee62..72b7436cc 100644
--- a/core/src/services/azblob/backend.rs
+++ b/core/src/services/azblob/backend.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
@@ -23,7 +24,9 @@ use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use http::Response;
use http::StatusCode;
+use http::Uri;
use log::debug;
+use percent_encoding::percent_decode_str;
use reqsign::AzureStorageConfig;
use reqsign::AzureStorageLoader;
use reqsign::AzureStorageSigner;
@@ -38,7 +41,7 @@ use super::error::parse_error;
use super::lister::AzblobLister;
use super::writer::AzblobWriter;
use super::writer::AzblobWriters;
-use super::DEFAULT_SCHEME;
+use super::AZBLOB_SCHEME;
use crate::raw::*;
use crate::services::AzblobConfig;
use crate::*;
@@ -59,6 +62,65 @@ impl From<AzureStorageConfig> for AzblobConfig {
impl Configurator for AzblobConfig {
type Builder = AzblobBuilder;
+ fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> {
+ let mut map = options.clone();
+
+ let has_container = map.get("container").map(|v| !v.is_empty()).unwrap_or(false);
+
+ let path = percent_decode_str(uri.path()).decode_utf8_lossy();
+
+ if has_container {
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ } else if let Some(authority) = uri.authority() {
+ let host = authority.host();
+ if host.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "azblob uri requires container",
+ ));
+ }
+ map.insert("container".to_string(), host.to_string());
+
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ } else {
+ let trimmed = path.trim_matches('/');
+ if trimmed.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "azblob uri requires container",
+ ));
+ }
+
+ let mut segments = trimmed.splitn(2, '/');
+ let container = segments.next().unwrap();
+ if container.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "azblob uri requires container",
+ ));
+ }
+ map.insert("container".to_string(), container.to_string());
+
+ if let Some(rest) = segments.next() {
+ if !rest.is_empty() && !map.contains_key("root") {
+ map.insert("root".to_string(), rest.to_string());
+ }
+ }
+ }
+
+ Self::from_iter(map)
+ }
+
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
AzblobBuilder {
@@ -397,7 +459,7 @@ impl Builder for AzblobBuilder {
core: Arc::new(AzblobCore {
info: {
let am = AccessorInfo::default();
- am.set_scheme(DEFAULT_SCHEME)
+ am.set_scheme(AZBLOB_SCHEME)
.set_root(&root)
.set_name(container)
.set_native_capability(Capability {
@@ -592,3 +654,27 @@ impl Access for AzblobBackend {
)))
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
+
+ #[test]
+ fn from_uri_with_host_container() {
+ let uri: Uri = "azblob://my-container/path/to/root".parse().unwrap();
+ let cfg = AzblobConfig::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.container, "my-container");
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ }
+
+ #[test]
+ fn from_uri_with_path_container() {
+ let uri: Uri = "azblob:///my-container/nested/root".parse().unwrap();
+ let cfg = AzblobConfig::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.container, "my-container");
+ assert_eq!(cfg.root.as_deref(), Some("nested/root"));
+ }
+}
diff --git a/core/src/services/azblob/mod.rs b/core/src/services/azblob/mod.rs
index 818ed4628..f014a29ef 100644
--- a/core/src/services/azblob/mod.rs
+++ b/core/src/services/azblob/mod.rs
@@ -17,7 +17,7 @@
/// Default scheme for azblob service.
#[cfg(feature = "services-azblob")]
-pub(super) const DEFAULT_SCHEME: &str = "azblob";
+pub const AZBLOB_SCHEME: &str = "azblob";
#[cfg(feature = "services-azblob")]
pub(crate) mod core;
#[cfg(feature = "services-azblob")]
diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs
index f1674a03b..868039c8f 100644
--- a/core/src/services/b2/backend.rs
+++ b/core/src/services/b2/backend.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
@@ -22,7 +23,9 @@ use std::sync::Arc;
use http::Request;
use http::Response;
use http::StatusCode;
+use http::Uri;
use log::debug;
+use percent_encoding::percent_decode_str;
use tokio::sync::RwLock;
use super::core::constants;
@@ -34,13 +37,67 @@ use super::error::parse_error;
use super::lister::B2Lister;
use super::writer::B2Writer;
use super::writer::B2Writers;
-use super::DEFAULT_SCHEME;
+use super::B2_SCHEME;
use crate::raw::*;
use crate::services::B2Config;
use crate::*;
impl Configurator for B2Config {
type Builder = B2Builder;
+ fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> {
+ let mut map = options.clone();
+
+ let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false);
+
+ let path = percent_decode_str(uri.path()).decode_utf8_lossy();
+
+ if has_bucket {
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ } else if let Some(authority) = uri.authority() {
+ let host = authority.host();
+ if !host.is_empty() {
+ map.insert("bucket".to_string(), host.to_string());
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ }
+ }
+
+ if !map.contains_key("bucket") {
+ let trimmed = path.trim_matches('/');
+ if trimmed.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "b2 uri requires bucket",
+ ));
+ }
+ let mut segments = trimmed.splitn(2, '/');
+ let bucket = segments.next().unwrap();
+ if bucket.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "b2 uri requires bucket",
+ ));
+ }
+ map.insert("bucket".to_string(), bucket.to_string());
+ if let Some(rest) = segments.next() {
+ if !rest.is_empty() && !map.contains_key("root") {
+ map.insert("root".to_string(), rest.to_string());
+ }
+ }
+ }
+
+ Self::from_iter(map)
+ }
+
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
B2Builder {
@@ -191,7 +248,7 @@ impl Builder for B2Builder {
core: Arc::new(B2Core {
info: {
let am = AccessorInfo::default();
- am.set_scheme(DEFAULT_SCHEME)
+ am.set_scheme(B2_SCHEME)
.set_root(&root)
.set_native_capability(Capability {
stat: true,
@@ -432,3 +489,23 @@ impl Access for B2Backend {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
+
+ #[test]
+ fn from_uri_extracts_bucket_and_root() {
+ let uri: Uri = "b2://example-bucket/path/to/root".parse().unwrap();
+ let mut options = HashMap::new();
+ options.insert("bucket_id".to_string(), "bucket-id".to_string());
+
+ let cfg = B2Config::from_uri(&uri, &options).unwrap();
+ assert_eq!(cfg.bucket, "example-bucket");
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ assert_eq!(cfg.bucket_id, "bucket-id");
+ }
+}
diff --git a/core/src/services/b2/mod.rs b/core/src/services/b2/mod.rs
index 09c6b4bfe..6a4778994 100644
--- a/core/src/services/b2/mod.rs
+++ b/core/src/services/b2/mod.rs
@@ -17,7 +17,7 @@
/// Default scheme for b2 service.
#[cfg(feature = "services-b2")]
-pub(super) const DEFAULT_SCHEME: &str = "b2";
+pub const B2_SCHEME: &str = "b2";
#[cfg(feature = "services-b2")]
mod core;
#[cfg(feature = "services-b2")]
diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs
index da2d29804..d1750fb5a 100644
--- a/core/src/services/cos/backend.rs
+++ b/core/src/services/cos/backend.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::HashMap;
use std::fmt::Debug;
use std::sync::Arc;
@@ -22,6 +23,7 @@ use http::Response;
use http::StatusCode;
use http::Uri;
use log::debug;
+use percent_encoding::percent_decode_str;
use reqsign::TencentCosConfig;
use reqsign::TencentCosCredentialLoader;
use reqsign::TencentCosSigner;
@@ -34,7 +36,7 @@ use super::lister::CosListers;
use super::lister::CosObjectVersionsLister;
use super::writer::CosWriter;
use super::writer::CosWriters;
-use super::DEFAULT_SCHEME;
+use super::COS_SCHEME;
use crate::raw::oio::PageLister;
use crate::raw::*;
use crate::services::CosConfig;
@@ -42,6 +44,62 @@ use crate::*;
impl Configurator for CosConfig {
type Builder = CosBuilder;
+ fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> {
+ let mut map = options.clone();
+
+ let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false);
+
+ let path = percent_decode_str(uri.path()).decode_utf8_lossy();
+
+ if has_bucket {
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ } else if let Some(authority) = uri.authority() {
+ let host = authority.host();
+ if !host.is_empty() {
+ map.insert("bucket".to_string(), host.to_string());
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ }
+ }
+
+ let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false);
+
+ if !has_bucket {
+ let trimmed = path.trim_matches('/');
+ if trimmed.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "cos uri requires bucket",
+ ));
+ }
+ let mut segments = trimmed.splitn(2, '/');
+ let bucket = segments.next().unwrap();
+ if bucket.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "cos uri requires bucket",
+ ));
+ }
+ map.insert("bucket".to_string(), bucket.to_string());
+ if let Some(rest) = segments.next() {
+ if !rest.is_empty() && !map.contains_key("root") {
+ map.insert("root".to_string(), rest.to_string());
+ }
+ }
+ }
+
+ Self::from_iter(map)
+ }
+
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
CosBuilder {
@@ -221,7 +279,7 @@ impl Builder for CosBuilder {
core: Arc::new(CosCore {
info: {
let am = AccessorInfo::default();
- am.set_scheme(DEFAULT_SCHEME)
+ am.set_scheme(COS_SCHEME)
.set_root(&root)
.set_name(&bucket)
.set_native_capability(Capability {
@@ -440,3 +498,19 @@ impl Access for CosBackend {
)))
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
+
+ #[test]
+ fn from_uri_extracts_bucket_and_root() {
+ let uri: Uri = "cos://example-bucket/path/to/root".parse().unwrap();
+ let cfg = CosConfig::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.bucket.as_deref(), Some("example-bucket"));
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ }
+}
diff --git a/core/src/services/cos/mod.rs b/core/src/services/cos/mod.rs
index 486597aa2..0de2ff00d 100644
--- a/core/src/services/cos/mod.rs
+++ b/core/src/services/cos/mod.rs
@@ -17,7 +17,7 @@
/// Default scheme for cos service.
#[cfg(feature = "services-cos")]
-pub(super) const DEFAULT_SCHEME: &str = "cos";
+pub const COS_SCHEME: &str = "cos";
#[cfg(feature = "services-cos")]
mod core;
#[cfg(feature = "services-cos")]
diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs
index ab0332020..93d0e40fd 100644
--- a/core/src/services/fs/backend.rs
+++ b/core/src/services/fs/backend.rs
@@ -297,3 +297,18 @@ impl Access for FsBackend {
Ok(RpRename::default())
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
+
+ #[test]
+ fn from_uri_extracts_root() {
+ let uri: Uri = "fs:///tmp/data".parse().unwrap();
+ let cfg = FsConfig::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.root.as_deref(), Some("/tmp/data"));
+ }
+}
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index d93efbce5..00848905d 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -15,13 +15,16 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
use http::Response;
use http::StatusCode;
+use http::Uri;
use log::debug;
+use percent_encoding::percent_decode_str;
use reqsign::GoogleCredentialLoader;
use reqsign::GoogleSigner;
use reqsign::GoogleTokenLoad;
@@ -33,7 +36,7 @@ use super::error::parse_error;
use super::lister::GcsLister;
use super::writer::GcsWriter;
use super::writer::GcsWriters;
-use super::DEFAULT_SCHEME;
+use super::GCS_SCHEME;
use crate::raw::oio::BatchDeleter;
use crate::raw::*;
use crate::services::GcsConfig;
@@ -44,6 +47,62 @@ const DEFAULT_GCS_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read
impl Configurator for GcsConfig {
type Builder = GcsBuilder;
+ fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> {
+ let mut map = options.clone();
+
+ let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true);
+
+ let path = percent_decode_str(uri.path()).decode_utf8_lossy();
+
+ if has_bucket {
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ } else if let Some(authority) = uri.authority() {
+ let host = authority.host();
+ if !host.is_empty() {
+ map.insert("bucket".to_string(), host.to_string());
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ }
+ }
+
+ let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true);
+
+ if !has_bucket {
+ let trimmed = path.trim_matches('/');
+ if trimmed.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "gcs uri requires bucket",
+ ));
+ }
+ let mut segments = trimmed.splitn(2, '/');
+ let bucket = segments.next().unwrap();
+ if bucket.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "gcs uri requires bucket",
+ ));
+ }
+ map.insert("bucket".to_string(), bucket.to_string());
+ if let Some(rest) = segments.next() {
+ if !rest.is_empty() && !map.contains_key("root") {
+ map.insert("root".to_string(), rest.to_string());
+ }
+ }
+ }
+
+ Self::from_iter(map)
+ }
+
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
GcsBuilder {
@@ -307,7 +366,7 @@ impl Builder for GcsBuilder {
core: Arc::new(GcsCore {
info: {
let am = AccessorInfo::default();
- am.set_scheme(DEFAULT_SCHEME)
+ am.set_scheme(GCS_SCHEME)
.set_root(&root)
.set_name(bucket)
.set_native_capability(Capability {
@@ -497,3 +556,19 @@ impl Access for GcsBackend {
)))
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
+
+ #[test]
+ fn from_uri_extracts_bucket_and_root() {
+ let uri: Uri = "gcs://example-bucket/path/to/root".parse().unwrap();
+ let cfg = GcsConfig::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.bucket, "example-bucket");
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ }
+}
diff --git a/core/src/services/gcs/mod.rs b/core/src/services/gcs/mod.rs
index a4d5d9712..0c8427158 100644
--- a/core/src/services/gcs/mod.rs
+++ b/core/src/services/gcs/mod.rs
@@ -17,7 +17,7 @@
/// Default scheme for gcs service.
#[cfg(feature = "services-gcs")]
-pub(super) const DEFAULT_SCHEME: &str = "gcs";
+pub const GCS_SCHEME: &str = "gcs";
#[cfg(feature = "services-gcs")]
mod core;
#[cfg(feature = "services-gcs")]
diff --git a/core/src/services/memory/backend.rs b/core/src/services/memory/backend.rs
index 8c1d0d1cb..b1108fd63 100644
--- a/core/src/services/memory/backend.rs
+++ b/core/src/services/memory/backend.rs
@@ -193,6 +193,9 @@ impl Access for MemoryAccessor {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
#[test]
fn test_accessor_metadata_name() {
@@ -202,4 +205,11 @@ mod tests {
let b2 = MemoryBuilder::default().build().unwrap();
assert_ne!(b1.info().name(), b2.info().name())
}
+
+ #[test]
+ fn from_uri_extracts_root() {
+ let uri: Uri = "memory://localhost/path/to/root".parse().unwrap();
+ let cfg = MemoryConfig::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ }
}
diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs
index 88e5f350d..f61167429 100644
--- a/core/src/services/obs/backend.rs
+++ b/core/src/services/obs/backend.rs
@@ -24,6 +24,7 @@ use http::Response;
use http::StatusCode;
use http::Uri;
use log::debug;
+use percent_encoding::percent_decode_str;
use reqsign::HuaweicloudObsConfig;
use reqsign::HuaweicloudObsCredentialLoader;
use reqsign::HuaweicloudObsSigner;
@@ -35,13 +36,69 @@ use super::error::parse_error;
use super::lister::ObsLister;
use super::writer::ObsWriter;
use super::writer::ObsWriters;
-use super::DEFAULT_SCHEME;
+use super::OBS_SCHEME;
use crate::raw::*;
use crate::services::ObsConfig;
use crate::*;
impl Configurator for ObsConfig {
type Builder = ObsBuilder;
+ fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> {
+ let mut map = options.clone();
+
+ let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false);
+
+ let path = percent_decode_str(uri.path()).decode_utf8_lossy();
+
+ if has_bucket {
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ } else if let Some(authority) = uri.authority() {
+ let host = authority.host();
+ if !host.is_empty() {
+ map.insert("bucket".to_string(), host.to_string());
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ }
+ }
+
+ let has_bucket = map.get("bucket").map(|v| !v.is_empty()).unwrap_or(false);
+
+ if !has_bucket {
+ let trimmed = path.trim_matches('/');
+ if trimmed.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "obs uri requires bucket",
+ ));
+ }
+ let mut segments = trimmed.splitn(2, '/');
+ let bucket = segments.next().unwrap();
+ if bucket.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "obs uri requires bucket",
+ ));
+ }
+ map.insert("bucket".to_string(), bucket.to_string());
+ if let Some(rest) = segments.next() {
+ if !rest.is_empty() && !map.contains_key("root") {
+ map.insert("root".to_string(), rest.to_string());
+ }
+ }
+ }
+
+ Self::from_iter(map)
+ }
+
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
ObsBuilder {
@@ -232,7 +289,7 @@ impl Builder for ObsBuilder {
core: Arc::new(ObsCore {
info: {
let am = AccessorInfo::default();
- am.set_scheme(DEFAULT_SCHEME)
+ am.set_scheme(OBS_SCHEME)
.set_root(&root)
.set_name(&bucket)
.set_native_capability(Capability {
@@ -440,3 +497,19 @@ impl Access for ObsBackend {
)))
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
+
+ #[test]
+ fn from_uri_extracts_bucket_and_root() {
+ let uri: Uri = "obs://example-bucket/path/to/root".parse().unwrap();
+ let cfg = ObsConfig::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.bucket.as_deref(), Some("example-bucket"));
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ }
+}
diff --git a/core/src/services/obs/mod.rs b/core/src/services/obs/mod.rs
index ed94697a1..c33719f6a 100644
--- a/core/src/services/obs/mod.rs
+++ b/core/src/services/obs/mod.rs
@@ -17,7 +17,7 @@
/// Default scheme for obs service.
#[cfg(feature = "services-obs")]
-pub(super) const DEFAULT_SCHEME: &str = "obs";
+pub const OBS_SCHEME: &str = "obs";
#[cfg(feature = "services-obs")]
mod core;
#[cfg(feature = "services-obs")]
diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs
index 6cf1a725b..140738275 100644
--- a/core/src/services/oss/backend.rs
+++ b/core/src/services/oss/backend.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
@@ -23,6 +24,7 @@ use http::Response;
use http::StatusCode;
use http::Uri;
use log::debug;
+use percent_encoding::percent_decode_str;
use reqsign::AliyunConfig;
use reqsign::AliyunLoader;
use reqsign::AliyunOssSigner;
@@ -35,7 +37,7 @@ use super::lister::OssListers;
use super::lister::OssObjectVersionsLister;
use super::writer::OssWriter;
use super::writer::OssWriters;
-use super::DEFAULT_SCHEME;
+use super::OSS_SCHEME;
use crate::raw::*;
use crate::services::OssConfig;
use crate::*;
@@ -44,6 +46,62 @@ const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000;
impl Configurator for OssConfig {
type Builder = OssBuilder;
+ fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> {
+ let mut map = options.clone();
+
+ let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true);
+
+ let path = percent_decode_str(uri.path()).decode_utf8_lossy();
+
+ if has_bucket {
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ } else if let Some(authority) = uri.authority() {
+ let host = authority.host();
+ if !host.is_empty() {
+ map.insert("bucket".to_string(), host.to_string());
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ }
+ }
+
+ let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true);
+
+ if !has_bucket {
+ let trimmed = path.trim_matches('/');
+ if trimmed.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "oss uri requires bucket",
+ ));
+ }
+ let mut segments = trimmed.splitn(2, '/');
+ let bucket = segments.next().unwrap();
+ if bucket.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "oss uri requires bucket",
+ ));
+ }
+ map.insert("bucket".to_string(), bucket.to_string());
+ if let Some(rest) = segments.next() {
+ if !rest.is_empty() && !map.contains_key("root") {
+ map.insert("root".to_string(), rest.to_string());
+ }
+ }
+ }
+
+ Self::from_iter(map)
+ }
+
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
OssBuilder {
@@ -511,7 +569,7 @@ impl Builder for OssBuilder {
core: Arc::new(OssCore {
info: {
let am = AccessorInfo::default();
- am.set_scheme(DEFAULT_SCHEME)
+ am.set_scheme(OSS_SCHEME)
.set_root(&root)
.set_name(bucket)
.set_native_capability(Capability {
@@ -732,3 +790,19 @@ impl Access for OssBackend {
)))
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
+
+ #[test]
+ fn from_uri_extracts_bucket_and_root() {
+ let uri: Uri = "oss://example-bucket/path/to/root".parse().unwrap();
+ let cfg = OssConfig::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.bucket, "example-bucket");
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ }
+}
diff --git a/core/src/services/oss/mod.rs b/core/src/services/oss/mod.rs
index e27e6a78b..e3e05c1b6 100644
--- a/core/src/services/oss/mod.rs
+++ b/core/src/services/oss/mod.rs
@@ -17,7 +17,7 @@
/// Default scheme for oss service.
#[cfg(feature = "services-oss")]
-pub(super) const DEFAULT_SCHEME: &str = "oss";
+pub const OSS_SCHEME: &str = "oss";
#[cfg(feature = "services-oss")]
mod core;
#[cfg(feature = "services-oss")]
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index a55c95493..4f3676866 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -1188,6 +1188,9 @@ impl Access for S3Backend {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
#[test]
fn test_is_valid_bucket() {
@@ -1290,4 +1293,12 @@ mod tests {
assert_eq!(region.as_deref(), expected, "{name}");
}
}
+
+ #[test]
+ fn from_uri_extracts_bucket_and_root() {
+ let uri: Uri = "s3://example-bucket/path/to/root".parse().unwrap();
+ let cfg = S3Config::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.bucket, "example-bucket");
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ }
}
diff --git a/core/src/services/upyun/backend.rs b/core/src/services/upyun/backend.rs
index 5154f6f83..c13f421c4 100644
--- a/core/src/services/upyun/backend.rs
+++ b/core/src/services/upyun/backend.rs
@@ -15,13 +15,16 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
use http::Response;
use http::StatusCode;
+use http::Uri;
use log::debug;
+use percent_encoding::percent_decode_str;
use super::core::*;
use super::delete::UpyunDeleter;
@@ -29,13 +32,69 @@ use super::error::parse_error;
use super::lister::UpyunLister;
use super::writer::UpyunWriter;
use super::writer::UpyunWriters;
-use super::DEFAULT_SCHEME;
+use super::UPYUN_SCHEME;
use crate::raw::*;
use crate::services::UpyunConfig;
use crate::*;
impl Configurator for UpyunConfig {
type Builder = UpyunBuilder;
+ fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> {
+ let mut map = options.clone();
+
+ let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true);
+
+ let path = percent_decode_str(uri.path()).decode_utf8_lossy();
+
+ if has_bucket {
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ } else if let Some(authority) = uri.authority() {
+ let host = authority.host();
+ if !host.is_empty() {
+ map.insert("bucket".to_string(), host.to_string());
+ if !map.contains_key("root") {
+ let trimmed = path.trim_matches('/');
+ if !trimmed.is_empty() {
+ map.insert("root".to_string(), trimmed.to_string());
+ }
+ }
+ }
+ }
+
+ let has_bucket = !map.get("bucket").map(|v| v.is_empty()).unwrap_or(true);
+
+ if !has_bucket {
+ let trimmed = path.trim_matches('/');
+ if trimmed.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "upyun uri requires bucket",
+ ));
+ }
+ let mut segments = trimmed.splitn(2, '/');
+ let bucket = segments.next().unwrap();
+ if bucket.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "upyun uri requires bucket",
+ ));
+ }
+ map.insert("bucket".to_string(), bucket.to_string());
+ if let Some(rest) = segments.next() {
+ if !rest.is_empty() && !map.contains_key("root") {
+ map.insert("root".to_string(), rest.to_string());
+ }
+ }
+ }
+
+ Self::from_iter(map)
+ }
+
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
UpyunBuilder {
@@ -169,7 +228,7 @@ impl Builder for UpyunBuilder {
core: Arc::new(UpyunCore {
info: {
let am = AccessorInfo::default();
- am.set_scheme(DEFAULT_SCHEME)
+ am.set_scheme(UPYUN_SCHEME)
.set_root(&root)
.set_native_capability(Capability {
stat: true,
@@ -315,3 +374,19 @@ impl Access for UpyunBackend {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::Configurator;
+ use http::Uri;
+ use std::collections::HashMap;
+
+ #[test]
+ fn from_uri_extracts_bucket_and_root() {
+ let uri: Uri = "upyun://example-bucket/path/to/root".parse().unwrap();
+ let cfg = UpyunConfig::from_uri(&uri, &HashMap::new()).unwrap();
+ assert_eq!(cfg.bucket, "example-bucket");
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ }
+}
diff --git a/core/src/services/upyun/mod.rs b/core/src/services/upyun/mod.rs
index eee374681..50db7493e 100644
--- a/core/src/services/upyun/mod.rs
+++ b/core/src/services/upyun/mod.rs
@@ -17,7 +17,7 @@
/// Default scheme for upyun service.
#[cfg(feature = "services-upyun")]
-pub(super) const DEFAULT_SCHEME: &str = "upyun";
+pub const UPYUN_SCHEME: &str = "upyun";
#[cfg(feature = "services-upyun")]
mod core;
#[cfg(feature = "services-upyun")]
diff --git a/core/src/types/operator/registry.rs b/core/src/types/operator/registry.rs
index 7f11c003f..22d05c408 100644
--- a/core/src/types/operator/registry.rs
+++ b/core/src/types/operator/registry.rs
@@ -97,6 +97,20 @@ fn register_builtin_services(registry: &OperatorRegistry) {
registry.register::<services::Fs>(services::FS_SCHEME);
#[cfg(feature = "services-s3")]
registry.register::<services::S3>(services::S3_SCHEME);
+ #[cfg(feature = "services-azblob")]
+ registry.register::<services::Azblob>(services::AZBLOB_SCHEME);
+ #[cfg(feature = "services-b2")]
+ registry.register::<services::B2>(services::B2_SCHEME);
+ #[cfg(feature = "services-cos")]
+ registry.register::<services::Cos>(services::COS_SCHEME);
+ #[cfg(feature = "services-gcs")]
+ registry.register::<services::Gcs>(services::GCS_SCHEME);
+ #[cfg(feature = "services-obs")]
+ registry.register::<services::Obs>(services::OBS_SCHEME);
+ #[cfg(feature = "services-oss")]
+ registry.register::<services::Oss>(services::OSS_SCHEME);
+ #[cfg(feature = "services-upyun")]
+ registry.register::<services::Upyun>(services::UPYUN_SCHEME);
}
/// Factory adapter that builds an operator from a configurator type.