You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/10/14 09:43:17 UTC
(opendal) branch main updated: feat: Add from_uri support for http/webdav/ftp/sftp (#6666)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new b2cd3ff64 feat: Add from_uri support for http/webdav/ftp/sftp (#6666)
b2cd3ff64 is described below
commit b2cd3ff645a27b58b81d4d49a884fc26890b2aed
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Oct 14 18:43:12 2025 +0900
feat: Add from_uri support for http/webdav/ftp/sftp (#6666)
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/ftp/backend.rs | 53 +++++++++++++++++++++++++++++
core/src/services/http/backend.rs | 66 +++++++++++++++++++++++++++++++++++++
core/src/services/sftp/backend.rs | 55 +++++++++++++++++++++++++++++++
core/src/services/webdav/backend.rs | 64 +++++++++++++++++++++++++++++++++++
core/src/types/operator/uri.rs | 41 +++++++++++++++++++----
5 files changed, 272 insertions(+), 7 deletions(-)
diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs
index ff7bf662a..4b5362a95 100644
--- a/core/src/services/ftp/backend.rs
+++ b/core/src/services/ftp/backend.rs
@@ -39,9 +39,30 @@ use super::reader::FtpReader;
use super::writer::FtpWriter;
use crate::raw::*;
use crate::services::FtpConfig;
+use crate::types::OperatorUri;
use crate::*;
impl Configurator for FtpConfig {
type Builder = FtpBuilder;
+
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
+ let authority = uri.authority().ok_or_else(|| {
+ Error::new(ErrorKind::ConfigInvalid, "uri authority is required")
+ .with_context("service", Scheme::Ftp)
+ })?;
+
+ let mut map = uri.options().clone();
+ map.insert(
+ "endpoint".to_string(),
+ format!("{DEFAULT_SCHEME}://{authority}"),
+ );
+
+ if let Some(root) = uri.root() {
+ map.insert("root".to_string(), root.to_string());
+ }
+
+ Self::from_iter(map)
+ }
+
fn into_builder(self) -> Self::Builder {
FtpBuilder { config: self }
}
@@ -355,6 +376,8 @@ impl FtpBackend {
#[cfg(test)]
mod build_test {
use super::FtpBuilder;
+ use crate::services::FtpConfig;
+ use crate::types::OperatorUri;
use crate::*;
#[test]
@@ -385,4 +408,34 @@ mod build_test {
let e = b.unwrap_err();
assert_eq!(e.kind(), ErrorKind::ConfigInvalid);
}
+
+ #[test]
+ fn from_uri_sets_endpoint_and_root() {
+ let uri = OperatorUri::new(
+ "ftp://example.com/public/data".parse().unwrap(),
+ Vec::<(String, String)>::new(),
+ )
+ .unwrap();
+
+ let cfg = FtpConfig::from_uri(&uri).unwrap();
+ assert_eq!(cfg.endpoint.as_deref(), Some("ftp://example.com"));
+ assert_eq!(cfg.root.as_deref(), Some("public/data"));
+ }
+
+ #[test]
+ fn from_uri_applies_credentials_from_query() {
+ let uri = OperatorUri::new(
+ "ftp://example.com/data".parse().unwrap(),
+ vec![
+ ("user".to_string(), "alice".to_string()),
+ ("password".to_string(), "secret".to_string()),
+ ],
+ )
+ .unwrap();
+
+ let cfg = FtpConfig::from_uri(&uri).unwrap();
+ assert_eq!(cfg.endpoint.as_deref(), Some("ftp://example.com"));
+ assert_eq!(cfg.user.as_deref(), Some("alice"));
+ assert_eq!(cfg.password.as_deref(), Some("secret"));
+ }
}
diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs
index 9aa0a32f1..99675cbf5 100644
--- a/core/src/services/http/backend.rs
+++ b/core/src/services/http/backend.rs
@@ -28,10 +28,30 @@ use super::core::HttpCore;
use super::error::parse_error;
use crate::raw::*;
use crate::services::HttpConfig;
+use crate::types::OperatorUri;
use crate::*;
impl Configurator for HttpConfig {
type Builder = HttpBuilder;
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
+ let authority = uri.authority().ok_or_else(|| {
+ Error::new(ErrorKind::ConfigInvalid, "uri authority is required")
+ .with_context("service", Scheme::Http)
+ })?;
+
+ let mut map = uri.options().clone();
+ map.insert(
+ "endpoint".to_string(),
+ format!("{}://{}", uri.scheme(), authority),
+ );
+
+ if let Some(root) = uri.root() {
+ map.insert("root".to_string(), root.to_string());
+ }
+
+ Self::from_iter(map)
+ }
+
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
HttpBuilder {
@@ -289,3 +309,49 @@ impl Access for HttpBackend {
)))
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn from_uri_sets_endpoint_and_root() {
+ let uri = OperatorUri::new(
+ "http://example.com/static/assets".parse().unwrap(),
+ Vec::<(String, String)>::new(),
+ )
+ .unwrap();
+
+ let cfg = HttpConfig::from_uri(&uri).unwrap();
+ assert_eq!(cfg.endpoint.as_deref(), Some("http://example.com"));
+ assert_eq!(cfg.root.as_deref(), Some("static/assets"));
+ }
+
+ #[test]
+ fn from_uri_preserves_query_options() {
+ let uri = OperatorUri::new(
+ "http://cdn.example.com/data?token=abc123".parse().unwrap(),
+ Vec::<(String, String)>::new(),
+ )
+ .unwrap();
+ let cfg = HttpConfig::from_uri(&uri).unwrap();
+
+ assert_eq!(cfg.endpoint.as_deref(), Some("http://cdn.example.com"));
+ assert_eq!(cfg.token.as_deref(), Some("abc123"));
+ }
+
+ #[test]
+ fn from_uri_ignores_endpoint_override() {
+ let uri = OperatorUri::new(
+ "http://example.com/data".parse().unwrap(),
+ vec![(
+ "endpoint".to_string(),
+ "https://cdn.example.com".to_string(),
+ )],
+ )
+ .unwrap();
+ let cfg = HttpConfig::from_uri(&uri).unwrap();
+
+ assert_eq!(cfg.endpoint.as_deref(), Some("http://example.com"));
+ }
+}
diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs
index 15dad4262..c3f592c9a 100644
--- a/core/src/services/sftp/backend.rs
+++ b/core/src/services/sftp/backend.rs
@@ -38,9 +38,27 @@ use super::reader::SftpReader;
use super::writer::SftpWriter;
use crate::raw::*;
use crate::services::SftpConfig;
+use crate::types::OperatorUri;
use crate::*;
impl Configurator for SftpConfig {
type Builder = SftpBuilder;
+
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
+ let authority = uri.authority().ok_or_else(|| {
+ Error::new(ErrorKind::ConfigInvalid, "uri authority is required")
+ .with_context("service", Scheme::Sftp)
+ })?;
+
+ let mut map = uri.options().clone();
+ map.insert("endpoint".to_string(), authority.to_string());
+
+ if let Some(root) = uri.root() {
+ map.insert("root".to_string(), root.to_string());
+ }
+
+ Self::from_iter(map)
+ }
+
fn into_builder(self) -> Self::Builder {
SftpBuilder { config: self }
}
@@ -395,3 +413,40 @@ impl Access for SftpBackend {
Ok(RpRename::default())
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn from_uri_sets_endpoint_and_root() {
+ let uri = OperatorUri::new(
+ "sftp://sftp.example.com/home/alice".parse().unwrap(),
+ Vec::<(String, String)>::new(),
+ )
+ .unwrap();
+
+ let cfg = SftpConfig::from_uri(&uri).unwrap();
+ assert_eq!(cfg.endpoint.as_deref(), Some("sftp.example.com"));
+ assert_eq!(cfg.root.as_deref(), Some("home/alice"));
+ }
+
+ #[test]
+ fn from_uri_applies_connection_overrides() {
+ let uri = OperatorUri::new(
+ "sftp://host".parse().unwrap(),
+ vec![
+ ("user".to_string(), "alice".to_string()),
+ ("key".to_string(), "/home/alice/.ssh/id_rsa".to_string()),
+ ("known_hosts_strategy".to_string(), "accept".to_string()),
+ ],
+ )
+ .unwrap();
+
+ let cfg = SftpConfig::from_uri(&uri).unwrap();
+ assert_eq!(cfg.endpoint.as_deref(), Some("host"));
+ assert_eq!(cfg.user.as_deref(), Some("alice"));
+ assert_eq!(cfg.key.as_deref(), Some("/home/alice/.ssh/id_rsa"));
+ assert_eq!(cfg.known_hosts_strategy.as_deref(), Some("accept"));
+ }
+}
diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs
index ef3d76e4f..c3dc48b64 100644
--- a/core/src/services/webdav/backend.rs
+++ b/core/src/services/webdav/backend.rs
@@ -32,10 +32,27 @@ use super::lister::WebdavLister;
use super::writer::WebdavWriter;
use crate::raw::*;
use crate::services::WebdavConfig;
+use crate::types::OperatorUri;
use crate::*;
impl Configurator for WebdavConfig {
type Builder = WebdavBuilder;
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
+ let authority = uri.authority().ok_or_else(|| {
+ Error::new(ErrorKind::ConfigInvalid, "uri authority is required")
+ .with_context("service", Scheme::Webdav)
+ })?;
+
+ let mut map = uri.options().clone();
+ map.insert("endpoint".to_string(), format!("https://{authority}"));
+
+ if let Some(root) = uri.root() {
+ map.insert("root".to_string(), root.to_string());
+ }
+
+ Self::from_iter(map)
+ }
+
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
WebdavBuilder {
@@ -316,3 +333,50 @@ impl Access for WebdavBackend {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn from_uri_sets_endpoint_and_root() {
+ let uri = OperatorUri::new(
+ "webdav://webdav.example.com/remote.php/webdav"
+ .parse()
+ .unwrap(),
+ Vec::<(String, String)>::new(),
+ )
+ .unwrap();
+
+ let cfg = WebdavConfig::from_uri(&uri).unwrap();
+ assert_eq!(cfg.endpoint.as_deref(), Some("https://webdav.example.com"));
+ assert_eq!(cfg.root.as_deref(), Some("remote.php/webdav"));
+ }
+
+ #[test]
+ fn from_uri_ignores_endpoint_override() {
+ let uri = OperatorUri::new(
+ "webdav://dav.internal/data".parse().unwrap(),
+ vec![(
+ "endpoint".to_string(),
+ "http://dav.internal:8080".to_string(),
+ )],
+ )
+ .unwrap();
+
+ let cfg = WebdavConfig::from_uri(&uri).unwrap();
+ assert_eq!(cfg.endpoint.as_deref(), Some("https://dav.internal"));
+ }
+
+ #[test]
+ fn from_uri_propagates_disable_copy() {
+ let uri = OperatorUri::new(
+ "webdav://dav.example.com".parse().unwrap(),
+ vec![("disable_copy".to_string(), "true".to_string())],
+ )
+ .unwrap();
+
+ let cfg = WebdavConfig::from_uri(&uri).unwrap();
+ assert!(cfg.disable_copy);
+ }
+}
diff --git a/core/src/types/operator/uri.rs b/core/src/types/operator/uri.rs
index 7cfc7900c..3d00d7b5d 100644
--- a/core/src/types/operator/uri.rs
+++ b/core/src/types/operator/uri.rs
@@ -26,6 +26,7 @@ use crate::{Error, ErrorKind, Result};
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct OperatorUri {
scheme: String,
+ authority: Option<String>,
name: Option<String>,
root: Option<String>,
options: HashMap<String, String>,
@@ -64,14 +65,19 @@ impl OperatorUri {
options.insert(key.to_ascii_lowercase(), value);
}
- let name = uri.authority().and_then(|authority| {
- let host = authority.host();
- if host.is_empty() {
- None
- } else {
- Some(host.to_string())
+ let (authority, name) = match uri.authority() {
+ Some(authority) => {
+ let authority_str = authority.as_str().to_string();
+ let host = authority.host();
+ let name = if host.is_empty() {
+ None
+ } else {
+ Some(host.to_string())
+ };
+ (Some(authority_str), name)
}
- });
+ None => (None, None),
+ };
let decoded_path = percent_decode_str(uri.path()).decode_utf8_lossy();
let trimmed = decoded_path.trim_matches('/');
@@ -83,6 +89,7 @@ impl OperatorUri {
Ok(Self {
scheme,
+ authority,
name,
root,
options,
@@ -99,6 +106,11 @@ impl OperatorUri {
self.name.as_deref()
}
+ /// Authority extracted from the URI, if present (host with optional port).
+ pub fn authority(&self) -> Option<&str> {
+ self.authority.as_deref()
+ }
+
/// Root path (without leading slash) extracted from the URI path, if present.
pub fn root(&self) -> Option<&str> {
self.root.as_deref()
@@ -235,6 +247,7 @@ mod tests {
.unwrap();
assert_eq!(uri.scheme(), "s3");
+ assert_eq!(uri.authority(), Some("example-bucket"));
assert_eq!(uri.name(), Some("example-bucket"));
assert_eq!(uri.root(), Some("photos/2024"));
assert!(uri.options().is_empty());
@@ -261,4 +274,18 @@ mod tests {
Some("https://custom")
);
}
+
+ #[test]
+ fn parse_uri_with_port_preserves_authority() {
+ let uri = OperatorUri::new(
+ "http://example.com:8080/root".parse().unwrap(),
+ Vec::<(String, String)>::new(),
+ )
+ .unwrap();
+
+ assert_eq!(uri.scheme(), "http");
+ assert_eq!(uri.authority(), Some("example.com:8080"));
+ assert_eq!(uri.name(), Some("example.com"));
+ assert_eq!(uri.root(), Some("root"));
+ }
}