You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/07/14 13:19:59 UTC
[incubator-opendal] 01/01: refactor(services/s3): Add more detect logic for detect_region
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch polish-details
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 445a54bd4605232101a4990631c82ae6fdff2634
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Jul 14 21:19:30 2023 +0800
refactor(services/s3): Add more detect logic for detect_region
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/s3/backend.rs | 152 ++++++++++++++++++++++++++++++++--------
1 file changed, 122 insertions(+), 30 deletions(-)
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 236781794..af13e752d 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -27,6 +27,7 @@ use base64::Engine;
use bytes::Buf;
use http::StatusCode;
use log::debug;
+use log::warn;
use md5::Digest;
use md5::Md5;
use once_cell::sync::Lazy;
@@ -56,6 +57,7 @@ static ENDPOINT_TEMPLATES: Lazy<HashMap<&'static str, &'static str>> = Lazy::new
const DEFAULT_WRITE_MIN_SIZE: usize = 8 * 1024 * 1024;
const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000;
+
/// Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support.
/// For more information about s3-compatible services, refer to [Compatible Services](#compatible-services).
#[doc = include_str!("docs.md")]
@@ -518,20 +520,42 @@ impl S3Builder {
self
}
- /// a helper function to make it easier to find region
- /// Reference: [Amazon S3 HeadBucket API](https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_HeadBucket.html)
+ /// Detect region of S3 bucket.
+ ///
/// # Args
///
- /// endpoint: the endpoint of S3 service
+ /// - endpoint: the endpoint of S3 service
+ /// - bucket: the bucket of S3 service
///
- /// bucket: the bucket of S3 service
/// # Return
- /// if get the region of given inputs, return Some(region)
- /// else return None
///
- /// # Usage
- /// let b = S3Builder::default();
- /// let region = b.detect_region("https://s3.amazonaws.com", "buckets").await;
+ /// - `Some(region)` means we detect the region successfully
+ /// - `None` means we can't detect the region or meeting errors.
+ ///
+ /// # Notes
+ ///
+ /// We will try to detect region by the following methods.
+ ///
+ /// - Match endpoint with given rules to get region
+ /// - Cloudflare R2
+ /// - AWS S3
+ /// - Aliyun OSS
+ /// - Send a `HEAD` request to endpoint with bucket name to get `x-amz-bucket-region`.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use opendal::services::S3;
+ ///
+ /// # async fn example() {
+ /// let builder = S3::default();
+ /// let region: Option<String> = builder.detect_region("https://s3.amazonaws.com", "example").await;
+ /// # }
+ /// ```
+ ///
+ /// # Reference
+ ///
+ /// - [Amazon S3 HeadBucket API](https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_HeadBucket.html)
pub async fn detect_region(&self, endpoint: &str, bucket: &str) -> Option<String> {
let mut endpoint = if endpoint.starts_with("http") {
endpoint.to_string()
@@ -540,24 +564,51 @@ impl S3Builder {
format!("https://{}", endpoint)
};
- endpoint = endpoint.replace(&format!("//{0}.", bucket), "//");
- let url = format!("{0}/{1}", endpoint, bucket);
+ // Remove bucket name from endpoint.
+ endpoint = endpoint.replace(&format!("//{bucket}."), "//");
+ let url = format!("{endpoint}/{bucket}");
- debug!("backend detect region with url: {url}");
+ debug!("detect region with url: {url}");
- let req = match http::Request::head(&url).body(AsyncBody::Empty) {
- Ok(reg) => reg,
- Err(_) => return None,
- };
+ // Try to detect region by endpoint.
- let client = match HttpClient::new() {
- Ok(client) => client,
- Err(_) => return None,
- };
- let res = match client.send(req).await {
- Ok(res) => res,
- Err(_) => return None,
- };
+ // If this bucket is R2, we can return auto directly.
+ //
+ // Reference: <https://developers.cloudflare.com/r2/api/s3/api/>
+ if endpoint.ends_with("r2.cloudflarestorage.com") {
+ return Some("auto".to_string());
+ }
+
+ // If this bucket is AWS, we can try to match the endpoint.
+ if let Some(v) = endpoint.strip_prefix("https://s3.") {
+ if let Some(region) = v.strip_suffix(".amazonaws.com") {
+ return Some(region.to_string());
+ }
+ }
+
+ // If this bucket is OSS, we can try to match the endpoint.
+ //
+ // - `oss-ap-southeast-1.aliyuncs.com` => `oss-ap-southeast-1`
+ // - `oss-cn-hangzhou-internal.aliyuncs.com` => `oss-cn-hangzhou`
+ if let Some(v) = endpoint.strip_prefix("https://") {
+ if let Some(region) = v.strip_suffix(".aliyuncs.com") {
+ return Some(region.to_string());
+ }
+
+ if let Some(region) = v.strip_suffix("-internal.aliyuncs.com") {
+ return Some(region.to_string());
+ }
+ }
+
+ // Try to detect region by HeadBucket.
+ let req = http::Request::head(&url).body(AsyncBody::Empty).ok()?;
+
+ let client = HttpClient::new().ok()?;
+ let res = client
+ .send(req)
+ .await
+ .map_err(|err| warn!("detect region failed for: {err:?}"))
+ .ok()?;
debug!(
"auto detect region got response: status {:?}, header: {:?}",
@@ -566,11 +617,9 @@ impl S3Builder {
);
match res.status() {
- // The endpoint works, return with not changed endpoint and
- // default region.
StatusCode::OK | StatusCode::FORBIDDEN | StatusCode::MOVED_PERMANENTLY => {
- let region = res.headers().get("x-amz-bucket-region").unwrap().to_str();
- if let Ok(regin) = region {
+ let region = res.headers().get("x-amz-bucket-region")?;
+ if let Ok(regin) = region.to_str() {
Some(regin.to_string())
} else {
None
@@ -1112,10 +1161,53 @@ mod tests {
assert_eq!(endpoint, "https://test.s3.us-east-2.amazonaws.com");
}
}
+
#[tokio::test]
async fn test_detect_region() {
+ let cases = vec![
+ (
+ "aws s3 without region in endpoint",
+ "https://s3.amazonaws.com",
+ "example",
+ Some("us-east-1"),
+ ),
+ (
+ "aws s3 with region in endpoint",
+ "https://s3.us-east-1.amazonaws.com",
+ "example",
+ Some("us-east-1"),
+ ),
+ (
+ "oss with public endpoint",
+ "https://oss-ap-southeast-1.aliyuncs.com",
+ "example",
+ Some("oss-ap-southeast-1"),
+ ),
+ (
+ "oss with internal endpoint",
+ "https://oss-cn-hangzhou-internal.aliyuncs.com",
+ "example",
+ Some("oss-cn-hangzhou-internal"),
+ ),
+ (
+ "r2",
+ "https://abc.xxxxx.r2.cloudflarestorage.com",
+ "example",
+ Some("auto"),
+ ),
+ (
+ "invalid service",
+ "https://opendal.apache.org",
+ "example",
+ None,
+ ),
+ ];
+
let b = S3Builder::default();
- let region = b.detect_region("https://s3.amazonaws.com", "buckets").await;
- assert_eq!(region, Some("us-east-1".to_string()))
+
+ for (name, endpoint, bucket, expected) in cases {
+ let region = b.detect_region(endpoint, bucket).await;
+ assert_eq!(region.as_deref(), expected, "{}", name);
+ }
}
}