You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ti...@apache.org on 2024/07/20 00:37:15 UTC

(opendal) 02/10: feat: add config for oss backend

This is an automated email from the ASF dual-hosted git repository.

tison pushed a commit to branch make-config-data-object
in repository https://gitbox.apache.org/repos/asf/opendal.git

commit f20358ae6aa77c4fadec7bbc5f5c7b86c7a77453
Author: tison <wa...@gmail.com>
AuthorDate: Fri Jul 19 16:09:38 2024 -0700

    feat: add config for oss backend
    
    Signed-off-by: tison <wa...@gmail.com>
---
 core/src/raw/serde_util.rs       |   2 +-
 core/src/services/oss/backend.rs | 140 ++++++++++++++++++++-------------------
 core/src/services/oss/mod.rs     |   1 +
 3 files changed, 75 insertions(+), 68 deletions(-)

diff --git a/core/src/raw/serde_util.rs b/core/src/raw/serde_util.rs
index 6507d3612a..f34833a47d 100644
--- a/core/src/raw/serde_util.rs
+++ b/core/src/raw/serde_util.rs
@@ -45,7 +45,7 @@ pub fn new_json_deserialize_error(e: serde_json::Error) -> Error {
 
 /// ConfigDeserializer is used to deserialize given configs from `HashMap<String, String>`.
 ///
-/// This is only used by our services config.
+/// This is only used by our services' config.
 pub struct ConfigDeserializer(MapDeserializer<'static, Pairs, de::value::Error>);
 
 impl ConfigDeserializer {
diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs
index b49d698965..789722ad88 100644
--- a/core/src/services/oss/backend.rs
+++ b/core/src/services/oss/backend.rs
@@ -21,6 +21,13 @@ use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::sync::Arc;
 
+use super::core::*;
+use super::error::parse_error;
+use super::lister::OssLister;
+use super::writer::OssWriter;
+use crate::raw::*;
+use crate::services::oss::writer::OssWriters;
+use crate::*;
 use bytes::Buf;
 use http::Response;
 use http::StatusCode;
@@ -29,42 +36,34 @@ use log::debug;
 use reqsign::AliyunConfig;
 use reqsign::AliyunLoader;
 use reqsign::AliyunOssSigner;
-
-use super::core::*;
-use super::error::parse_error;
-use super::lister::OssLister;
-use super::writer::OssWriter;
-use crate::raw::*;
-use crate::services::oss::writer::OssWriters;
-use crate::*;
+use serde::{Deserialize, Serialize};
 
 const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000;
 
-/// Aliyun Object Storage Service (OSS) support
-#[doc = include_str!("docs.md")]
-#[derive(Default)]
-pub struct OssBuilder {
-    root: Option<String>,
+/// Config for Aliyun Object Storage Service (OSS) support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct OssConfig {
+    pub root: Option<String>,
 
-    endpoint: Option<String>,
-    presign_endpoint: Option<String>,
-    bucket: String,
+    pub endpoint: Option<String>,
+    pub presign_endpoint: Option<String>,
+    pub bucket: String,
 
     // OSS features
-    server_side_encryption: Option<String>,
-    server_side_encryption_key_id: Option<String>,
-    allow_anonymous: bool,
+    pub server_side_encryption: Option<String>,
+    pub server_side_encryption_key_id: Option<String>,
+    pub allow_anonymous: bool,
 
     // authenticate options
-    access_key_id: Option<String>,
-    access_key_secret: Option<String>,
-
-    http_client: Option<HttpClient>,
+    pub access_key_id: Option<String>,
+    pub access_key_secret: Option<String>,
     /// batch_max_operations
-    batch_max_operations: Option<usize>,
+    pub batch_max_operations: Option<usize>,
 }
 
-impl Debug for OssBuilder {
+impl Debug for OssConfig {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         let mut d = f.debug_struct("Builder");
         d.field("root", &self.root)
@@ -76,12 +75,29 @@ impl Debug for OssBuilder {
     }
 }
 
+/// Aliyun Object Storage Service (OSS) support
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct OssBuilder {
+    config: OssConfig,
+    http_client: Option<HttpClient>,
+}
+
+impl Debug for OssBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("OssBuilder");
+
+        d.field("config", &self.config);
+        d.finish_non_exhaustive()
+    }
+}
+
 impl OssBuilder {
     /// Set root of this backend.
     ///
     /// All operations will happen under this root.
     pub fn root(&mut self, root: &str) -> &mut Self {
-        self.root = if root.is_empty() {
+        self.config.root = if root.is_empty() {
             None
         } else {
             Some(root.to_string())
@@ -92,7 +108,7 @@ impl OssBuilder {
 
     /// Set bucket name of this backend.
     pub fn bucket(&mut self, bucket: &str) -> &mut Self {
-        self.bucket = bucket.to_string();
+        self.config.bucket = bucket.to_string();
 
         self
     }
@@ -101,7 +117,7 @@ impl OssBuilder {
     pub fn endpoint(&mut self, endpoint: &str) -> &mut Self {
         if !endpoint.is_empty() {
             // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
-            self.endpoint = Some(endpoint.trim_end_matches('/').to_string())
+            self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string())
         }
 
         self
@@ -118,7 +134,7 @@ impl OssBuilder {
     pub fn presign_endpoint(&mut self, endpoint: &str) -> &mut Self {
         if !endpoint.is_empty() {
             // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
-            self.presign_endpoint = Some(endpoint.trim_end_matches('/').to_string())
+            self.config.presign_endpoint = Some(endpoint.trim_end_matches('/').to_string())
         }
 
         self
@@ -130,7 +146,7 @@ impl OssBuilder {
     /// - If not, we will try to load it from environment.
     pub fn access_key_id(&mut self, v: &str) -> &mut Self {
         if !v.is_empty() {
-            self.access_key_id = Some(v.to_string())
+            self.config.access_key_id = Some(v.to_string())
         }
 
         self
@@ -142,7 +158,7 @@ impl OssBuilder {
     /// - If not, we will try to load it from environment.
     pub fn access_key_secret(&mut self, v: &str) -> &mut Self {
         if !v.is_empty() {
-            self.access_key_secret = Some(v.to_string())
+            self.config.access_key_secret = Some(v.to_string())
         }
 
         self
@@ -220,7 +236,7 @@ impl OssBuilder {
     ///     4. If a specific CMK ID is specified, include the `x-oss-server-side-encryption-key-id` parameter in the request, and set its value to the specified CMK ID.
     pub fn server_side_encryption(&mut self, v: &str) -> &mut Self {
         if !v.is_empty() {
-            self.server_side_encryption = Some(v.to_string())
+            self.config.server_side_encryption = Some(v.to_string())
         }
         self
     }
@@ -232,14 +248,14 @@ impl OssBuilder {
     /// This option only takes effect when server_side_encryption equals to KMS.
     pub fn server_side_encryption_key_id(&mut self, v: &str) -> &mut Self {
         if !v.is_empty() {
-            self.server_side_encryption_key_id = Some(v.to_string())
+            self.config.server_side_encryption_key_id = Some(v.to_string())
         }
         self
     }
 
     /// Set maximum batch operations of this backend.
     pub fn batch_max_operations(&mut self, batch_max_operations: usize) -> &mut Self {
-        self.batch_max_operations = Some(batch_max_operations);
+        self.config.batch_max_operations = Some(batch_max_operations);
 
         self
     }
@@ -247,7 +263,7 @@ impl OssBuilder {
     /// Allow anonymous will allow opendal to send request without signing
     /// when credential is not loaded.
     pub fn allow_anonymous(&mut self) -> &mut Self {
-        self.allow_anonymous = true;
+        self.config.allow_anonymous = true;
         self
     }
 }
@@ -257,38 +273,26 @@ impl Builder for OssBuilder {
     type Accessor = OssBackend;
 
     fn from_map(map: HashMap<String, String>) -> Self {
-        let mut builder = OssBuilder::default();
-
-        map.get("root").map(|v| builder.root(v));
-        map.get("bucket").map(|v| builder.bucket(v));
-        map.get("endpoint").map(|v| builder.endpoint(v));
-        map.get("presign_endpoint")
-            .map(|v| builder.presign_endpoint(v));
-        map.get("access_key_id").map(|v| builder.access_key_id(v));
-        map.get("access_key_secret")
-            .map(|v| builder.access_key_secret(v));
-        map.get("server_side_encryption")
-            .map(|v| builder.server_side_encryption(v));
-        map.get("server_side_encryption_key_id")
-            .map(|v| builder.server_side_encryption_key_id(v));
-        map.get("batch_max_operations")
-            .map(|v| builder.batch_max_operations(v.parse::<usize>().unwrap()));
-        map.get("allow_anonymous")
-            .filter(|v| *v == "on" || *v == "true")
-            .map(|_| builder.allow_anonymous());
-
-        builder
+        // Deserialize the configuration from the HashMap.
+        let config = OssConfig::deserialize(ConfigDeserializer::new(map))
+            .expect("config deserialize must succeed");
+
+        // Create an OssBuilder instance with the deserialized config.
+        OssBuilder {
+            config,
+            http_client: None,
+        }
     }
 
     fn build(&mut self) -> Result<Self::Accessor> {
         debug!("backend build started: {:?}", &self);
 
-        let root = normalize_root(&self.root.clone().unwrap_or_default());
+        let root = normalize_root(&self.config.root.clone().unwrap_or_default());
         debug!("backend use root {}", &root);
 
         // Handle endpoint, region and bucket name.
-        let bucket = match self.bucket.is_empty() {
-            false => Ok(&self.bucket),
+        let bucket = match self.config.bucket.is_empty() {
+            false => Ok(&self.config.bucket),
             true => Err(
                 Error::new(ErrorKind::ConfigInvalid, "The bucket is misconfigured")
                     .with_context("service", Scheme::Oss),
@@ -306,17 +310,18 @@ impl Builder for OssBuilder {
 
         // Retrieve endpoint and host by parsing the endpoint option and bucket. If presign_endpoint is not
         // set, take endpoint as default presign_endpoint.
-        let (endpoint, host) = self.parse_endpoint(&self.endpoint, bucket)?;
+        let (endpoint, host) = self.parse_endpoint(&self.config.endpoint, bucket)?;
         debug!("backend use bucket {}, endpoint: {}", &bucket, &endpoint);
 
-        let presign_endpoint = if self.presign_endpoint.is_some() {
-            self.parse_endpoint(&self.presign_endpoint, bucket)?.0
+        let presign_endpoint = if self.config.presign_endpoint.is_some() {
+            self.parse_endpoint(&self.config.presign_endpoint, bucket)?
+                .0
         } else {
             endpoint.clone()
         };
         debug!("backend use presign_endpoint: {}", &presign_endpoint);
 
-        let server_side_encryption = match &self.server_side_encryption {
+        let server_side_encryption = match &self.config.server_side_encryption {
             None => None,
             Some(v) => Some(
                 build_header_value(v)
@@ -324,7 +329,7 @@ impl Builder for OssBuilder {
             ),
         };
 
-        let server_side_encryption_key_id = match &self.server_side_encryption_key_id {
+        let server_side_encryption_key_id = match &self.config.server_side_encryption_key_id {
             None => None,
             Some(v) => Some(
                 build_header_value(v)
@@ -336,11 +341,11 @@ impl Builder for OssBuilder {
         // Load cfg from env first.
         cfg = cfg.from_env();
 
-        if let Some(v) = self.access_key_id.take() {
+        if let Some(v) = self.config.access_key_id.take() {
             cfg.access_key_id = Some(v);
         }
 
-        if let Some(v) = self.access_key_secret.take() {
+        if let Some(v) = self.config.access_key_secret.take() {
             cfg.access_key_secret = Some(v);
         }
 
@@ -349,6 +354,7 @@ impl Builder for OssBuilder {
         let signer = AliyunOssSigner::new(bucket);
 
         let batch_max_operations = self
+            .config
             .batch_max_operations
             .unwrap_or(DEFAULT_BATCH_MAX_OPERATIONS);
         debug!("Backend build finished");
@@ -360,7 +366,7 @@ impl Builder for OssBuilder {
                 endpoint,
                 host,
                 presign_endpoint,
-                allow_anonymous: self.allow_anonymous,
+                allow_anonymous: self.config.allow_anonymous,
                 signer,
                 loader,
                 client,
diff --git a/core/src/services/oss/mod.rs b/core/src/services/oss/mod.rs
index 3954a9885c..4839cff2d3 100644
--- a/core/src/services/oss/mod.rs
+++ b/core/src/services/oss/mod.rs
@@ -17,6 +17,7 @@
 
 mod backend;
 pub use backend::OssBuilder as Oss;
+pub use backend::OssConfig;
 
 mod core;
 mod error;