You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ti...@apache.org on 2024/07/20 00:37:15 UTC
(opendal) 02/10: feat: add config for oss backend
This is an automated email from the ASF dual-hosted git repository.
tison pushed a commit to branch make-config-data-object
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit f20358ae6aa77c4fadec7bbc5f5c7b86c7a77453
Author: tison <wa...@gmail.com>
AuthorDate: Fri Jul 19 16:09:38 2024 -0700
feat: add config for oss backend
Signed-off-by: tison <wa...@gmail.com>
---
core/src/raw/serde_util.rs | 2 +-
core/src/services/oss/backend.rs | 140 ++++++++++++++++++++-------------------
core/src/services/oss/mod.rs | 1 +
3 files changed, 75 insertions(+), 68 deletions(-)
diff --git a/core/src/raw/serde_util.rs b/core/src/raw/serde_util.rs
index 6507d3612a..f34833a47d 100644
--- a/core/src/raw/serde_util.rs
+++ b/core/src/raw/serde_util.rs
@@ -45,7 +45,7 @@ pub fn new_json_deserialize_error(e: serde_json::Error) -> Error {
/// ConfigDeserializer is used to deserialize given configs from `HashMap<String, String>`.
///
-/// This is only used by our services config.
+/// This is only used by our services' config.
pub struct ConfigDeserializer(MapDeserializer<'static, Pairs, de::value::Error>);
impl ConfigDeserializer {
diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs
index b49d698965..789722ad88 100644
--- a/core/src/services/oss/backend.rs
+++ b/core/src/services/oss/backend.rs
@@ -21,6 +21,13 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
+use super::core::*;
+use super::error::parse_error;
+use super::lister::OssLister;
+use super::writer::OssWriter;
+use crate::raw::*;
+use crate::services::oss::writer::OssWriters;
+use crate::*;
use bytes::Buf;
use http::Response;
use http::StatusCode;
@@ -29,42 +36,34 @@ use log::debug;
use reqsign::AliyunConfig;
use reqsign::AliyunLoader;
use reqsign::AliyunOssSigner;
-
-use super::core::*;
-use super::error::parse_error;
-use super::lister::OssLister;
-use super::writer::OssWriter;
-use crate::raw::*;
-use crate::services::oss::writer::OssWriters;
-use crate::*;
+use serde::{Deserialize, Serialize};
const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000;
-/// Aliyun Object Storage Service (OSS) support
-#[doc = include_str!("docs.md")]
-#[derive(Default)]
-pub struct OssBuilder {
- root: Option<String>,
+/// Config for Aliyun Object Storage Service (OSS) support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct OssConfig {
+ pub root: Option<String>,
- endpoint: Option<String>,
- presign_endpoint: Option<String>,
- bucket: String,
+ pub endpoint: Option<String>,
+ pub presign_endpoint: Option<String>,
+ pub bucket: String,
// OSS features
- server_side_encryption: Option<String>,
- server_side_encryption_key_id: Option<String>,
- allow_anonymous: bool,
+ pub server_side_encryption: Option<String>,
+ pub server_side_encryption_key_id: Option<String>,
+ pub allow_anonymous: bool,
// authenticate options
- access_key_id: Option<String>,
- access_key_secret: Option<String>,
-
- http_client: Option<HttpClient>,
+ pub access_key_id: Option<String>,
+ pub access_key_secret: Option<String>,
/// batch_max_operations
- batch_max_operations: Option<usize>,
+ pub batch_max_operations: Option<usize>,
}
-impl Debug for OssBuilder {
+impl Debug for OssConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut d = f.debug_struct("Builder");
d.field("root", &self.root)
@@ -76,12 +75,29 @@ impl Debug for OssBuilder {
}
}
+/// Aliyun Object Storage Service (OSS) support
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct OssBuilder {
+ config: OssConfig,
+ http_client: Option<HttpClient>,
+}
+
+impl Debug for OssBuilder {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut d = f.debug_struct("OssBuilder");
+
+ d.field("config", &self.config);
+ d.finish_non_exhaustive()
+ }
+}
+
impl OssBuilder {
/// Set root of this backend.
///
/// All operations will happen under this root.
pub fn root(&mut self, root: &str) -> &mut Self {
- self.root = if root.is_empty() {
+ self.config.root = if root.is_empty() {
None
} else {
Some(root.to_string())
@@ -92,7 +108,7 @@ impl OssBuilder {
/// Set bucket name of this backend.
pub fn bucket(&mut self, bucket: &str) -> &mut Self {
- self.bucket = bucket.to_string();
+ self.config.bucket = bucket.to_string();
self
}
@@ -101,7 +117,7 @@ impl OssBuilder {
pub fn endpoint(&mut self, endpoint: &str) -> &mut Self {
if !endpoint.is_empty() {
// Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
- self.endpoint = Some(endpoint.trim_end_matches('/').to_string())
+ self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string())
}
self
@@ -118,7 +134,7 @@ impl OssBuilder {
pub fn presign_endpoint(&mut self, endpoint: &str) -> &mut Self {
if !endpoint.is_empty() {
// Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
- self.presign_endpoint = Some(endpoint.trim_end_matches('/').to_string())
+ self.config.presign_endpoint = Some(endpoint.trim_end_matches('/').to_string())
}
self
@@ -130,7 +146,7 @@ impl OssBuilder {
/// - If not, we will try to load it from environment.
pub fn access_key_id(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
- self.access_key_id = Some(v.to_string())
+ self.config.access_key_id = Some(v.to_string())
}
self
@@ -142,7 +158,7 @@ impl OssBuilder {
/// - If not, we will try to load it from environment.
pub fn access_key_secret(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
- self.access_key_secret = Some(v.to_string())
+ self.config.access_key_secret = Some(v.to_string())
}
self
@@ -220,7 +236,7 @@ impl OssBuilder {
/// 4. If a specific CMK ID is specified, include the `x-oss-server-side-encryption-key-id` parameter in the request, and set its value to the specified CMK ID.
pub fn server_side_encryption(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
- self.server_side_encryption = Some(v.to_string())
+ self.config.server_side_encryption = Some(v.to_string())
}
self
}
@@ -232,14 +248,14 @@ impl OssBuilder {
/// This option only takes effect when server_side_encryption equals to KMS.
pub fn server_side_encryption_key_id(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
- self.server_side_encryption_key_id = Some(v.to_string())
+ self.config.server_side_encryption_key_id = Some(v.to_string())
}
self
}
/// Set maximum batch operations of this backend.
pub fn batch_max_operations(&mut self, batch_max_operations: usize) -> &mut Self {
- self.batch_max_operations = Some(batch_max_operations);
+ self.config.batch_max_operations = Some(batch_max_operations);
self
}
@@ -247,7 +263,7 @@ impl OssBuilder {
/// Allow anonymous will allow opendal to send request without signing
/// when credential is not loaded.
pub fn allow_anonymous(&mut self) -> &mut Self {
- self.allow_anonymous = true;
+ self.config.allow_anonymous = true;
self
}
}
@@ -257,38 +273,26 @@ impl Builder for OssBuilder {
type Accessor = OssBackend;
fn from_map(map: HashMap<String, String>) -> Self {
- let mut builder = OssBuilder::default();
-
- map.get("root").map(|v| builder.root(v));
- map.get("bucket").map(|v| builder.bucket(v));
- map.get("endpoint").map(|v| builder.endpoint(v));
- map.get("presign_endpoint")
- .map(|v| builder.presign_endpoint(v));
- map.get("access_key_id").map(|v| builder.access_key_id(v));
- map.get("access_key_secret")
- .map(|v| builder.access_key_secret(v));
- map.get("server_side_encryption")
- .map(|v| builder.server_side_encryption(v));
- map.get("server_side_encryption_key_id")
- .map(|v| builder.server_side_encryption_key_id(v));
- map.get("batch_max_operations")
- .map(|v| builder.batch_max_operations(v.parse::<usize>().unwrap()));
- map.get("allow_anonymous")
- .filter(|v| *v == "on" || *v == "true")
- .map(|_| builder.allow_anonymous());
-
- builder
+ // Deserialize the configuration from the HashMap.
+ let config = OssConfig::deserialize(ConfigDeserializer::new(map))
+ .expect("config deserialize must succeed");
+
+ // Create an OssBuilder instance with the deserialized config.
+ OssBuilder {
+ config,
+ http_client: None,
+ }
}
fn build(&mut self) -> Result<Self::Accessor> {
debug!("backend build started: {:?}", &self);
- let root = normalize_root(&self.root.clone().unwrap_or_default());
+ let root = normalize_root(&self.config.root.clone().unwrap_or_default());
debug!("backend use root {}", &root);
// Handle endpoint, region and bucket name.
- let bucket = match self.bucket.is_empty() {
- false => Ok(&self.bucket),
+ let bucket = match self.config.bucket.is_empty() {
+ false => Ok(&self.config.bucket),
true => Err(
Error::new(ErrorKind::ConfigInvalid, "The bucket is misconfigured")
.with_context("service", Scheme::Oss),
@@ -306,17 +310,18 @@ impl Builder for OssBuilder {
// Retrieve endpoint and host by parsing the endpoint option and bucket. If presign_endpoint is not
// set, take endpoint as default presign_endpoint.
- let (endpoint, host) = self.parse_endpoint(&self.endpoint, bucket)?;
+ let (endpoint, host) = self.parse_endpoint(&self.config.endpoint, bucket)?;
debug!("backend use bucket {}, endpoint: {}", &bucket, &endpoint);
- let presign_endpoint = if self.presign_endpoint.is_some() {
- self.parse_endpoint(&self.presign_endpoint, bucket)?.0
+ let presign_endpoint = if self.config.presign_endpoint.is_some() {
+ self.parse_endpoint(&self.config.presign_endpoint, bucket)?
+ .0
} else {
endpoint.clone()
};
debug!("backend use presign_endpoint: {}", &presign_endpoint);
- let server_side_encryption = match &self.server_side_encryption {
+ let server_side_encryption = match &self.config.server_side_encryption {
None => None,
Some(v) => Some(
build_header_value(v)
@@ -324,7 +329,7 @@ impl Builder for OssBuilder {
),
};
- let server_side_encryption_key_id = match &self.server_side_encryption_key_id {
+ let server_side_encryption_key_id = match &self.config.server_side_encryption_key_id {
None => None,
Some(v) => Some(
build_header_value(v)
@@ -336,11 +341,11 @@ impl Builder for OssBuilder {
// Load cfg from env first.
cfg = cfg.from_env();
- if let Some(v) = self.access_key_id.take() {
+ if let Some(v) = self.config.access_key_id.take() {
cfg.access_key_id = Some(v);
}
- if let Some(v) = self.access_key_secret.take() {
+ if let Some(v) = self.config.access_key_secret.take() {
cfg.access_key_secret = Some(v);
}
@@ -349,6 +354,7 @@ impl Builder for OssBuilder {
let signer = AliyunOssSigner::new(bucket);
let batch_max_operations = self
+ .config
.batch_max_operations
.unwrap_or(DEFAULT_BATCH_MAX_OPERATIONS);
debug!("Backend build finished");
@@ -360,7 +366,7 @@ impl Builder for OssBuilder {
endpoint,
host,
presign_endpoint,
- allow_anonymous: self.allow_anonymous,
+ allow_anonymous: self.config.allow_anonymous,
signer,
loader,
client,
diff --git a/core/src/services/oss/mod.rs b/core/src/services/oss/mod.rs
index 3954a9885c..4839cff2d3 100644
--- a/core/src/services/oss/mod.rs
+++ b/core/src/services/oss/mod.rs
@@ -17,6 +17,7 @@
mod backend;
pub use backend::OssBuilder as Oss;
+pub use backend::OssConfig;
mod core;
mod error;