You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/04/05 11:33:23 UTC
[incubator-opendal] branch main updated: feat(s3): allow users to specify storage_class (#1854)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new eca0e957 feat(s3): allow users to specify storage_class (#1854)
eca0e957 is described below
commit eca0e957cd83023444cb9e952cbfab3f372f0ca7
Author: Jian Zeng <an...@gmail.com>
AuthorDate: Wed Apr 5 19:33:18 2023 +0800
feat(s3): allow users to specify storage_class (#1854)
* feat(s3): allow users to specify storage_class
Signed-off-by: Jian Zeng <an...@gmail.com>
* fix: OpWrite should be unaware of storage_class
Signed-off-by: Jian Zeng <an...@gmail.com>
* style: make clippy happy about s3_put_object_request
Signed-off-by: Jian Zeng <an...@gmail.com>
* refactor: rename to default_storage_class
Signed-off-by: Jian Zeng <an...@gmail.com>
---------
Signed-off-by: Jian Zeng <an...@gmail.com>
---
core/src/services/s3/backend.rs | 52 ++++++++++++++++++++++++++++++++++++++++-
core/src/types/ops.rs | 8 +------
2 files changed, 52 insertions(+), 8 deletions(-)
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index da00b864..6bc626ec 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -73,6 +73,7 @@ mod constants {
"x-amz-server-side-encryption-customer-key-md5";
pub const X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID: &str =
"x-amz-server-side-encryption-aws-kms-key-id";
+ pub const X_AMZ_STORAGE_CLASS: &str = "x-amz-storage-class";
pub const RESPONSE_CONTENT_DISPOSITION: &str = "response-content-disposition";
pub const RESPONSE_CACHE_CONTROL: &str = "response-cache-control";
@@ -100,6 +101,7 @@ mod constants {
/// - `access_key_id`: Set the access_key_id for backend.
/// - `secret_access_key`: Set the secret_access_key for backend.
/// - `security_token`: Set the security_token for backend.
+/// - `default_storage_class`: Set the default storage_class for backend.
/// - `server_side_encryption`: Set the server_side_encryption for backend.
/// - `server_side_encryption_aws_kms_key_id`: Set the server_side_encryption_aws_kms_key_id for backend.
/// - `server_side_encryption_customer_algorithm`: Set the server_side_encryption_customer_algorithm for backend.
@@ -314,6 +316,7 @@ pub struct S3Builder {
server_side_encryption_customer_algorithm: Option<String>,
server_side_encryption_customer_key: Option<String>,
server_side_encryption_customer_key_md5: Option<String>,
+ default_storage_class: Option<String>,
/// temporary credentials, check the official [doc](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) for detail
security_token: Option<String>,
@@ -336,7 +339,8 @@ impl Debug for S3Builder {
.field("role_arn", &self.role_arn)
.field("external_id", &self.external_id)
.field("disable_config_load", &self.disable_config_load)
- .field("enable_virtual_host_style", &self.enable_virtual_host_style);
+ .field("enable_virtual_host_style", &self.enable_virtual_host_style)
+ .field("default_storage_class", &self.default_storage_class);
if self.access_key_id.is_some() {
d.field("access_key_id", &"<redacted>");
@@ -463,6 +467,26 @@ impl S3Builder {
self
}
+ /// Set default storage_class for this backend.
+ ///
+ /// Available values:
+ /// - `DEEP_ARCHIVE`
+ /// - `GLACIER`
+ /// - `GLACIER_IR`
+ /// - `INTELLIGENT_TIERING`
+ /// - `ONEZONE_IA`
+ /// - `OUTPOSTS`
+ /// - `REDUCED_REDUNDANCY`
+ /// - `STANDARD`
+ /// - `STANDARD_IA`
+ pub fn default_storage_class(&mut self, v: &str) -> &mut Self {
+ if !v.is_empty() {
+ self.default_storage_class = Some(v.to_string())
+ }
+
+ self
+ }
+
/// Set server_side_encryption for this backend.
///
/// Available values: `AES256`, `aws:kms`.
@@ -743,6 +767,8 @@ impl Builder for S3Builder {
map.get("enable_virtual_host_style")
.filter(|v| *v == "on" || *v == "true")
.map(|_| builder.enable_virtual_host_style());
+ map.get("default_storage_class")
+ .map(|v| builder.default_storage_class(v));
builder
}
@@ -763,6 +789,18 @@ impl Builder for S3Builder {
}?;
debug!("backend use bucket {}", &bucket);
+ let default_storage_class = match &self.default_storage_class {
+ None => None,
+ Some(v) => Some(v.parse().map_err(|e| {
+ Error::new(
+ ErrorKind::ConfigInvalid,
+ "default_storage_class value is invalid",
+ )
+ .with_context("value", v)
+ .set_source(e)
+ })?),
+ };
+
let server_side_encryption = match &self.server_side_encryption {
None => None,
Some(v) => Some(v.parse().map_err(|e| {
@@ -916,6 +954,7 @@ impl Builder for S3Builder {
server_side_encryption_customer_algorithm,
server_side_encryption_customer_key,
server_side_encryption_customer_key_md5,
+ default_storage_class,
})
}
}
@@ -935,6 +974,7 @@ pub struct S3Backend {
server_side_encryption_customer_algorithm: Option<HeaderValue>,
server_side_encryption_customer_key: Option<HeaderValue>,
server_side_encryption_customer_key_md5: Option<HeaderValue>,
+ default_storage_class: Option<HeaderValue>,
}
impl S3Backend {
@@ -1335,6 +1375,11 @@ impl S3Backend {
req = req.header(CACHE_CONTROL, cache_control)
}
+ // Set storage class header
+ if let Some(v) = &self.default_storage_class {
+ req = req.header(HeaderName::from_static(constants::X_AMZ_STORAGE_CLASS), v);
+ }
+
// Set SSE headers.
req = self.insert_sse_headers(req, true);
@@ -1451,6 +1496,11 @@ impl S3Backend {
req = req.header(CACHE_CONTROL, cache_control)
}
+ // Set storage class header
+ if let Some(v) = &self.default_storage_class {
+ req = req.header(HeaderName::from_static(constants::X_AMZ_STORAGE_CLASS), v);
+ }
+
// Set SSE headers.
let req = self.insert_sse_headers(req, true);
diff --git a/core/src/types/ops.rs b/core/src/types/ops.rs
index 97af062f..3abcc2d8 100644
--- a/core/src/types/ops.rs
+++ b/core/src/types/ops.rs
@@ -316,13 +316,7 @@ impl OpWrite {
///
/// If input path is not a file path, an error will be returned.
pub fn new() -> Self {
- Self {
- append: false,
-
- content_type: None,
- content_disposition: None,
- cache_control: None,
- }
+ Self::default()
}
pub(crate) fn with_append(mut self) -> Self {