You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/04/26 03:50:42 UTC
[incubator-opendal] branch main updated: feat(services/gcs): Add supporting presign support (#2126)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 5edc4fd8 feat(services/gcs): Add supporting presign support (#2126)
5edc4fd8 is described below
commit 5edc4fd8f191fad316f8db150586d3faf5a43b65
Author: silver-ymz <yi...@gmail.com>
AuthorDate: Wed Apr 26 11:50:37 2023 +0800
feat(services/gcs): Add supporting presign support (#2126)
* feat(services/gcs): Add supporting presign support
* fix(services/gcs): Change presign operation to XML API
---
core/src/services/gcs/backend.rs | 34 ++++++++-
core/src/services/gcs/core.rs | 155 ++++++++++++++++++++++++++++++++++++++-
2 files changed, 185 insertions(+), 4 deletions(-)
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 006c3b1b..555a65d5 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -52,7 +52,7 @@ const DEFAULT_GCS_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read
/// - [x] list
/// - [x] scan
/// - [x] copy
-/// - [ ] presign
+/// - [x] presign
/// - [ ] blocking
///
/// # Configuration
@@ -389,6 +389,7 @@ impl Accessor for GcsBackend {
list_with_start_after: true,
scan: true,
copy: true,
+ presign: true,
..Default::default()
});
@@ -520,6 +521,37 @@ impl Accessor for GcsBackend {
GcsPager::new(self.core.clone(), path, "", args.limit(), None),
))
}
+
+ async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
+ // We will not send this request out, just for signing.
+ let mut req = match args.operation() {
+ PresignOperation::Stat(v) => {
+ self.core
+ .gcs_head_object_xml_request(path, v.if_match(), v.if_none_match())?
+ }
+ PresignOperation::Read(v) => self.core.gcs_get_object_xml_request(
+ path,
+ v.range(),
+ v.if_match(),
+ v.if_none_match(),
+ )?,
+ PresignOperation::Write(_) => {
+ self.core
+ .gcs_insert_object_xml_request(path, None, None, None, AsyncBody::Empty)?
+ }
+ };
+
+ self.core.sign_query(&mut req, args.expire()).await?;
+
+ // We don't need this request anymore, consume it directly.
+ let (parts, _) = req.into_parts();
+
+ Ok(RpPresign::new(PresignedRequest::new(
+ parts.method,
+ parts.uri,
+ parts.headers,
+ )))
+ }
}
/// The raw json response returned by [`get`](https://cloud.google.com/storage/docs/json_api/v1/objects/get)
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index 323cd015..1ae48e2d 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -18,6 +18,7 @@
use std::fmt::Debug;
use std::fmt::Formatter;
use std::fmt::Write;
+use std::time::Duration;
use backon::ExponentialBuilder;
use backon::Retryable;
@@ -32,6 +33,7 @@ use http::header::IF_NONE_MATCH;
use http::Request;
use http::Response;
use once_cell::sync::Lazy;
+use reqsign::GoogleCredential;
use reqsign::GoogleCredentialLoader;
use reqsign::GoogleSigner;
use reqsign::GoogleToken;
@@ -85,12 +87,36 @@ impl GcsCore {
}
}
+ fn load_credential(&self) -> Result<GoogleCredential> {
+ let cred = self
+ .credential_loader
+ .load()
+ .map_err(new_request_credential_error)?;
+
+ if let Some(cred) = cred {
+ Ok(cred)
+ } else {
+ Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "no valid credential found",
+ ))
+ }
+ }
+
pub async fn sign<T>(&self, req: &mut Request<T>) -> Result<()> {
let cred = self.load_token().await?;
self.signer.sign(req, &cred).map_err(new_request_sign_error)
}
+ pub async fn sign_query<T>(&self, req: &mut Request<T>, duration: Duration) -> Result<()> {
+ let cred = self.load_credential()?;
+
+ self.signer
+ .sign_query(req, duration, &cred)
+ .map_err(new_request_sign_error)
+ }
+
#[inline]
pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> {
self.client.send(req).await
@@ -133,6 +159,42 @@ impl GcsCore {
Ok(req)
}
+ // It's for presign operation. Gcs only supports query sign over XML API.
+ pub fn gcs_get_object_xml_request(
+ &self,
+ path: &str,
+ range: BytesRange,
+ if_match: Option<&str>,
+ if_none_match: Option<&str>,
+ ) -> Result<Request<AsyncBody>> {
+ let p = build_abs_path(&self.root, path);
+
+ let url = format!(
+ "{}/{}/{}",
+ self.endpoint,
+ self.bucket,
+ percent_encode_path(&p)
+ );
+
+ let mut req = Request::get(&url);
+
+ if let Some(if_match) = if_match {
+ req = req.header(IF_MATCH, if_match);
+ }
+ if let Some(if_none_match) = if_none_match {
+ req = req.header(IF_NONE_MATCH, if_none_match);
+ }
+ if !range.is_full() {
+ req = req.header(http::header::RANGE, range.to_header());
+ }
+
+ let req = req
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
+
+ Ok(req)
+ }
+
pub async fn gcs_get_object(
&self,
path: &str,
@@ -214,12 +276,55 @@ impl GcsCore {
}
}
- pub async fn gcs_get_object_metadata(
+ // It's for presign operation. Gcs only supports query sign over XML API.
+ pub fn gcs_insert_object_xml_request(
+ &self,
+ path: &str,
+ size: Option<usize>,
+ content_type: Option<&str>,
+ cache_control: Option<&str>,
+ body: AsyncBody,
+ ) -> Result<Request<AsyncBody>> {
+ let p = build_abs_path(&self.root, path);
+
+ let url = format!(
+ "{}/{}/{}",
+ self.endpoint,
+ self.bucket,
+ percent_encode_path(&p)
+ );
+
+ let mut req = Request::put(&url);
+
+ req = req.header(CONTENT_LENGTH, size.unwrap_or_default());
+
+ if let Some(content_type) = content_type {
+ req = req.header(CONTENT_TYPE, content_type);
+ }
+
+ if let Some(cache_control) = cache_control {
+ req = req.header(CACHE_CONTROL, cache_control)
+ }
+
+ if let Some(acl) = &self.predefined_acl {
+ req = req.header("x-goog-acl", acl);
+ }
+
+ if let Some(storage_class) = &self.default_storage_class {
+ req = req.header("x-goog-storage-class", storage_class);
+ }
+
+ let req = req.body(body).map_err(new_request_build_error)?;
+
+ Ok(req)
+ }
+
+ pub fn gcs_head_object_request(
&self,
path: &str,
if_match: Option<&str>,
if_none_match: Option<&str>,
- ) -> Result<Response<IncomingAsyncBody>> {
+ ) -> Result<Request<AsyncBody>> {
let p = build_abs_path(&self.root, path);
let url = format!(
@@ -239,10 +344,54 @@ impl GcsCore {
req = req.header(IF_MATCH, if_match);
}
- let mut req = req
+ let req = req
.body(AsyncBody::Empty)
.map_err(new_request_build_error)?;
+ Ok(req)
+ }
+
+ // It's for presign operation. Gcs only supports query sign over XML API.
+ pub fn gcs_head_object_xml_request(
+ &self,
+ path: &str,
+ if_match: Option<&str>,
+ if_none_match: Option<&str>,
+ ) -> Result<Request<AsyncBody>> {
+ let p = build_abs_path(&self.root, path);
+
+ let url = format!(
+ "{}/{}/{}",
+ self.endpoint,
+ self.bucket,
+ percent_encode_path(&p)
+ );
+
+ let mut req = Request::head(&url);
+
+ if let Some(if_none_match) = if_none_match {
+ req = req.header(IF_NONE_MATCH, if_none_match);
+ }
+
+ if let Some(if_match) = if_match {
+ req = req.header(IF_MATCH, if_match);
+ }
+
+ let req = req
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
+
+ Ok(req)
+ }
+
+ pub async fn gcs_get_object_metadata(
+ &self,
+ path: &str,
+ if_match: Option<&str>,
+ if_none_match: Option<&str>,
+ ) -> Result<Response<IncomingAsyncBody>> {
+ let mut req = self.gcs_head_object_request(path, if_match, if_none_match)?;
+
self.sign(&mut req).await?;
self.send(req).await