You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/04/28 13:13:41 UTC
[incubator-opendal] 02/05: Implement multipart mixed support
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch multipart-mixed
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit e3ed71f2ef76b3da8829ed8790af52b15af12c84
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Apr 28 19:04:08 2023 +0800
Implement multipart mixed support
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/http_util/mod.rs | 1 +
core/src/raw/http_util/multipart.rs | 350 +++++++++++++++++++++++++++++++++++-
2 files changed, 345 insertions(+), 6 deletions(-)
diff --git a/core/src/raw/http_util/mod.rs b/core/src/raw/http_util/mod.rs
index e579d542..135aa675 100644
--- a/core/src/raw/http_util/mod.rs
+++ b/core/src/raw/http_util/mod.rs
@@ -62,4 +62,5 @@ pub use bytes_content_range::BytesContentRange;
mod multipart;
pub use multipart::FormDataPart;
+pub use multipart::MixedPart;
pub use multipart::Multipart;
diff --git a/core/src/raw/http_util/multipart.rs b/core/src/raw/http_util/multipart.rs
index feb57edf..24cfb8a2 100644
--- a/core/src/raw/http_util/multipart.rs
+++ b/core/src/raw/http_util/multipart.rs
@@ -15,8 +15,13 @@
// specific language governing permissions and limitations
// under the License.
+use std::str::FromStr;
+
use bytes::{Bytes, BytesMut};
-use http::{header::CONTENT_DISPOSITION, HeaderMap, HeaderName, HeaderValue};
+use http::{
+ header::{CONTENT_DISPOSITION, CONTENT_TYPE},
+ HeaderMap, HeaderName, HeaderValue, Method, Request, Uri, Version,
+};
/// Multipart is a builder for multipart/form-data.
#[derive(Debug)]
@@ -143,6 +148,132 @@ impl From<FormDataPart> for Bytes {
}
}
+/// MixedPart is a builder for multipart/mixed part.
+#[derive(Debug)]
+pub struct MixedPart {
+ part_headers: HeaderMap,
+
+ method: Method,
+ uri: Uri,
+ version: Version,
+ headers: HeaderMap,
+ content: Bytes,
+}
+
+impl MixedPart {
+ /// Create a new mixed part with gien uri.
+ pub fn new(uri: &str) -> Self {
+ let mut part_headers = HeaderMap::new();
+ part_headers.insert(CONTENT_TYPE, "application/http".parse().unwrap());
+ part_headers.insert("content-transfer-encoding", "binary".parse().unwrap());
+
+ let uri = Uri::from_str(uri).expect("the uri used to build a mixed part must be valid");
+
+ Self {
+ part_headers,
+ method: Method::GET,
+ uri,
+ version: Version::HTTP_11,
+ headers: HeaderMap::new(),
+ content: Bytes::new(),
+ }
+ }
+
+ /// Build a mixed part from a request.
+ pub fn from_request(req: Request<Bytes>) -> Self {
+ let mut part_headers = HeaderMap::new();
+ part_headers.insert(CONTENT_TYPE, "application/http".parse().unwrap());
+ part_headers.insert("content-transfer-encoding", "binary".parse().unwrap());
+
+ let (parts, body) = req.into_parts();
+
+ Self {
+ part_headers,
+ method: parts.method,
+ // TODO: Maybe we should support query too?;
+ uri: Uri::from_str(parts.uri.path())
+ .expect("the uri used to build a mixed part must be valid"),
+ version: parts.version,
+ headers: parts.headers,
+ content: body,
+ }
+ }
+
+ /// Insert a part header into part.
+ pub fn part_header(mut self, key: HeaderName, value: HeaderValue) -> Self {
+ self.part_headers.insert(key, value);
+ self
+ }
+
+ /// Set the method for request in this part.
+ pub fn method(mut self, method: Method) -> Self {
+ self.method = method;
+ self
+ }
+
+ /// Set the version for request in this part.
+ pub fn version(mut self, version: Version) -> Self {
+ self.version = version;
+ self
+ }
+
+ /// Insert a header into part.
+ pub fn header(mut self, key: HeaderName, value: HeaderValue) -> Self {
+ self.headers.insert(key, value);
+ self
+ }
+
+ /// Set the content for this part.
+ pub fn content(mut self, content: impl Into<Bytes>) -> Self {
+ self.content = content.into();
+ self
+ }
+
+ pub(crate) fn build(self) -> Bytes {
+ let mut bs = BytesMut::new();
+
+ // Write parts headers.
+ for (k, v) in self.part_headers.iter() {
+ bs.extend_from_slice(k.as_str().as_bytes());
+ bs.extend_from_slice(b": ");
+ bs.extend_from_slice(v.as_bytes());
+ bs.extend_from_slice(b"\r\n");
+ }
+
+ // Write request line: `DELETE /container0/blob0 HTTP/1.1`
+ bs.extend_from_slice(b"\r\n");
+ bs.extend_from_slice(self.method.as_str().as_bytes());
+ bs.extend_from_slice(b" ");
+ bs.extend_from_slice(self.uri.path().as_bytes());
+ bs.extend_from_slice(b" ");
+ bs.extend_from_slice(format!("{:?}", self.version).as_bytes());
+ bs.extend_from_slice(b"\r\n");
+
+ // Write request headers.
+ for (k, v) in self.headers.iter() {
+ bs.extend_from_slice(k.as_str().as_bytes());
+ bs.extend_from_slice(b": ");
+ bs.extend_from_slice(v.as_bytes());
+ bs.extend_from_slice(b"\r\n");
+ }
+
+ // Write content.
+ bs.extend_from_slice(b"\r\n");
+ if !self.content.is_empty() {
+ bs.extend_from_slice(&self.content);
+ bs.extend_from_slice(b"\r\n");
+ }
+
+ bs.freeze()
+ }
+}
+
+impl From<MixedPart> for Bytes {
+ fn from(val: MixedPart) -> Bytes {
+ val.build()
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -151,7 +282,7 @@ mod tests {
use pretty_assertions::assert_eq;
#[test]
- fn test_multipart_basic() {
+ fn test_multipart_formdata_basic() {
let multipart = Multipart::new()
.part(FormDataPart::new("foo").content(Bytes::from("bar")))
.part(FormDataPart::new("hello").content(Bytes::from("world")));
@@ -173,9 +304,9 @@ mod tests {
assert_eq!(expected, String::from_utf8(body.to_vec()).unwrap());
}
- /// This test is inspired from <https://docs.aws.amazon.com/AmazonS3/latest/userguide/HTTPPOSTExamples.html>
+ /// This test is inspired by <https://docs.aws.amazon.com/AmazonS3/latest/userguide/HTTPPOSTExamples.html>
#[test]
- fn test_multipart_s3_form_upload() {
+ fn test_multipart_formdata_s3_form_upload() {
let multipart = Multipart::new()
.with_boundary("9431149156168")
.part(FormDataPart::new("key").content("user/eric/MyPicture.jpg"))
@@ -183,7 +314,7 @@ mod tests {
.part(FormDataPart::new("success_action_redirect").content(
"https://awsexamplebucket1.s3.us-west-1.amazonaws.com/successful_upload.html",
))
- .part(FormDataPart::new("Content-Type").content("image/jpeg"))
+ .part(FormDataPart::new("content-type").content("image/jpeg"))
.part(FormDataPart::new("x-amz-meta-uuid").content("14365123651274"))
.part(FormDataPart::new("x-amz-meta-tag").content("Some,Tag,For,Picture"))
.part(FormDataPart::new("AWSAccessKeyId").content("AKIAIOSFODNN7EXAMPLE"))
@@ -206,7 +337,7 @@ content-disposition: form-data; name="success_action_redirect"
https://awsexamplebucket1.s3.us-west-1.amazonaws.com/successful_upload.html
--9431149156168
-content-disposition: form-data; name="Content-Type"
+content-disposition: form-data; name="content-type"
image/jpeg
--9431149156168
@@ -239,6 +370,213 @@ content-disposition: form-data; name="submit"
Upload to Amazon S3
--9431149156168--
+"#;
+
+ assert_eq!(
+ expected,
+ // Rust can't represent `\r` in a string literal, so we
+ // replace `\r\n` with `\n` for comparison
+ String::from_utf8(body.to_vec())
+ .unwrap()
+ .replace("\r\n", "\n")
+ );
+ }
+
+ /// This test is inspired by <https://cloud.google.com/storage/docs/batch>
+ #[test]
+ fn test_multipart_mixed_gcs_batch_metadata() {
+ let multipart = Multipart::new()
+ .with_boundary("===============7330845974216740156==")
+ .part(
+ MixedPart::new("/storage/v1/b/example-bucket/o/obj1")
+ .method(Method::PATCH)
+ .part_header(
+ "content-id".parse().unwrap(),
+ "<b29c5de2-0db4-490b-b421-6a51b598bd22+1>".parse().unwrap(),
+ )
+ .header(
+ "content-type".parse().unwrap(),
+ "application/json".parse().unwrap(),
+ )
+ .header(
+ "accept".parse().unwrap(),
+ "application/json".parse().unwrap(),
+ )
+ .header("content-length".parse().unwrap(), "31".parse().unwrap())
+ .content(r#"{"metadata": {"type": "tabby"}}"#),
+ )
+ .part(
+ MixedPart::new("/storage/v1/b/example-bucket/o/obj2")
+ .method(Method::PATCH)
+ .part_header(
+ "content-id".parse().unwrap(),
+ "<b29c5de2-0db4-490b-b421-6a51b598bd22+2>".parse().unwrap(),
+ )
+ .header(
+ "content-type".parse().unwrap(),
+ "application/json".parse().unwrap(),
+ )
+ .header(
+ "accept".parse().unwrap(),
+ "application/json".parse().unwrap(),
+ )
+ .header("content-length".parse().unwrap(), "32".parse().unwrap())
+ .content(r#"{"metadata": {"type": "tuxedo"}}"#),
+ )
+ .part(
+ MixedPart::new("/storage/v1/b/example-bucket/o/obj3")
+ .method(Method::PATCH)
+ .part_header(
+ "content-id".parse().unwrap(),
+ "<b29c5de2-0db4-490b-b421-6a51b598bd22+3>".parse().unwrap(),
+ )
+ .header(
+ "content-type".parse().unwrap(),
+ "application/json".parse().unwrap(),
+ )
+ .header(
+ "accept".parse().unwrap(),
+ "application/json".parse().unwrap(),
+ )
+ .header("content-length".parse().unwrap(), "32".parse().unwrap())
+ .content(r#"{"metadata": {"type": "calico"}}"#),
+ );
+
+ let (_, body) = multipart.build();
+
+ let expected = r#"--===============7330845974216740156==
+content-type: application/http
+content-transfer-encoding: binary
+content-id: <b29c5de2-0db4-490b-b421-6a51b598bd22+1>
+
+PATCH /storage/v1/b/example-bucket/o/obj1 HTTP/1.1
+content-type: application/json
+accept: application/json
+content-length: 31
+
+{"metadata": {"type": "tabby"}}
+--===============7330845974216740156==
+content-type: application/http
+content-transfer-encoding: binary
+content-id: <b29c5de2-0db4-490b-b421-6a51b598bd22+2>
+
+PATCH /storage/v1/b/example-bucket/o/obj2 HTTP/1.1
+content-type: application/json
+accept: application/json
+content-length: 32
+
+{"metadata": {"type": "tuxedo"}}
+--===============7330845974216740156==
+content-type: application/http
+content-transfer-encoding: binary
+content-id: <b29c5de2-0db4-490b-b421-6a51b598bd22+3>
+
+PATCH /storage/v1/b/example-bucket/o/obj3 HTTP/1.1
+content-type: application/json
+accept: application/json
+content-length: 32
+
+{"metadata": {"type": "calico"}}
+--===============7330845974216740156==--
+"#;
+
+ assert_eq!(
+ expected,
+ // Rust can't represent `\r` in a string literal, so we
+ // replace `\r\n` with `\n` for comparison
+ String::from_utf8(body.to_vec())
+ .unwrap()
+ .replace("\r\n", "\n")
+ );
+ }
+
+ /// This test is inspired by <https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=azure-ad>
+ #[test]
+ fn test_multipart_mixed_azblob_batch_delete() {
+ let multipart = Multipart::new()
+ .with_boundary("batch_357de4f7-6d0b-4e02-8cd2-6361411a9525")
+ .part(
+ MixedPart::new("/container0/blob0")
+ .method(Method::DELETE)
+ .part_header("content-id".parse().unwrap(), "0".parse().unwrap())
+ .header(
+ "x-ms-date".parse().unwrap(),
+ "Thu, 14 Jun 2018 16:46:54 GMT".parse().unwrap(),
+ )
+ .header(
+ "authorization".parse().unwrap(),
+ "SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE="
+ .parse()
+ .unwrap(),
+ )
+ .header("content-length".parse().unwrap(), "0".parse().unwrap()),
+ )
+ .part(
+ MixedPart::new("/container1/blob1")
+ .method(Method::DELETE)
+ .part_header("content-id".parse().unwrap(), "1".parse().unwrap())
+ .header(
+ "x-ms-date".parse().unwrap(),
+ "Thu, 14 Jun 2018 16:46:54 GMT".parse().unwrap(),
+ )
+ .header(
+ "authorization".parse().unwrap(),
+ "SharedKey account:IvCoYDQ+0VcaA/hKFjUmQmIxXv2RT3XwwTsOTHL39HI="
+ .parse()
+ .unwrap(),
+ )
+ .header("content-length".parse().unwrap(), "0".parse().unwrap()),
+ )
+ .part(
+ MixedPart::new("/container2/blob2")
+ .method(Method::DELETE)
+ .part_header("content-id".parse().unwrap(), "2".parse().unwrap())
+ .header(
+ "x-ms-date".parse().unwrap(),
+ "Thu, 14 Jun 2018 16:46:54 GMT".parse().unwrap(),
+ )
+ .header(
+ "authorization".parse().unwrap(),
+ "SharedKey account:S37N2JTjcmOQVLHLbDmp2johz+KpTJvKhbVc4M7+UqI="
+ .parse()
+ .unwrap(),
+ )
+ .header("content-length".parse().unwrap(), "0".parse().unwrap()),
+ );
+
+ let (_, body) = multipart.build();
+
+ let expected = r#"--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525
+content-type: application/http
+content-transfer-encoding: binary
+content-id: 0
+
+DELETE /container0/blob0 HTTP/1.1
+x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT
+authorization: SharedKey account:G4jjBXA7LI/RnWKIOQ8i9xH4p76pAQ+4Fs4R1VxasaE=
+content-length: 0
+
+--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525
+content-type: application/http
+content-transfer-encoding: binary
+content-id: 1
+
+DELETE /container1/blob1 HTTP/1.1
+x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT
+authorization: SharedKey account:IvCoYDQ+0VcaA/hKFjUmQmIxXv2RT3XwwTsOTHL39HI=
+content-length: 0
+
+--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525
+content-type: application/http
+content-transfer-encoding: binary
+content-id: 2
+
+DELETE /container2/blob2 HTTP/1.1
+x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT
+authorization: SharedKey account:S37N2JTjcmOQVLHLbDmp2johz+KpTJvKhbVc4M7+UqI=
+content-length: 0
+
+--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525--
"#;
assert_eq!(