You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/03/08 15:56:39 UTC
(opendal) branch main updated: feat(services/gcs): Implement multipart/related and use it with Gcs (#5691)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new befd8e7fb feat(services/gcs): Implement multipart/related and use it with Gcs (#5691)
befd8e7fb is described below
commit befd8e7fb6dee4b3b8bbece1ed6421724e021e1c
Author: William Linna <wi...@gmail.com>
AuthorDate: Sat Mar 8 17:56:33 2025 +0200
feat(services/gcs): Implement multipart/related and use it with Gcs (#5691)
* Implement multipart/related and use it with Gcs
* Fix formatting
* Implement Default for RelatedPart (to silence new_without_default warning of clippy)
* Rustfmt http_util/mod.rs
* Add a simple unit test
* Run cargo fmt
---
core/src/raw/http_util/mod.rs | 1 +
core/src/raw/http_util/multipart.rs | 148 ++++++++++++++++++++++++++++++++++++
core/src/services/gcs/core.rs | 16 +++-
3 files changed, 162 insertions(+), 3 deletions(-)
diff --git a/core/src/raw/http_util/mod.rs b/core/src/raw/http_util/mod.rs
index 4b3619b66..86ebce174 100644
--- a/core/src/raw/http_util/mod.rs
+++ b/core/src/raw/http_util/mod.rs
@@ -74,3 +74,4 @@ pub use multipart::FormDataPart;
pub use multipart::MixedPart;
pub use multipart::Multipart;
pub use multipart::Part;
+pub use multipart::RelatedPart;
diff --git a/core/src/raw/http_util/multipart.rs b/core/src/raw/http_util/multipart.rs
index 94a0805ae..556677c67 100644
--- a/core/src/raw/http_util/multipart.rs
+++ b/core/src/raw/http_util/multipart.rs
@@ -523,6 +523,111 @@ impl Part for MixedPart {
}
}
+/// RelatedPart is a builder for multipart/related part.
+pub struct RelatedPart {
+ /// Common
+ headers: HeaderMap,
+ content: Buffer,
+}
+
+impl Default for RelatedPart {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl RelatedPart {
+ /// Create a new related
+ pub fn new() -> Self {
+ Self {
+ headers: HeaderMap::new(),
+ content: Buffer::new(),
+ }
+ }
+
+ /// Build a mixed part from a request.
+ pub fn from_request(req: Request<Buffer>) -> Self {
+ let (parts, content) = req.into_parts();
+
+ Self {
+ headers: parts.headers,
+ content,
+ }
+ }
+
+ /// Consume a mixed part to build a response.
+ pub fn into_response(mut self) -> Response<Buffer> {
+ let mut builder = Response::builder();
+
+ // Swap headers directly instead of copy the entire map.
+ mem::swap(builder.headers_mut().unwrap(), &mut self.headers);
+
+ builder
+ .body(self.content)
+ .expect("a related part must be valid response")
+ }
+
+ /// Insert a header into part.
+ pub fn header(mut self, key: HeaderName, value: HeaderValue) -> Self {
+ self.headers.insert(key, value);
+ self
+ }
+
+ /// Set the content for this part.
+ pub fn content(mut self, content: impl Into<Buffer>) -> Self {
+ self.content = content.into();
+ self
+ }
+}
+
+impl Part for RelatedPart {
+ const TYPE: &'static str = "related";
+
+ fn format(self) -> Buffer {
+ // This is what multipart/related body might look for an insert in GCS
+ // https://cloud.google.com/storage/docs/uploading-objects
+ /*
+ --separator_string
+ Content-Type: application/json; charset=UTF-8
+
+ {"name":"my-document.txt"}
+
+ --separator_string
+ Content-Type: text/plain
+
+ This is a text file.
+ --separator_string--
+ */
+
+ let mut bufs = Vec::with_capacity(3);
+ let mut bs = BytesMut::new();
+
+ // Write request headers.
+ for (k, v) in self.headers.iter() {
+ bs.extend_from_slice(k.as_str().as_bytes());
+ bs.extend_from_slice(b": ");
+ bs.extend_from_slice(v.as_bytes());
+ bs.extend_from_slice(b"\r\n");
+ }
+ bs.extend_from_slice(b"\r\n");
+ bufs.push(Buffer::from(bs.freeze()));
+
+ if !self.content.is_empty() {
+ bufs.push(self.content);
+ bufs.push(Buffer::from("\r\n"))
+ }
+
+ bufs.into_iter().flatten().collect()
+ }
+
+ fn parse(_s: &str) -> Result<Self> {
+ Err(Error::new(
+ ErrorKind::Unsupported,
+ "parsing multipart/related is not supported",
+ ))
+ }
+}
+
#[cfg(test)]
mod tests {
use http::header::CONTENT_TYPE;
@@ -1029,4 +1134,47 @@ Content-Length: 846
Some(StatusCode::from_u16(200).unwrap())
);
}
+
+ #[test]
+ fn test_multipart_related_gcs_simple() {
+ // This is what multipart/related body might look for an insert in GCS
+ // https://cloud.google.com/storage/docs/uploading-objects
+ let expected = r#"--separator_string
+content-type: application/json; charset=UTF-8
+
+{"name":"my-document.txt"}
+--separator_string
+content-type: text/plain
+
+This is a text file.
+--separator_string--
+"#;
+
+ let multipart = Multipart::new()
+ .with_boundary("separator_string")
+ .part(
+ RelatedPart::new()
+ .header(
+ "Content-Type".parse().unwrap(),
+ "application/json; charset=UTF-8".parse().unwrap(),
+ )
+ .content(r#"{"name":"my-document.txt"}"#),
+ )
+ .part(
+ RelatedPart::new()
+ .header(
+ "Content-Type".parse().unwrap(),
+ "text/plain".parse().unwrap(),
+ )
+ .content("This is a text file."),
+ );
+
+ let bs = multipart.build();
+
+ let output = String::from_utf8(bs.to_bytes().to_vec())
+ .unwrap()
+ .replace("\r\n", "\n");
+
+ assert_eq!(output, expected);
+ }
}
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index b7db51e99..1191edd84 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -309,18 +309,26 @@ impl GcsCore {
Ok(req)
} else {
let mut multipart = Multipart::new();
- let metadata_part = FormDataPart::new("metadata")
+ let metadata_part = RelatedPart::new()
.header(
CONTENT_TYPE,
"application/json; charset=UTF-8".parse().unwrap(),
)
.content(
serde_json::to_vec(&request_metadata)
- .expect("metadata serialization should success"),
+ .expect("metadata serialization should succeed"),
);
multipart = multipart.part(metadata_part);
- let media_part = FormDataPart::new("media").content(body);
+ // Content-Type must be set, even if it is set in the metadata part
+ let content_type = op
+ .content_type()
+ .unwrap_or("application/octet-stream")
+ .parse()
+ .expect("Failed to parse content-type");
+ let media_part = RelatedPart::new()
+ .header(CONTENT_TYPE, content_type)
+ .content(body);
multipart = multipart.part(media_part);
let req = multipart.apply(Request::post(url))?;
@@ -662,6 +670,8 @@ impl InsertRequestMetadata<'_> {
&& self.content_encoding.is_none()
&& self.storage_class.is_none()
&& self.cache_control.is_none()
+ // We could also put content-encoding in the url parameters
+ && self.content_encoding.is_none()
&& self.metadata.is_none()
}
}