You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/20 16:58:36 UTC
(opendal) branch main updated: feat(services/tos): implement stat and multipart upload (#7552)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 0c3668265 feat(services/tos): implement stat and multipart upload (#7552)
0c3668265 is described below
commit 0c3668265541de017427861ca3faee81ff6d76ec
Author: Xin Sun <dd...@gmail.com>
AuthorDate: Thu May 21 00:58:30 2026 +0800
feat(services/tos): implement stat and multipart upload (#7552)
Change-Id: I952d7e9e064d4869d4778edd3f655e6ed3a6bfb4
---
core/services/tos/src/backend.rs | 6 +-
core/services/tos/src/core.rs | 177 +++++++++++++++++++++++++++++++++++++++
core/services/tos/src/writer.rs | 127 ++++++++++++++++++++++------
3 files changed, 282 insertions(+), 28 deletions(-)
diff --git a/core/services/tos/src/backend.rs b/core/services/tos/src/backend.rs
index 3512d93c3..01a93ba39 100644
--- a/core/services/tos/src/backend.rs
+++ b/core/services/tos/src/backend.rs
@@ -204,7 +204,11 @@ impl Builder for TosBuilder {
list_with_start_after: true,
list_with_recursive: true,
- stat: false,
+ stat: true,
+ stat_with_if_match: true,
+ stat_with_if_none_match: true,
+ stat_with_if_modified_since: true,
+ stat_with_if_unmodified_since: true,
shared: false,
diff --git a/core/services/tos/src/core.rs b/core/services/tos/src/core.rs
index f4e7602ef..3477552fb 100644
--- a/core/services/tos/src/core.rs
+++ b/core/services/tos/src/core.rs
@@ -505,6 +505,183 @@ impl TosCore {
self.send(req).await
}
+
+ pub async fn tos_initiate_multipart_upload(
+ &self,
+ path: &str,
+ args: &OpWrite,
+ ) -> Result<Response<Buffer>> {
+ let p = build_abs_path(&self.root, path);
+
+ let url = format!(
+ "https://{}.{}/{}?uploads",
+ self.bucket,
+ self.endpoint_domain,
+ percent_encode_path(&p)
+ );
+
+ let mut req = Request::post(&url);
+
+ if let Some(mime) = args.content_type() {
+ req = req.header(CONTENT_TYPE, mime);
+ }
+
+ if let Some(cache_control) = args.cache_control() {
+ req = req.header(http::header::CACHE_CONTROL, cache_control);
+ }
+
+ if let Some(content_encoding) = args.content_encoding() {
+ req = req.header(http::header::CONTENT_ENCODING, content_encoding);
+ }
+
+ if let Some(content_disposition) = args.content_disposition() {
+ req = req.header(http::header::CONTENT_DISPOSITION, content_disposition);
+ }
+
+ if let Some(v) = &self.default_storage_class {
+ req = req.header(
+ http::HeaderName::from_static(constants::X_TOS_STORAGE_CLASS),
+ v,
+ );
+ }
+
+ if let Some(user_metadata) = args.user_metadata() {
+ for (key, value) in user_metadata {
+ req = req.header(format!("{}{}", constants::X_TOS_META_PREFIX, key), value);
+ }
+ }
+
+ req = req.extension(Operation::Write);
+
+ let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+ self.send(req).await
+ }
+
+ pub fn tos_upload_part_request(
+ &self,
+ path: &str,
+ upload_id: &str,
+ part_number: usize,
+ size: u64,
+ body: Buffer,
+ ) -> Result<Request<Buffer>> {
+ let p = build_abs_path(&self.root, path);
+
+ let url = format!(
+ "https://{}.{}/{}?partNumber={}&uploadId={}",
+ self.bucket,
+ self.endpoint_domain,
+ percent_encode_path(&p),
+ part_number,
+ percent_encode_query(upload_id)
+ );
+
+ let req = Request::put(&url)
+ .header(CONTENT_LENGTH, size)
+ .extension(Operation::Write)
+ .body(body)
+ .map_err(new_request_build_error)?;
+
+ Ok(req)
+ }
+
+ pub async fn tos_complete_multipart_upload(
+ &self,
+ path: &str,
+ upload_id: &str,
+ parts: Vec<CompleteMultipartUploadRequestPart>,
+ args: &OpWrite,
+ ) -> Result<Response<Buffer>> {
+ let p = build_abs_path(&self.root, path);
+
+ let url = format!(
+ "https://{}.{}/{}?uploadId={}",
+ self.bucket,
+ self.endpoint_domain,
+ percent_encode_path(&p),
+ percent_encode_query(upload_id)
+ );
+
+ let content = serde_json::to_string(&CompleteMultipartUploadRequest { parts })
+ .map_err(new_json_serialize_error)?;
+
+ let mut req = Request::post(&url)
+ .header(CONTENT_LENGTH, content.len())
+ .header(CONTENT_TYPE, "application/json");
+
+ if let Some(if_match) = args.if_match() {
+ req = req.header(http::header::IF_MATCH, if_match);
+ }
+ if args.if_not_exists() {
+ req = req.header(http::header::IF_NONE_MATCH, "*");
+ }
+
+ let req = req
+ .extension(Operation::Write)
+ .body(Buffer::from(content))
+ .map_err(new_request_build_error)?;
+
+ self.send(req).await
+ }
+
+ pub async fn tos_abort_multipart_upload(
+ &self,
+ path: &str,
+ upload_id: &str,
+ ) -> Result<Response<Buffer>> {
+ let p = build_abs_path(&self.root, path);
+
+ let url = format!(
+ "https://{}.{}/{}?uploadId={}",
+ self.bucket,
+ self.endpoint_domain,
+ percent_encode_path(&p),
+ percent_encode_query(upload_id)
+ );
+
+ let req = Request::delete(&url)
+ .extension(Operation::Write)
+ .body(Buffer::new())
+ .map_err(new_request_build_error)?;
+
+ self.send(req).await
+ }
+}
+
+#[derive(Default, Debug, Serialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct CompleteMultipartUploadRequest {
+ pub parts: Vec<CompleteMultipartUploadRequestPart>,
+}
+
+#[derive(Clone, Default, Debug, Serialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct CompleteMultipartUploadRequestPart {
+ pub part_number: usize,
+ #[serde(rename = "ETag")]
+ pub etag: String,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct InitiateMultipartUploadResult {
+ #[serde(alias = "UploadID")]
+ pub upload_id: String,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct CompleteMultipartUploadResult {
+ pub bucket: String,
+ pub key: String,
+ #[serde(rename = "ETag")]
+ pub etag: String,
+ pub location: String,
+ pub version_id: String,
+ pub code: String,
+ pub message: String,
+ pub request_id: String,
}
#[derive(Default, Debug, Serialize)]
diff --git a/core/services/tos/src/writer.rs b/core/services/tos/src/writer.rs
index a0a439094..f9b8552f6 100644
--- a/core/services/tos/src/writer.rs
+++ b/core/services/tos/src/writer.rs
@@ -17,6 +17,7 @@
use std::sync::Arc;
+use bytes::Buf;
use constants::X_TOS_OBJECT_SIZE;
use constants::X_TOS_VERSION_ID;
use http::StatusCode;
@@ -79,44 +80,116 @@ impl oio::MultipartWrite for TosWriter {
}
async fn initiate_part(&self) -> Result<String> {
- // TODO: Implement multipart upload in follow-up PR
- Err(Error::new(
- ErrorKind::Unexpected,
- "Multipart upload not yet implemented for TOS",
- ))
+ let resp = self
+ .core
+ .tos_initiate_multipart_upload(&self.path, &self.op)
+ .await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let result: InitiateMultipartUploadResult =
+ serde_json::from_reader(resp.into_body().reader())
+ .map_err(new_json_deserialize_error)?;
+
+ Ok(result.upload_id)
+ }
+ _ => Err(parse_error(resp)),
+ }
}
async fn write_part(
&self,
- _upload_id: &str,
- _part_number: usize,
- _size: u64,
- _body: Buffer,
+ upload_id: &str,
+ part_number: usize,
+ size: u64,
+ body: Buffer,
) -> Result<oio::MultipartPart> {
- // TODO: Implement multipart upload in follow-up PR
- Err(Error::new(
- ErrorKind::Unexpected,
- "Multipart upload not yet implemented for TOS",
- ))
+ let part_number = part_number + 1;
+
+ let req =
+ self.core
+ .tos_upload_part_request(&self.path, upload_id, part_number, size, body)?;
+
+ let resp = self.core.send(req).await?;
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let etag = tos_parse_etag(resp.headers())?
+ .ok_or_else(|| {
+ Error::new(
+ ErrorKind::Unexpected,
+ "ETag not present in returning response",
+ )
+ })?
+ .to_string();
+
+ Ok(oio::MultipartPart {
+ part_number,
+ etag,
+ checksum: None,
+ size: None,
+ })
+ }
+ _ => Err(parse_error(resp)),
+ }
}
async fn complete_part(
&self,
- _upload_id: &str,
- _parts: &[oio::MultipartPart],
+ upload_id: &str,
+ parts: &[oio::MultipartPart],
) -> Result<Metadata> {
- // TODO: Implement multipart upload in follow-up PR
- Err(Error::new(
- ErrorKind::Unexpected,
- "Multipart upload not yet implemented for TOS",
- ))
+ let parts = parts
+ .iter()
+ .map(|p| CompleteMultipartUploadRequestPart {
+ part_number: p.part_number,
+ etag: p.etag.clone(),
+ })
+ .collect();
+
+ let resp = self
+ .core
+ .tos_complete_multipart_upload(&self.path, upload_id, parts, &self.op)
+ .await?;
+
+ let status = resp.status();
+ let mut meta = TosWriter::parse_header_into_meta(&self.path, resp.headers())?;
+
+ match status {
+ StatusCode::OK => {
+ let ret: CompleteMultipartUploadResult =
+ serde_json::from_reader(resp.into_body().reader())
+ .map_err(new_json_deserialize_error)?;
+ if !ret.code.is_empty() {
+ return Err(Error::new(ErrorKind::Unexpected, ret.message));
+ }
+ if !ret.etag.is_empty() {
+ // CompleteMultipartUpload response wraps ETag in quotes:
+ // https://www.volcengine.com/docs/6349/74868
+ meta.set_etag(ret.etag.trim_matches('"'));
+ }
+ if !ret.version_id.is_empty() {
+ meta.set_version(&ret.version_id);
+ }
+
+ Ok(meta)
+ }
+ _ => Err(parse_error(resp)),
+ }
}
- async fn abort_part(&self, _upload_id: &str) -> Result<()> {
- // TODO: Implement multipart upload in follow-up PR
- Err(Error::new(
- ErrorKind::Unexpected,
- "Multipart upload not yet implemented for TOS",
- ))
+ async fn abort_part(&self, upload_id: &str) -> Result<()> {
+ let resp = self
+ .core
+ .tos_abort_multipart_upload(&self.path, upload_id)
+ .await?;
+
+ match resp.status() {
+ StatusCode::NO_CONTENT => Ok(()),
+ _ => Err(parse_error(resp)),
+ }
}
}