You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/20 16:58:36 UTC

(opendal) branch main updated: feat(services/tos): implement stat and multipart upload (#7552)

This is an automated email from the ASF dual-hosted git repository.

Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 0c3668265 feat(services/tos): implement stat and multipart upload (#7552)
0c3668265 is described below

commit 0c3668265541de017427861ca3faee81ff6d76ec
Author: Xin Sun <dd...@gmail.com>
AuthorDate: Thu May 21 00:58:30 2026 +0800

    feat(services/tos): implement stat and multipart upload (#7552)
    
    Change-Id: I952d7e9e064d4869d4778edd3f655e6ed3a6bfb4
---
 core/services/tos/src/backend.rs |   6 +-
 core/services/tos/src/core.rs    | 177 +++++++++++++++++++++++++++++++++++++++
 core/services/tos/src/writer.rs  | 127 ++++++++++++++++++++++------
 3 files changed, 282 insertions(+), 28 deletions(-)

diff --git a/core/services/tos/src/backend.rs b/core/services/tos/src/backend.rs
index 3512d93c3..01a93ba39 100644
--- a/core/services/tos/src/backend.rs
+++ b/core/services/tos/src/backend.rs
@@ -204,7 +204,11 @@ impl Builder for TosBuilder {
                     list_with_start_after: true,
                     list_with_recursive: true,
 
-                    stat: false,
+                    stat: true,
+                    stat_with_if_match: true,
+                    stat_with_if_none_match: true,
+                    stat_with_if_modified_since: true,
+                    stat_with_if_unmodified_since: true,
 
                     shared: false,
 
diff --git a/core/services/tos/src/core.rs b/core/services/tos/src/core.rs
index f4e7602ef..3477552fb 100644
--- a/core/services/tos/src/core.rs
+++ b/core/services/tos/src/core.rs
@@ -505,6 +505,183 @@ impl TosCore {
 
         self.send(req).await
     }
+
+    pub async fn tos_initiate_multipart_upload(
+        &self,
+        path: &str,
+        args: &OpWrite,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://{}.{}/{}?uploads",
+            self.bucket,
+            self.endpoint_domain,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::post(&url);
+
+        if let Some(mime) = args.content_type() {
+            req = req.header(CONTENT_TYPE, mime);
+        }
+
+        if let Some(cache_control) = args.cache_control() {
+            req = req.header(http::header::CACHE_CONTROL, cache_control);
+        }
+
+        if let Some(content_encoding) = args.content_encoding() {
+            req = req.header(http::header::CONTENT_ENCODING, content_encoding);
+        }
+
+        if let Some(content_disposition) = args.content_disposition() {
+            req = req.header(http::header::CONTENT_DISPOSITION, content_disposition);
+        }
+
+        if let Some(v) = &self.default_storage_class {
+            req = req.header(
+                http::HeaderName::from_static(constants::X_TOS_STORAGE_CLASS),
+                v,
+            );
+        }
+
+        if let Some(user_metadata) = args.user_metadata() {
+            for (key, value) in user_metadata {
+                req = req.header(format!("{}{}", constants::X_TOS_META_PREFIX, key), value);
+            }
+        }
+
+        req = req.extension(Operation::Write);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub fn tos_upload_part_request(
+        &self,
+        path: &str,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        body: Buffer,
+    ) -> Result<Request<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://{}.{}/{}?partNumber={}&uploadId={}",
+            self.bucket,
+            self.endpoint_domain,
+            percent_encode_path(&p),
+            part_number,
+            percent_encode_query(upload_id)
+        );
+
+        let req = Request::put(&url)
+            .header(CONTENT_LENGTH, size)
+            .extension(Operation::Write)
+            .body(body)
+            .map_err(new_request_build_error)?;
+
+        Ok(req)
+    }
+
+    pub async fn tos_complete_multipart_upload(
+        &self,
+        path: &str,
+        upload_id: &str,
+        parts: Vec<CompleteMultipartUploadRequestPart>,
+        args: &OpWrite,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://{}.{}/{}?uploadId={}",
+            self.bucket,
+            self.endpoint_domain,
+            percent_encode_path(&p),
+            percent_encode_query(upload_id)
+        );
+
+        let content = serde_json::to_string(&CompleteMultipartUploadRequest { parts })
+            .map_err(new_json_serialize_error)?;
+
+        let mut req = Request::post(&url)
+            .header(CONTENT_LENGTH, content.len())
+            .header(CONTENT_TYPE, "application/json");
+
+        if let Some(if_match) = args.if_match() {
+            req = req.header(http::header::IF_MATCH, if_match);
+        }
+        if args.if_not_exists() {
+            req = req.header(http::header::IF_NONE_MATCH, "*");
+        }
+
+        let req = req
+            .extension(Operation::Write)
+            .body(Buffer::from(content))
+            .map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn tos_abort_multipart_upload(
+        &self,
+        path: &str,
+        upload_id: &str,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://{}.{}/{}?uploadId={}",
+            self.bucket,
+            self.endpoint_domain,
+            percent_encode_path(&p),
+            percent_encode_query(upload_id)
+        );
+
+        let req = Request::delete(&url)
+            .extension(Operation::Write)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+}
+
+#[derive(Default, Debug, Serialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct CompleteMultipartUploadRequest {
+    pub parts: Vec<CompleteMultipartUploadRequestPart>,
+}
+
+#[derive(Clone, Default, Debug, Serialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct CompleteMultipartUploadRequestPart {
+    pub part_number: usize,
+    #[serde(rename = "ETag")]
+    pub etag: String,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct InitiateMultipartUploadResult {
+    #[serde(alias = "UploadID")]
+    pub upload_id: String,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct CompleteMultipartUploadResult {
+    pub bucket: String,
+    pub key: String,
+    #[serde(rename = "ETag")]
+    pub etag: String,
+    pub location: String,
+    pub version_id: String,
+    pub code: String,
+    pub message: String,
+    pub request_id: String,
 }
 
 #[derive(Default, Debug, Serialize)]
diff --git a/core/services/tos/src/writer.rs b/core/services/tos/src/writer.rs
index a0a439094..f9b8552f6 100644
--- a/core/services/tos/src/writer.rs
+++ b/core/services/tos/src/writer.rs
@@ -17,6 +17,7 @@
 
 use std::sync::Arc;
 
+use bytes::Buf;
 use constants::X_TOS_OBJECT_SIZE;
 use constants::X_TOS_VERSION_ID;
 use http::StatusCode;
@@ -79,44 +80,116 @@ impl oio::MultipartWrite for TosWriter {
     }
 
     async fn initiate_part(&self) -> Result<String> {
-        // TODO: Implement multipart upload in follow-up PR
-        Err(Error::new(
-            ErrorKind::Unexpected,
-            "Multipart upload not yet implemented for TOS",
-        ))
+        let resp = self
+            .core
+            .tos_initiate_multipart_upload(&self.path, &self.op)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let result: InitiateMultipartUploadResult =
+                    serde_json::from_reader(resp.into_body().reader())
+                        .map_err(new_json_deserialize_error)?;
+
+                Ok(result.upload_id)
+            }
+            _ => Err(parse_error(resp)),
+        }
     }
 
     async fn write_part(
         &self,
-        _upload_id: &str,
-        _part_number: usize,
-        _size: u64,
-        _body: Buffer,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        body: Buffer,
     ) -> Result<oio::MultipartPart> {
-        // TODO: Implement multipart upload in follow-up PR
-        Err(Error::new(
-            ErrorKind::Unexpected,
-            "Multipart upload not yet implemented for TOS",
-        ))
+        let part_number = part_number + 1;
+
+        let req =
+            self.core
+                .tos_upload_part_request(&self.path, upload_id, part_number, size, body)?;
+
+        let resp = self.core.send(req).await?;
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let etag = tos_parse_etag(resp.headers())?
+                    .ok_or_else(|| {
+                        Error::new(
+                            ErrorKind::Unexpected,
+                            "ETag not present in returning response",
+                        )
+                    })?
+                    .to_string();
+
+                Ok(oio::MultipartPart {
+                    part_number,
+                    etag,
+                    checksum: None,
+                    size: None,
+                })
+            }
+            _ => Err(parse_error(resp)),
+        }
     }
 
     async fn complete_part(
         &self,
-        _upload_id: &str,
-        _parts: &[oio::MultipartPart],
+        upload_id: &str,
+        parts: &[oio::MultipartPart],
     ) -> Result<Metadata> {
-        // TODO: Implement multipart upload in follow-up PR
-        Err(Error::new(
-            ErrorKind::Unexpected,
-            "Multipart upload not yet implemented for TOS",
-        ))
+        let parts = parts
+            .iter()
+            .map(|p| CompleteMultipartUploadRequestPart {
+                part_number: p.part_number,
+                etag: p.etag.clone(),
+            })
+            .collect();
+
+        let resp = self
+            .core
+            .tos_complete_multipart_upload(&self.path, upload_id, parts, &self.op)
+            .await?;
+
+        let status = resp.status();
+        let mut meta = TosWriter::parse_header_into_meta(&self.path, resp.headers())?;
+
+        match status {
+            StatusCode::OK => {
+                let ret: CompleteMultipartUploadResult =
+                    serde_json::from_reader(resp.into_body().reader())
+                        .map_err(new_json_deserialize_error)?;
+                if !ret.code.is_empty() {
+                    return Err(Error::new(ErrorKind::Unexpected, ret.message));
+                }
+                if !ret.etag.is_empty() {
+                    // CompleteMultipartUpload response wraps ETag in quotes:
+                    // https://www.volcengine.com/docs/6349/74868
+                    meta.set_etag(ret.etag.trim_matches('"'));
+                }
+                if !ret.version_id.is_empty() {
+                    meta.set_version(&ret.version_id);
+                }
+
+                Ok(meta)
+            }
+            _ => Err(parse_error(resp)),
+        }
     }
 
-    async fn abort_part(&self, _upload_id: &str) -> Result<()> {
-        // TODO: Implement multipart upload in follow-up PR
-        Err(Error::new(
-            ErrorKind::Unexpected,
-            "Multipart upload not yet implemented for TOS",
-        ))
+    async fn abort_part(&self, upload_id: &str) -> Result<()> {
+        let resp = self
+            .core
+            .tos_abort_multipart_upload(&self.path, upload_id)
+            .await?;
+
+        match resp.status() {
+            StatusCode::NO_CONTENT => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
     }
 }