(opendal) branch s3-crc-fast created (now 0f4c58d0e)

[ps...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

PsiACE pushed a change to branch s3-crc-fast
in repository https://gitbox.apache.org/repos/asf/opendal.git


      at 0f4c58d0e refactor: switch S3 crc32c checksum to crc-fast

This branch includes the following new commits:

     new 0f4c58d0e refactor: switch S3 crc32c checksum to crc-fast

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(opendal) 01/01: refactor: switch S3 crc32c checksum to crc-fast

[ps...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

PsiACE pushed a commit to branch s3-crc-fast
in repository https://gitbox.apache.org/repos/asf/opendal.git

commit 0f4c58d0e3e786600828ee1b5e8c648079620f7a
Author: Chojan Shang <ps...@apache.org>
AuthorDate: Mon Jun 1 17:55:40 2026 +0800

    refactor: switch S3 crc32c checksum to crc-fast
---
 core/Cargo.lock              | 11 +----------
 core/services/s3/Cargo.toml  |  2 +-
 core/services/s3/src/core.rs | 17 +++++++++++------
 3 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/core/Cargo.lock b/core/Cargo.lock
index f2d0b3de1..f958ff7da 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -2049,15 +2049,6 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff"
 
-[[package]]
-name = "crc32c"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
-dependencies = [
- "rustc_version",
-]
-
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -7336,7 +7327,7 @@ version = "0.57.0"
 dependencies = [
  "base64 0.22.1",
  "bytes",
- "crc32c",
+ "crc-fast",
  "http 1.4.0",
  "log",
  "md-5 0.11.0",
diff --git a/core/services/s3/Cargo.toml b/core/services/s3/Cargo.toml
index 7e9c0926f..52b4343cf 100644
--- a/core/services/s3/Cargo.toml
+++ b/core/services/s3/Cargo.toml
@@ -33,7 +33,7 @@ all-features = true
 [dependencies]
 base64 = { workspace = true }
 bytes = { workspace = true }
-crc32c = "0.6.6"
+crc-fast = "1.6.0"
 http = { workspace = true }
 log = { workspace = true }
 md-5 = "0.11.0"
diff --git a/core/services/s3/src/core.rs b/core/services/s3/src/core.rs
index 944a1d32c..831ff7c24 100644
--- a/core/services/s3/src/core.rs
+++ b/core/services/s3/src/core.rs
@@ -24,6 +24,8 @@ use base64::Engine;
 use base64::prelude::BASE64_STANDARD;
 use bytes::Bytes;
 use constants::X_AMZ_META_PREFIX;
+use crc_fast::CrcAlgorithm;
+use crc_fast::Digest as CrcDigest;
 use http::HeaderValue;
 use http::Request;
 use http::Response;
@@ -114,6 +116,14 @@ pub(crate) struct S3UploadPartCopyRequest<'a> {
     pub(crate) range: BytesRange,
 }
 
+fn format_crc32c_iter(body: Buffer) -> String {
+    let mut digest = CrcDigest::new(CrcAlgorithm::Crc32Iscsi);
+    body.for_each(|b| digest.update(&b));
+
+    let crc = digest.finalize() as u32;
+    BASE64_STANDARD.encode(crc.to_be_bytes())
+}
+
 impl Debug for S3Core {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("S3Core")
@@ -264,12 +274,7 @@ impl S3Core {
     pub fn calculate_checksum(&self, body: &Buffer) -> Option<String> {
         match self.checksum_algorithm {
             None => None,
-            Some(ChecksumAlgorithm::Crc32c) => {
-                let mut crc = 0u32;
-                body.clone()
-                    .for_each(|b| crc = crc32c::crc32c_append(crc, &b));
-                Some(BASE64_STANDARD.encode(crc.to_be_bytes()))
-            }
+            Some(ChecksumAlgorithm::Crc32c) => Some(format_crc32c_iter(body.clone())),
             Some(ChecksumAlgorithm::Md5) => Some(format_content_md5_iter(body.clone())),
         }
     }