You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/08/24 07:52:56 UTC
(opendal) 01/01: feat(services/s3): support S3 Express session auth
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/s3-express-session-auth
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 5cce8c3c2736091e64a5664443b50af2b2a429cb
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Aug 24 15:52:43 2026 +0800
feat(services/s3): support S3 Express session auth
---
core/Cargo.lock | 17 ++---
core/services/s3/Cargo.toml | 6 +-
core/services/s3/src/backend.rs | 143 ++++++++++++++++++++++++++++++++++++++--
core/services/s3/src/core.rs | 79 ++++++++++++++++++++--
core/services/s3/src/docs.md | 24 +++++++
5 files changed, 248 insertions(+), 21 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 68fa68fc1..a2b71601a 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -9197,9 +9197,9 @@ dependencies = [
[[package]]
name = "reqsign-aws-core"
-version = "3.1.0"
+version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c"
+checksum = "bac4749b7dfa7bfaccd01eb03e9dc795ed37e3f20d6f0f38e2c67ee85ad6bc86"
dependencies = [
"bytes",
"form_urlencoded",
@@ -9218,9 +9218,9 @@ dependencies = [
[[package]]
name = "reqsign-aws-v4"
-version = "3.2.0"
+version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a0c499f4ed12d04c3d4c78fe4cb01aee22c9dae22848c14db2c6313d9df9f43"
+checksum = "ff250f0fd0b913fbd565e405acc553da0f13bde30bfb5403178c9d0313cdc15f"
dependencies = [
"bytes",
"http 1.4.2",
@@ -9254,9 +9254,9 @@ dependencies = [
[[package]]
name = "reqsign-core"
-version = "3.3.0"
+version = "3.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa"
+checksum = "ff052daffb0599681c50f85c59e7236438976efe991ab864edd9f3b235501a0f"
dependencies = [
"anyhow",
"base64 0.23.0",
@@ -9267,6 +9267,7 @@ dependencies = [
"http 1.4.2",
"jiff",
"log",
+ "mea",
"percent-encoding",
"rsa",
"serde",
@@ -9278,9 +9279,9 @@ dependencies = [
[[package]]
name = "reqsign-file-read-tokio"
-version = "3.0.5"
+version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95c3371bfc7e5c7f9627a04133af3583fd6c28715e7c83f79db38f3b384f535f"
+checksum = "b3235df90a6bca681aa47dd86f2393d122a6d77042aa8a7c81e218cd45c5bfc0"
dependencies = [
"anyhow",
"reqsign-core",
diff --git a/core/services/s3/Cargo.toml b/core/services/s3/Cargo.toml
index 1502cd361..bc0d60720 100644
--- a/core/services/s3/Cargo.toml
+++ b/core/services/s3/Cargo.toml
@@ -42,9 +42,9 @@ opendal-core = { path = "../../core", version = "0.58.2", default-features = fal
"reqsign",
] }
quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
-reqsign-aws-v4 = { version = "3.2.0", default-features = false }
-reqsign-core = { version = "3.3.0", default-features = false }
-reqsign-file-read-tokio = { version = "3.0.5", default-features = false }
+reqsign-aws-v4 = { version = "3.3.0", default-features = false }
+reqsign-core = { version = "3.3.1", default-features = false }
+reqsign-file-read-tokio = { version = "3.0.6", default-features = false }
serde = { workspace = true, features = ["derive"] }
url = { workspace = true }
diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs
index 160e737d2..38d8ab94c 100644
--- a/core/services/s3/src/backend.rs
+++ b/core/services/s3/src/backend.rs
@@ -32,6 +32,8 @@ use reqsign_aws_v4::AssumeRoleCredentialProvider;
use reqsign_aws_v4::Credential;
use reqsign_aws_v4::DefaultCredentialProvider;
use reqsign_aws_v4::RequestSigner as AwsV4Signer;
+use reqsign_aws_v4::S3ExpressSessionConfig;
+use reqsign_aws_v4::S3ExpressSessionProvider;
use reqsign_aws_v4::StaticCredentialProvider;
use reqsign_core::Context;
use reqsign_core::OsEnv;
@@ -559,6 +561,66 @@ impl S3Builder {
endpoint
}
+ fn is_aws_endpoint(config: &S3Config, region: &str) -> bool {
+ let Some(endpoint) = &config.endpoint else {
+ return true;
+ };
+
+ let endpoint = if endpoint.starts_with("http") {
+ endpoint.to_string()
+ } else {
+ format!("https://{endpoint}")
+ };
+ let Ok(endpoint) = Url::parse(&endpoint) else {
+ return false;
+ };
+ if endpoint.path() != "/"
+ || endpoint.query().is_some()
+ || endpoint.fragment().is_some()
+ || endpoint.port().is_some()
+ || !endpoint.username().is_empty()
+ || endpoint.password().is_some()
+ {
+ return false;
+ }
+
+ let Some(host) = endpoint.host_str() else {
+ return false;
+ };
+ let host = host
+ .strip_prefix(&format!("{}.", config.bucket))
+ .unwrap_or(host);
+ let standard_suffix = format!(".{region}.amazonaws.com");
+ let china_suffix = format!(".{region}.amazonaws.com.cn");
+
+ host == "s3.amazonaws.com"
+ || host == format!("s3.{region}.amazonaws.com")
+ || host == format!("s3.{region}.amazonaws.com.cn")
+ || (host.starts_with("s3express-")
+ && (host.ends_with(&standard_suffix) || host.ends_with(&china_suffix)))
+ }
+
+ fn resolve_s3_express_config(
+ config: &S3Config,
+ region: &str,
+ ) -> Result<Option<S3ExpressSessionConfig>> {
+ if !config.bucket.ends_with("--x-s3") || !Self::is_aws_endpoint(config, region) {
+ return Ok(None);
+ }
+
+ S3ExpressSessionConfig::from_bucket(&config.bucket, region)
+ .map(Some)
+ .map_err(|err| {
+ Error::new(
+ ErrorKind::ConfigInvalid,
+ "invalid AWS S3 directory bucket configuration",
+ )
+ .with_operation("Builder::build")
+ .with_context("service", S3_SCHEME)
+ .set_source(err)
+ })
+ }
+
/// Deprecated: S3 delete batch capability is enabled by default.
#[deprecated(
since = "0.57.0",
@@ -852,8 +914,13 @@ impl Builder for S3Builder {
}
}
+ let s3_express_config = Self::resolve_s3_express_config(&config, ®ion)?;
+
// Building endpoint.
- let endpoint = Self::build_endpoint(&config, ®ion);
+ let endpoint = match &s3_express_config {
+ Some(config) => config.endpoint().to_string(),
+ None => Self::build_endpoint(&config, ®ion),
+ };
debug!("backend use endpoint: {endpoint}");
// The base signer context only carries local config readers. HTTP
@@ -928,11 +995,27 @@ impl Builder for S3Builder {
provider
};
- // Create request signer for S3
- let request_signer = AwsV4Signer::new("s3", ®ion);
-
- // Create the signer
- let signer = Signer::new(ctx, provider, request_signer);
+ let (signer, session_signer) = if s3_express_config.is_some() {
+ let provider = Arc::new(provider);
+ let iam_signer = Signer::new(
+ ctx.clone(),
+ provider.clone(),
+ AwsV4Signer::new("s3express", ®ion).with_standard_session_token(),
+ );
+ let session_provider =
+ S3ExpressSessionProvider::new(bucket, provider).with_region(®ion);
+ let session_signer = Signer::new(
+ ctx,
+ session_provider,
+ AwsV4Signer::new("s3express", ®ion),
+ );
+ (iam_signer, Some(session_signer))
+ } else {
+ (
+ Signer::new(ctx, provider, AwsV4Signer::new("s3", ®ion)),
+ None,
+ )
+ };
Ok(S3Backend {
core: Arc::new(S3Core {
@@ -1045,6 +1128,7 @@ impl Builder for S3Builder {
disable_list_objects_v2: config.disable_list_objects_v2,
enable_request_payer: config.enable_request_payer,
signer,
+ session_signer,
checksum_algorithm,
default_acl: config.default_acl,
}),
@@ -1299,6 +1383,53 @@ mod tests {
}
}
+ #[test]
+ fn test_resolve_s3_express_config() {
+ let directory_bucket = "example--usw2-az1--x-s3";
+
+ for endpoint in [
+ None,
+ Some("s3.amazonaws.com"),
+ Some("https://s3.us-west-2.amazonaws.com"),
+ Some("https://example--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com"),
+ ] {
+ let mut builder = S3Builder::default().bucket(directory_bucket);
+ if let Some(endpoint) = endpoint {
+ builder = builder.endpoint(endpoint);
+ }
+
+ let config = S3Builder::resolve_s3_express_config(&builder.config, "us-west-2")
+ .expect("valid directory bucket must resolve")
+ .expect("AWS directory bucket must enable session authentication");
+ assert_eq!(
+ config.endpoint(),
+ "https://example--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com"
+ );
+ }
+ }
+
+ #[test]
+ fn test_s3_express_classification_does_not_change_compatible_services() {
+ let builder = S3Builder::default()
+ .bucket("example--usw2-az1--x-s3")
+ .endpoint("https://s3.example.com");
+
+ assert!(
+ S3Builder::resolve_s3_express_config(&builder.config, "us-west-2")
+ .expect("custom endpoint classification must succeed")
+ .is_none()
+ );
+ }
+
+ #[test]
+ fn test_invalid_aws_directory_bucket_configuration_is_rejected() {
+ let builder = S3Builder::default().bucket("example--usw2-az1--x-s3");
+
+ let err = S3Builder::resolve_s3_express_config(&builder.config, "us-east-1")
+ .expect_err("zone and region mismatch must be rejected");
+ assert_eq!(err.kind(), ErrorKind::ConfigInvalid);
+ }
+
#[tokio::test]
async fn test_detect_region() {
let cases = vec![
diff --git a/core/services/s3/src/core.rs b/core/services/s3/src/core.rs
index a2278a19b..7d73b0147 100644
--- a/core/services/s3/src/core.rs
+++ b/core/services/s3/src/core.rs
@@ -105,9 +105,28 @@ pub struct S3Core {
pub default_acl: Option<String>,
pub signer: Signer<Credential>,
+ pub session_signer: Option<Signer<Credential>>,
pub checksum_algorithm: Option<ChecksumAlgorithm>,
}
+fn uses_s3_express_session(operation: Option<&ServiceOperation>) -> bool {
+ matches!(
+ operation.map(|operation| operation.0),
+ Some(
+ "HeadObject"
+ | "GetObject"
+ | "PutObject"
+ | "DeleteObject"
+ | "DeleteObjects"
+ | "ListObjectsV2"
+ | "CreateMultipartUpload"
+ | "UploadPart"
+ | "CompleteMultipartUpload"
+ | "AbortMultipartUpload"
+ )
+ )
+}
+
pub(crate) struct S3UploadPartCopyRequest<'a> {
pub(crate) from: &'a str,
pub(crate) to: &'a str,
@@ -136,7 +155,24 @@ impl Debug for S3Core {
}
impl S3Core {
- fn signer(&self, ctx: &OperationContext) -> Signer<Credential> {
+ fn signer(
+ &self,
+ ctx: &OperationContext,
+ operation: Option<&ServiceOperation>,
+ ) -> Signer<Credential> {
+ let signer = match (&self.session_signer, uses_s3_express_session(operation)) {
+ (Some(signer), true) => signer,
+ _ => &self.signer,
+ };
+ signer.clone().with_context(
+ Context::new()
+ .with_file_read(reqsign_file_read_tokio::TokioFileRead)
+ .with_http_send(ctx.http_transport().clone())
+ .with_env(reqsign_core::OsEnv),
+ )
+ }
+
+ fn iam_signer(&self, ctx: &OperationContext) -> Signer<Credential> {
self.signer.clone().with_context(
Context::new()
.with_file_read(reqsign_file_read_tokio::TokioFileRead)
@@ -158,7 +194,7 @@ impl S3Core {
// Sign the request with presigned URL
let (mut parts, body) = req.into_parts();
- self.signer(ctx)
+ self.iam_signer(ctx)
.sign(&mut parts, Some(duration))
.await
.map_err(|e| new_request_sign_error(e.into()))?;
@@ -185,7 +221,7 @@ impl S3Core {
let (mut parts, body) = req.into_parts();
- self.signer(ctx)
+ self.signer(ctx, parts.extensions.get::<ServiceOperation>())
.sign(&mut parts, None)
.await
.map_err(|e| new_request_sign_error(e.into()))?;
@@ -214,7 +250,7 @@ impl S3Core {
let (mut parts, body) = req.into_parts();
- self.signer(ctx)
+ self.signer(ctx, parts.extensions.get::<ServiceOperation>())
.sign(&mut parts, None)
.await
.map_err(|e| new_request_sign_error(e.into()))?;
@@ -1596,6 +1632,41 @@ mod tests {
use super::*;
+ #[test]
+ fn test_s3_express_session_operation_routing() {
+ for operation in [
+ "HeadObject",
+ "GetObject",
+ "PutObject",
+ "DeleteObject",
+ "DeleteObjects",
+ "ListObjectsV2",
+ "CreateMultipartUpload",
+ "UploadPart",
+ "CompleteMultipartUpload",
+ "AbortMultipartUpload",
+ ] {
+ assert!(
+ uses_s3_express_session(Some(&ServiceOperation(operation))),
+ "{operation} must use S3 Express session credentials"
+ );
+ }
+
+ for operation in [
+ "CopyObject",
+ "UploadPartCopy",
+ "HeadBucket",
+ "CreateSession",
+ "ListObjects",
+ ] {
+ assert!(
+ !uses_s3_express_session(Some(&ServiceOperation(operation))),
+ "{operation} must use IAM credentials"
+ );
+ }
+ assert!(!uses_s3_express_session(None));
+ }
+
/// This example is from https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html#API_CreateMultipartUpload_Examples
#[test]
fn test_deserialize_initiate_multipart_upload_result() {
diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md
index 66dfe5ab0..31985f634 100644
--- a/core/services/s3/src/docs.md
+++ b/core/services/s3/src/docs.md
@@ -21,6 +21,30 @@ Use [`crate::S3Config`] for serializable configuration and this builder's
methods for direct construction. The field and method documentation defines
accepted values, defaults, and environment interaction.
+## S3 Express One Zone
+
+OpenDAL recognizes valid AWS directory bucket names on AWS endpoints and uses
+the bucket's zonal endpoint automatically. It creates and refreshes `ReadWrite`
+S3 Express sessions through the configured AWS credential provider. No extra
+configuration is required:
+
+```rust,no_run
+use opendal_core::Operator;
+use opendal_core::Result;
+use opendal_service_s3::S3;
+
+fn build_operator() -> Result<Operator> {
+ Operator::new(
+ S3::default()
+ .bucket("amzn-s3-demo-bucket--usw2-az1--x-s3")
+ .region("us-west-2"),
+ )
+}
+```
+
+This automatic behavior applies only to AWS endpoints. S3-compatible services
+retain the existing IAM authentication behavior.
+
## Temporary security credentials
OpenDAL now provides support for S3 temporary security credentials in IAM.