You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/08/02 16:40:09 UTC
(opendal) 01/01: feat(services/s3): add R2 and MinIO presets
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/s3-provider-presets
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit dfa837e3b4b9ae26f6526eaebf2675496f88d248
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Aug 3 00:39:51 2026 +0800
feat(services/s3): add R2 and MinIO presets
---
.github/scripts/test_behavior/plan.py | 17 +-
.github/scripts/test_behavior/test_plan.py | 30 +++
.../services/{s3 => minio}/0_minio_s3/action.yml | 13 +-
.../minio_s3_with_anonymous/action.yml | 12 +-
.../r2/disabled_action.yml => r2/r2/action.yml} | 15 +-
bindings/dart/rust/Cargo.toml | 2 +
bindings/dotnet/Cargo.toml | 2 +
bindings/haskell/Cargo.toml | 2 +
bindings/java/Cargo.toml | 2 +
.../java/org/apache/opendal/ServiceConfig.java | 166 ++++++++++++
bindings/lua/Cargo.toml | 2 +
bindings/nodejs/Cargo.toml | 4 +
bindings/ocaml/Cargo.toml | 2 +
bindings/php/Cargo.toml | 2 +
bindings/python/Cargo.toml | 4 +
bindings/python/python/opendal/config.py | 50 ++++
bindings/python/src/services.rs | 8 +
bindings/ruby/Cargo.toml | 2 +
core/Cargo.toml | 2 +
.../core/src/docs/rfcs/7948_s3_provider_presets.md | 39 ++-
core/fuzz/Cargo.toml | 2 +
core/services/s3/README.md | 25 +-
core/services/s3/src/backend.rs | 113 ++++----
core/services/s3/src/compatible_services.md | 46 +---
core/services/s3/src/docs.md | 2 +-
core/services/s3/src/lib.rs | 66 ++++-
core/services/s3/src/minio.md | 51 ++++
core/services/s3/src/minio.rs | 257 ++++++++++++++++++
core/services/s3/src/minio_config.rs | 161 +++++++++++
core/services/s3/src/preset.rs | 180 +++++++++++++
core/services/s3/src/r2.md | 53 ++++
core/services/s3/src/r2.rs | 298 +++++++++++++++++++++
core/services/s3/src/r2_config.rs | 158 +++++++++++
core/src/lib.rs | 51 +++-
dev/src/generate/docs.rs | 54 +++-
dev/src/generate/parser.rs | 112 ++++++--
dev/src/generate/python.rs | 8 +-
justfile | 2 +-
website/data/services.json | 205 ++++++++++++++
39 files changed, 2060 insertions(+), 160 deletions(-)
diff --git a/.github/scripts/test_behavior/plan.py b/.github/scripts/test_behavior/plan.py
index 3481be704..bfa71585d 100755
--- a/.github/scripts/test_behavior/plan.py
+++ b/.github/scripts/test_behavior/plan.py
@@ -35,6 +35,10 @@ LANGUAGE_BINDING = ["java", "python", "ruby", "nodejs", "go", "c", "cpp", "dotne
INTEGRATIONS = ["object_store"]
+SHARED_SERVICE_SCHEMES = {
+ "s3": {"s3", "minio", "r2"},
+}
+
def provided_cases() -> list[dict[str, str]]:
root_dir = f"{GITHUB_DIR}/services"
@@ -116,9 +120,10 @@ def calculate_hint(changed_files: list[str]) -> Hint:
for integration in INTEGRATIONS:
setattr(hint, f"integration_{integration}", True)
- hint.services.add(service)
- hint.services.add(service.replace("-", "_"))
- hint.services.add(service.replace("_", "-"))
+ for scheme in SHARED_SERVICE_SCHEMES.get(service, {service}):
+ hint.services.add(scheme)
+ hint.services.add(scheme.replace("-", "_"))
+ hint.services.add(scheme.replace("_", "-"))
for p in changed_files:
# workflow behavior tests affected
@@ -290,6 +295,10 @@ def generate_language_binding_cases(
# Remove invalid cases for go.
if language == "go":
cases = [v for v in cases if v["service"] not in [
+ # Provider presets share the Rust S3 crate and don't have separate
+ # opendal-go-services packages.
+ "minio",
+ "r2",
# opendal-go-services doesn't provide TOS yet.
"tos",
]]
@@ -309,6 +318,8 @@ def generate_language_binding_cases(
"memory",
"obs",
"oss",
+ "minio",
+ "r2",
"s3",
"webdav",
"webhdfs",
diff --git a/.github/scripts/test_behavior/test_plan.py b/.github/scripts/test_behavior/test_plan.py
index 76abd6511..82cac7618 100644
--- a/.github/scripts/test_behavior/test_plan.py
+++ b/.github/scripts/test_behavior/test_plan.py
@@ -53,6 +53,36 @@ class BehaviorTestPlan(unittest.TestCase):
self.assertTrue("hdfs_native" in cases)
self.assertFalse("fs" in cases)
+ @patch.dict("os.environ", {"GITHUB_HAS_SECRETS": "true"})
+ def test_s3_crate_schedules_provider_cases(self):
+ result = plan(["core/services/s3/src/lib.rs"])
+ core_cases = {
+ (case["service"], case["feature"])
+ for target in result["core"]
+ for case in target["cases"]
+ }
+ self.assertEqual(
+ core_cases,
+ {
+ ("s3", "services-s3"),
+ ("minio", "services-minio"),
+ ("r2", "services-r2"),
+ },
+ )
+
+ go_services = {
+ case["service"]
+ for target in result["binding_go"]
+ for case in target["cases"]
+ }
+ ruby_services = {
+ case["service"]
+ for target in result["binding_ruby"]
+ for case in target["cases"]
+ }
+ self.assertTrue({"minio", "r2"}.isdisjoint(go_services))
+ self.assertTrue({"minio", "r2"} <= ruby_services)
+
def test_binding_java(self):
result = plan(["bindings/java/pom.xml"])
self.assertFalse(result["components"]["core"])
diff --git a/.github/services/s3/0_minio_s3/action.yml b/.github/services/minio/0_minio_s3/action.yml
similarity index 85%
rename from .github/services/s3/0_minio_s3/action.yml
rename to .github/services/minio/0_minio_s3/action.yml
index f74a07701..6b3c8204b 100644
--- a/.github/services/s3/0_minio_s3/action.yml
+++ b/.github/services/minio/0_minio_s3/action.yml
@@ -15,8 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-name: minio_s3
-description: 'Behavior test for Minio S3.'
+name: minio
+description: 'Behavior test for the MinIO provider preset.'
runs:
using: "composite"
@@ -38,10 +38,9 @@ runs:
shell: bash
run: |
cat << EOF >> $GITHUB_ENV
- OPENDAL_S3_BUCKET=test
- OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000
- OPENDAL_S3_ACCESS_KEY_ID=minioadmin
- OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin
- OPENDAL_S3_REGION=us-east-1
+ OPENDAL_MINIO_BUCKET=test
+ OPENDAL_MINIO_ENDPOINT=http://127.0.0.1:9000
+ OPENDAL_MINIO_ACCESS_KEY_ID=minioadmin
+ OPENDAL_MINIO_SECRET_ACCESS_KEY=minioadmin
OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,write_can_append=false,copy_with_if_not_exists=false,copy_with_if_match=false
EOF
diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml b/.github/services/minio/minio_s3_with_anonymous/action.yml
similarity index 86%
rename from .github/services/s3/minio_s3_with_anonymous/action.yml
rename to .github/services/minio/minio_s3_with_anonymous/action.yml
index 5bd46d42f..d6cf441f1 100644
--- a/.github/services/s3/minio_s3_with_anonymous/action.yml
+++ b/.github/services/minio/minio_s3_with_anonymous/action.yml
@@ -15,8 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-name: minio_s3_with_anonymous
-description: 'Behavior test for Minio S3 with anonymous access.'
+name: minio_with_anonymous
+description: 'Behavior test for the MinIO provider preset with anonymous access.'
runs:
using: "composite"
@@ -44,10 +44,8 @@ runs:
shell: bash
run: |
cat << EOF >> $GITHUB_ENV
- OPENDAL_S3_BUCKET=test
- OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000
- OPENDAL_S3_REGION=us-east-1
- OPENDAL_S3_ALLOW_ANONYMOUS=on
- OPENDAL_S3_DISABLE_EC2_METADATA=on
+ OPENDAL_MINIO_BUCKET=test
+ OPENDAL_MINIO_ENDPOINT=http://127.0.0.1:9000
+ OPENDAL_MINIO_SKIP_SIGNATURE=true
OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,write_can_append=false,copy_with_if_not_exists=false,copy_with_if_match=false
EOF
diff --git a/.github/services/s3/r2/disabled_action.yml b/.github/services/r2/r2/action.yml
similarity index 71%
rename from .github/services/s3/r2/disabled_action.yml
rename to .github/services/r2/r2/action.yml
index f9952414e..f4f912318 100644
--- a/.github/services/s3/r2/disabled_action.yml
+++ b/.github/services/r2/r2/action.yml
@@ -16,7 +16,7 @@
# under the License.
name: r2
-description: "Behavior test for Cloudflare R2. This service is sponsored by @Xuanwo."
+description: "Behavior test for the Cloudflare R2 provider preset. This service is sponsored by @Xuanwo."
runs:
using: "composite"
@@ -26,17 +26,14 @@ runs:
with:
export-env: true
env:
- OPENDAL_S3_BUCKET: op://services/r2/bucket
- OPENDAL_S3_ENDPOINT: op://services/r2/endpoint
- OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
- OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
+ OPENDAL_R2_BUCKET: op://services/r2/bucket
+ OPENDAL_R2_ENDPOINT: op://services/r2/endpoint
+ OPENDAL_R2_ACCESS_KEY_ID: op://services/r2/access_key_id
+ OPENDAL_R2_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
- # R2 has a lower delete batch limit and doesn't support stat override response queries.
- # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
- - name: Add extra settings
+ - name: Add test overrides
shell: bash
run: |
cat << EOF >> $GITHUB_ENV
- OPENDAL_S3_REGION=auto
OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,write_can_append=false,delete_max_size=700,stat_with_override_cache_control=false,stat_with_override_content_disposition=false,stat_with_override_content_type=false
EOF
diff --git a/bindings/dart/rust/Cargo.toml b/bindings/dart/rust/Cargo.toml
index 84c15d573..60e89cb43 100644
--- a/bindings/dart/rust/Cargo.toml
+++ b/bindings/dart/rust/Cargo.toml
@@ -36,8 +36,10 @@ opendal = { path = "../../../core", features = [
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-webdav",
"services-webhdfs",
diff --git a/bindings/dotnet/Cargo.toml b/bindings/dotnet/Cargo.toml
index 56ab13bb1..41070aeb9 100644
--- a/bindings/dotnet/Cargo.toml
+++ b/bindings/dotnet/Cargo.toml
@@ -55,8 +55,10 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-tos",
"services-webdav",
diff --git a/bindings/haskell/Cargo.toml b/bindings/haskell/Cargo.toml
index 1dacd2602..96caaf26a 100644
--- a/bindings/haskell/Cargo.toml
+++ b/bindings/haskell/Cargo.toml
@@ -45,8 +45,10 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-webdav",
"services-webhdfs",
diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml
index f75d42e76..a246c17df 100644
--- a/bindings/java/Cargo.toml
+++ b/bindings/java/Cargo.toml
@@ -54,8 +54,10 @@ opendal = { version = ">=0", path = "../../core", default-features = false, feat
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-tos",
"services-webdav",
diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
index d545a3ee5..560dd7c15 100644
--- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
+++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
@@ -2023,6 +2023,88 @@ public interface ServiceConfig {
}
}
+ /**
+ * Configuration for service minio.
+ */
+ @Builder
+ @Data
+ @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+ class Minio implements ServiceConfig {
+ /**
+ * <p>Access key ID.</p>
+ * <p>Set this field together with <code>secret_access_key</code>.</p>
+ */
+ public final String accessKeyId;
+ /**
+ * <p>Bucket name.</p>
+ * <p>This field is required.</p>
+ */
+ public final @NonNull String bucket;
+ /**
+ * <p>MinIO endpoint.</p>
+ * <p>This field is required because MinIO deployments do not share a
+ * universal endpoint.</p>
+ */
+ public final @NonNull String endpoint;
+ /**
+ * <p>Signing region.</p>
+ * <p>The default is <code>auto</code>. Set this field when the deployment requires a
+ * configured region.</p>
+ */
+ public final String region;
+ /**
+ * <p>Root within the bucket.</p>
+ * <p>All operations happen under this root. The default is <code>/</code>.</p>
+ */
+ public final String root;
+ /**
+ * <p>Secret access key.</p>
+ * <p>Set this field together with <code>access_key_id</code>.</p>
+ */
+ public final String secretAccessKey;
+ /**
+ * <p>Session token for temporary credentials.</p>
+ * <p>This field requires <code>access_key_id</code> and <code>secret_access_key</code>.</p>
+ */
+ public final String sessionToken;
+ /**
+ * <p>Send requests without signing them.</p>
+ * <p>This option cannot be combined with direct credentials.</p>
+ */
+ public final Boolean skipSignature;
+
+ @Override
+ public String scheme() {
+ return "minio";
+ }
+
+ @Override
+ public Map<String, String> configMap() {
+ final HashMap<String, String> map = new HashMap<>();
+ if (accessKeyId != null) {
+ map.put("access_key_id", accessKeyId);
+ }
+ map.put("bucket", bucket);
+ map.put("endpoint", endpoint);
+ if (region != null) {
+ map.put("region", region);
+ }
+ if (root != null) {
+ map.put("root", root);
+ }
+ if (secretAccessKey != null) {
+ map.put("secret_access_key", secretAccessKey);
+ }
+ if (sessionToken != null) {
+ map.put("session_token", sessionToken);
+ }
+ if (skipSignature != null) {
+ map.put("skip_signature", String.valueOf(skipSignature));
+ }
+ return map;
+ }
+ }
+
/**
* Configuration for service moka.
*/
@@ -2752,6 +2834,90 @@ public interface ServiceConfig {
}
}
+ /**
+ * Configuration for service r2.
+ */
+ @Builder
+ @Data
+ @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+ class R2 implements ServiceConfig {
+ /**
+ * <p>Access key ID.</p>
+ * <p>Set this field together with <code>secret_access_key</code>.</p>
+ */
+ public final String accessKeyId;
+ /**
+ * <p>Cloudflare account ID used to derive the R2 endpoint.</p>
+ * <p>Set exactly one of <code>account_id</code> and <code>endpoint</code>.</p>
+ */
+ public final String accountId;
+ /**
+ * <p>Bucket name.</p>
+ * <p>This field is required.</p>
+ */
+ public final @NonNull String bucket;
+ /**
+ * <p>Explicit R2-compatible endpoint.</p>
+ * <p>Use this field for a proxy, gateway, or test server. Set exactly one of
+ * <code>endpoint</code> and <code>account_id</code>.</p>
+ */
+ public final String endpoint;
+ /**
+ * <p>R2 jurisdiction.</p>
+ * <p>Supported values are <code>eu</code> and <code>fedramp</code>. This field requires
+ * <code>account_id</code> and cannot be used with <code>endpoint</code>.</p>
+ */
+ public final String jurisdiction;
+ /**
+ * <p>Root within the bucket.</p>
+ * <p>All operations happen under this root. The default is <code>/</code>.</p>
+ */
+ public final String root;
+ /**
+ * <p>Secret access key.</p>
+ * <p>Set this field together with <code>access_key_id</code>.</p>
+ */
+ public final String secretAccessKey;
+ /**
+ * <p>Session token for temporary credentials.</p>
+ * <p>This field requires <code>access_key_id</code> and <code>secret_access_key</code>.</p>
+ */
+ public final String sessionToken;
+
+ @Override
+ public String scheme() {
+ return "r2";
+ }
+
+ @Override
+ public Map<String, String> configMap() {
+ final HashMap<String, String> map = new HashMap<>();
+ if (accessKeyId != null) {
+ map.put("access_key_id", accessKeyId);
+ }
+ if (accountId != null) {
+ map.put("account_id", accountId);
+ }
+ map.put("bucket", bucket);
+ if (endpoint != null) {
+ map.put("endpoint", endpoint);
+ }
+ if (jurisdiction != null) {
+ map.put("jurisdiction", jurisdiction);
+ }
+ if (root != null) {
+ map.put("root", root);
+ }
+ if (secretAccessKey != null) {
+ map.put("secret_access_key", secretAccessKey);
+ }
+ if (sessionToken != null) {
+ map.put("session_token", sessionToken);
+ }
+ return map;
+ }
+ }
+
/**
* Configuration for service redb.
*/
diff --git a/bindings/lua/Cargo.toml b/bindings/lua/Cargo.toml
index 761445ae6..5b3289c73 100644
--- a/bindings/lua/Cargo.toml
+++ b/bindings/lua/Cargo.toml
@@ -51,8 +51,10 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-webdav",
"services-webhdfs",
diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml
index e39be989b..a9847adfe 100644
--- a/bindings/nodejs/Cargo.toml
+++ b/bindings/nodejs/Cargo.toml
@@ -37,8 +37,10 @@ default = [
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-webdav",
"services-webhdfs",
@@ -102,8 +104,10 @@ services-ghac = ["opendal/services-ghac"]
services-http = ["opendal/services-http"]
services-ipmfs = ["opendal/services-ipmfs"]
services-memory = ["opendal/services-memory"]
+services-minio = ["opendal/services-minio"]
services-obs = ["opendal/services-obs"]
services-oss = ["opendal/services-oss"]
+services-r2 = ["opendal/services-r2"]
services-s3 = ["opendal/services-s3"]
services-webdav = ["opendal/services-webdav"]
services-webhdfs = ["opendal/services-webhdfs"]
diff --git a/bindings/ocaml/Cargo.toml b/bindings/ocaml/Cargo.toml
index 74e501446..1c976a265 100644
--- a/bindings/ocaml/Cargo.toml
+++ b/bindings/ocaml/Cargo.toml
@@ -45,8 +45,10 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-webdav",
"services-webhdfs",
diff --git a/bindings/php/Cargo.toml b/bindings/php/Cargo.toml
index 8e6ce7ac2..5d2cc404b 100644
--- a/bindings/php/Cargo.toml
+++ b/bindings/php/Cargo.toml
@@ -44,8 +44,10 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-webdav",
"services-webhdfs",
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index a30fc777d..9fc44e274 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -39,8 +39,10 @@ default = [
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-webdav",
"services-webhdfs",
@@ -141,6 +143,7 @@ services-lakefs = [
services-memcached = ["opendal/services-memcached"]
services-memory = ["opendal/services-memory"]
services-mini-moka = ["opendal/services-mini-moka"]
+services-minio = ["opendal/services-minio"]
services-moka = ["opendal/services-moka"]
services-mongodb = ["opendal/services-mongodb"]
services-monoiofs = [
@@ -158,6 +161,7 @@ services-pcloud = [
] # FIXME EXCLUDED: Needs tests/ maintenance
services-persy = ["opendal/services-persy"]
services-postgresql = ["opendal/services-postgresql"]
+services-r2 = ["opendal/services-r2"]
services-redb = ["opendal/services-redb"]
services-redis = ["opendal/services-redis"]
services-rocksdb = [
diff --git a/bindings/python/python/opendal/config.py b/bindings/python/python/opendal/config.py
index 54a4b7a63..9835d7bf5 100644
--- a/bindings/python/python/opendal/config.py
+++ b/bindings/python/python/opendal/config.py
@@ -492,6 +492,29 @@ class MiniMokaConfig(TypedDict):
"""Sets the time to live of the cache. Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live)"""
+class MinioConfig(TypedDict):
+ """Configuration for the `minio` service."""
+
+ scheme: Required[Literal["minio"]]
+ """The service scheme; fixed to `"minio"`."""
+ access_key_id: NotRequired[str]
+ """Access key ID. Set this field together with `secret_access_key`."""
+ bucket: Required[str]
+ """Bucket name. This field is required."""
+ endpoint: Required[str]
+ """MinIO endpoint. This field is required because MinIO deployments do not share a universal endpoint."""
+ region: NotRequired[str]
+ """Signing region. The default is `auto`. Set this field when the deployment requires a configured region."""
+ root: NotRequired[str | os.PathLike[str]]
+ """Root within the bucket. All operations happen under this root. The default is `/`."""
+ secret_access_key: NotRequired[str]
+ """Secret access key. Set this field together with `access_key_id`."""
+ session_token: NotRequired[str]
+ """Session token for temporary credentials. This field requires `access_key_id` and `secret_access_key`."""
+ skip_signature: NotRequired[bool]
+ """Send requests without signing them. This option cannot be combined with direct credentials."""
+
+
class MokaConfig(TypedDict):
"""Configuration for the `moka` service."""
@@ -664,6 +687,29 @@ class PostgresqlConfig(TypedDict):
"""the value field of postgresql"""
+class R2Config(TypedDict):
+ """Configuration for the `r2` service."""
+
+ scheme: Required[Literal["r2"]]
+ """The service scheme; fixed to `"r2"`."""
+ access_key_id: NotRequired[str]
+ """Access key ID. Set this field together with `secret_access_key`."""
+ account_id: NotRequired[str]
+ """Cloudflare account ID used to derive the R2 endpoint. Set exactly one of `account_id` and `endpoint`."""
+ bucket: Required[str]
+ """Bucket name. This field is required."""
+ endpoint: NotRequired[str]
+ """Explicit R2-compatible endpoint. Use this field for a proxy, gateway, or test server. Set exactly one of `endpoint` and `account_id`."""
+ jurisdiction: NotRequired[str]
+ """R2 jurisdiction. Supported values are `eu` and `fedramp`. This field requires `account_id` and cannot be used with `endpoint`."""
+ root: NotRequired[str | os.PathLike[str]]
+ """Root within the bucket. All operations happen under this root. The default is `/`."""
+ secret_access_key: NotRequired[str]
+ """Secret access key. Set this field together with `access_key_id`."""
+ session_token: NotRequired[str]
+ """Session token for temporary credentials. This field requires `access_key_id` and `secret_access_key`."""
+
+
class RedbConfig(TypedDict):
"""Configuration for the `redb` service."""
@@ -1010,6 +1056,7 @@ ServiceConfig = (
| MemcachedConfig
| MemoryConfig
| MiniMokaConfig
+ | MinioConfig
| MokaConfig
| MongodbConfig
| MysqlConfig
@@ -1018,6 +1065,7 @@ ServiceConfig = (
| OssConfig
| PersyConfig
| PostgresqlConfig
+ | R2Config
| RedbConfig
| RedisConfig
| S3Config
@@ -1064,6 +1112,7 @@ __all__ = [
"MemcachedConfig",
"MemoryConfig",
"MiniMokaConfig",
+ "MinioConfig",
"MokaConfig",
"MongodbConfig",
"MysqlConfig",
@@ -1072,6 +1121,7 @@ __all__ = [
"OssConfig",
"PersyConfig",
"PostgresqlConfig",
+ "R2Config",
"RedbConfig",
"RedisConfig",
"S3Config",
diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs
index a26e498dd..413106920 100644
--- a/bindings/python/src/services.rs
+++ b/bindings/python/src/services.rs
@@ -81,6 +81,8 @@ pub enum Scheme {
Memory,
#[cfg(feature = "services-mini-moka")]
MiniMoka,
+ #[cfg(feature = "services-minio")]
+ Minio,
#[cfg(feature = "services-moka")]
Moka,
#[cfg(feature = "services-mongodb")]
@@ -97,6 +99,8 @@ pub enum Scheme {
Persy,
#[cfg(feature = "services-postgresql")]
Postgresql,
+ #[cfg(feature = "services-r2")]
+ R2,
#[cfg(feature = "services-redb")]
Redb,
#[cfg(feature = "services-redis")]
@@ -221,6 +225,8 @@ impl_enum_to_str!(
Memory => "memory",
#[cfg(feature = "services-mini-moka")]
MiniMoka => "mini-moka",
+ #[cfg(feature = "services-minio")]
+ Minio => "minio",
#[cfg(feature = "services-moka")]
Moka => "moka",
#[cfg(feature = "services-mongodb")]
@@ -237,6 +243,8 @@ impl_enum_to_str!(
Persy => "persy",
#[cfg(feature = "services-postgresql")]
Postgresql => "postgresql",
+ #[cfg(feature = "services-r2")]
+ R2 => "r2",
#[cfg(feature = "services-redb")]
Redb => "redb",
#[cfg(feature = "services-redis")]
diff --git a/bindings/ruby/Cargo.toml b/bindings/ruby/Cargo.toml
index 21dc44f21..f8a48993e 100644
--- a/bindings/ruby/Cargo.toml
+++ b/bindings/ruby/Cargo.toml
@@ -52,8 +52,10 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-http",
"services-ipmfs",
"services-memory",
+ "services-minio",
"services-obs",
"services-oss",
+ "services-r2",
"services-s3",
"services-webdav",
"services-webhdfs",
diff --git a/core/Cargo.toml b/core/Cargo.toml
index d7d686e71..5443c7450 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -186,6 +186,7 @@ services-memcached = ["dep:opendal-service-memcached"]
# Deprecated: memory service is always enabled.
services-memory = ["opendal-core/services-memory"]
services-mini-moka = ["dep:opendal-service-mini-moka"]
+services-minio = ["dep:opendal-service-s3"]
services-moka = ["dep:opendal-service-moka"]
services-mongodb = ["dep:opendal-service-mongodb"]
services-monoiofs = ["dep:opendal-service-monoiofs"]
@@ -197,6 +198,7 @@ services-oss = ["dep:opendal-service-oss"]
services-pcloud = ["dep:opendal-service-pcloud"]
services-persy = ["dep:opendal-service-persy"]
services-postgresql = ["dep:opendal-service-postgresql"]
+services-r2 = ["dep:opendal-service-s3"]
services-redb = ["dep:opendal-service-redb"]
services-redis = ["dep:opendal-service-redis", "opendal-service-redis?/rustls"]
services-redis-native-tls = [
diff --git a/core/core/src/docs/rfcs/7948_s3_provider_presets.md b/core/core/src/docs/rfcs/7948_s3_provider_presets.md
index e64f69291..b07f27e90 100644
--- a/core/core/src/docs/rfcs/7948_s3_provider_presets.md
+++ b/core/core/src/docs/rfcs/7948_s3_provider_presets.md
@@ -1,7 +1,7 @@
- Proposal Name: `s3_provider_presets`
- Start Date: 2026-07-24
- RFC PR: [apache/opendal#7948](https://github.com/apache/opendal/pull/7948)
-- Tracking Issue: [apache/opendal#0000](https://github.com/apache/opendal/issues/0000)
+- Tracking Issue: [apache/opendal#8003](https://github.com/apache/opendal/issues/8003)
# Summary
@@ -9,6 +9,10 @@ Add first-class `r2` and `minio` services to `opendal-service-s3`. Each service
has its own URI scheme, config type, and builder. Its config exposes only the
connection, location, and authentication fields supported by that provider.
+The `opendal` facade exposes `services-r2` and `services-minio` features while
+all providers continue to share the `opendal-service-s3` implementation crate.
+The existing `services-s3` feature continues to enable only generic S3.
+
The provider builder validates those fields, converts them into an internal
`S3Config`, and delegates requests to the existing S3 implementation. Existing
`s3` construction and behavior remain unchanged.
@@ -87,18 +91,33 @@ the complete S3 configuration surface continue to use `s3`.
The `opendal-service-s3` crate exports `R2`, `R2Config`, `R2_SCHEME`, `Minio`,
`MinioConfig`, and `MINIO_SCHEME` alongside `S3`, `S3Config`, and `S3_SCHEME`.
-Enabling `services-s3` registers all three schemes:
+The facade features select the public builders and URI registrations:
-```rust,ignore
-registry.register::<S3>(S3_SCHEME);
-registry.register::<R2>(R2_SCHEME);
-registry.register::<Minio>(MINIO_SCHEME);
-```
+- `services-r2` exports `R2` and `R2Config` and registers `r2`.
+- `services-minio` exports `Minio` and `MinioConfig` and registers `minio`.
+- `services-s3` exports `S3` and `S3Config` and registers only `s3`.
+
+All three facade features activate the same optional `opendal-service-s3`
+dependency. Provider presets do not create additional crates. A compatible
+provider receives a facade feature only when OpenDAL gives it a dedicated
+config type, builder, and URI scheme.
Each provider implements the existing `Configurator` and `Builder` contracts.
`Operator::from_uri`, `Operator::via_iter`, and `Operator::from_config`
therefore work without changes to `OperatorRegistry`.
+## Documentation ownership
+
+The provider config types and builder documentation are the source of truth for
+R2 and MinIO setup. Generated service and binding references read those config
+types, so their field lists stay aligned with the deserialization contract.
+
+`compatible_services.md` remains the long-tail guide for using the generic
+`S3` service with providers that do not have a built-in preset, and for users
+who intentionally need the S3 escape hatch. When OpenDAL adds a preset, that
+guide replaces the provider's manual configuration recipe with a link to the
+preset instead of maintaining both narratives.
+
## Provider configuration
Provider config types are independent structs. They do not embed or flatten
@@ -204,8 +223,10 @@ Provider validation includes:
# Compatibility and migration
-This proposal adds schemes and types without changing `s3`. OpenDAL never
-reinterprets an existing S3 endpoint as a provider preset.
+This proposal adds schemes, types, and provider features without changing
+`s3`. Existing `services-s3` users retain generic S3 without implicitly
+enabling provider presets. OpenDAL never reinterprets an existing S3 endpoint
+as a provider preset.
An application can migrate by changing `s3` to `r2` or `minio`, removing
AWS-only options, and supplying the provider's required fields. Applications
diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml
index ecf4cea1c..820408caa 100644
--- a/core/fuzz/Cargo.toml
+++ b/core/fuzz/Cargo.toml
@@ -57,6 +57,7 @@ services-ipmfs = ["opendal/services-ipmfs"]
services-memcached = ["opendal/services-memcached"]
services-memory = ["opendal/services-memory"]
services-mini-moka = ["opendal/services-mini-moka"]
+services-minio = ["opendal/services-minio"]
services-moka = ["opendal/services-moka"]
services-mongodb = ["opendal/services-mongodb"]
services-mysql = ["opendal/services-mysql"]
@@ -65,6 +66,7 @@ services-onedrive = ["opendal/services-onedrive"]
services-oss = ["opendal/services-oss"]
services-persy = ["opendal/services-persy"]
services-postgresql = ["opendal/services-postgresql"]
+services-r2 = ["opendal/services-r2"]
services-redb = ["opendal/services-redb"]
services-redis = ["opendal/services-redis"]
services-rocksdb = ["opendal/services-rocksdb"]
diff --git a/core/services/s3/README.md b/core/services/s3/README.md
index 611e62a58..449ef2cd2 100644
--- a/core/services/s3/README.md
+++ b/core/services/s3/README.md
@@ -1,19 +1,26 @@
# Apache OpenDAL™ Amazon S3 Service
`opendal-service-s3` provides access to Amazon S3 and S3-compatible object storage for
-applications built with Apache OpenDAL™.
+applications built with Apache OpenDAL™. It includes provider presets for Cloudflare R2
+and MinIO.
## Use through `opendal`
-Applications should normally enable this service through the `opendal` facade with the
-`services-s3` feature:
+Applications should enable the service or provider they use through its
+matching `opendal` facade feature:
```shell
+# Generic S3
cargo add opendal --features services-s3
+# Cloudflare R2
+cargo add opendal --features services-r2
+# MinIO
+cargo add opendal --features services-minio
```
-The service is available as `opendal::services::S3`. Configure the
-service builder, then pass it to `opendal::Operator::new`.
+The matching builders are available as `opendal::services::S3`,
+`opendal::services::R2`, and `opendal::services::Minio`. Configure a builder,
+then pass it to `opendal::Operator::new`.
## Use with `opendal-core`
@@ -38,8 +45,10 @@ fn register_for_uri() {
}
```
-`register_for_uri` is only needed for scheme-driven construction through
-`Operator::from_uri` or `Operator::via_iter`.
+`register_for_uri` registers only the `s3` scheme. Use
+`register_r2_service` or `register_minio_service` for the corresponding
+provider scheme. Registration is only needed for scheme-driven construction
+through `Operator::from_uri` or `Operator::via_iter`.
Services that send HTTP requests also require an HTTP transport in
`OperationContext`. See the
@@ -48,6 +57,8 @@ Services that send HTTP requests also require an HTTP transport in
## Documentation
- [Service configuration and examples](https://opendal.apache.org/services/s3)
+- [Cloudflare R2 preset](https://opendal.apache.org/services/r2)
+- [MinIO preset](https://opendal.apache.org/services/minio)
- [Rust API documentation](https://docs.rs/opendal-service-s3)
- [Apache OpenDAL documentation](https://opendal.apache.org/docs/)
diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs
index 9d0869789..e032d3ee1 100644
--- a/core/services/s3/src/backend.rs
+++ b/core/services/s3/src/backend.rs
@@ -725,6 +725,26 @@ impl Builder for S3Builder {
type Config = S3Config;
fn build(self) -> Result<impl Service> {
+ self.build_inner(S3_SCHEME)
+ }
+}
+
+impl S3Builder {
+ pub(crate) fn from_provider_config(
+ config: S3Config,
+ credential_providers: ProvideCredentialChain<Credential>,
+ ) -> Self {
+ Self {
+ config,
+ credential_providers: Some(credential_providers),
+ }
+ }
+
+ pub(crate) fn build_with_scheme(self, scheme: &'static str) -> Result<S3Backend> {
+ self.build_inner(scheme)
+ }
+
+ fn build_inner(self, scheme: &'static str) -> Result<S3Backend> {
debug!("backend build started: {:?}", self);
let S3Builder {
@@ -746,7 +766,7 @@ impl Builder for S3Builder {
} else {
Err(
Error::new(ErrorKind::ConfigInvalid, "The bucket is misconfigured")
- .with_context("service", S3_SCHEME),
+ .with_context("service", scheme),
)
}?;
debug!("backend use bucket {}", bucket);
@@ -822,7 +842,7 @@ impl Builder for S3Builder {
"region is missing. Please find it by S3::detect_region() or set them in env.",
)
.with_operation("Builder::build")
- .with_context("service", S3_SCHEME)
+ .with_context("service", scheme)
})?
};
debug!("backend use region: {region}");
@@ -847,7 +867,9 @@ impl Builder for S3Builder {
// operation.
let ctx = Context::new().with_file_read(TokioFileRead).with_env(OsEnv);
- let mut provider = {
+ let provider = if let Some(credential_providers) = credential_providers {
+ credential_providers
+ } else {
let mut builder = DefaultCredentialProvider::builder();
if config.disable_config_load {
@@ -858,53 +880,50 @@ impl Builder for S3Builder {
builder = builder.no_imds();
}
- ProvideCredentialChain::new().push(builder.build())
- };
-
- // Insert static key if user provided.
- if let (Some(ak), Some(sk)) = (&config.access_key_id, &config.secret_access_key) {
- let static_provider = if let Some(token) = config.session_token.as_deref() {
- StaticCredentialProvider::new(ak, sk).with_session_token(token)
- } else {
- StaticCredentialProvider::new(ak, sk)
- };
- provider = provider.push_front(static_provider);
- }
+ let mut provider = ProvideCredentialChain::new().push(builder.build());
- // Insert assume role provider if user provided.
- if let Some(role_arn) = &config.role_arn {
- // The assume-role provider owns its STS signer, so give it a
- // concrete HTTP sender instead of relying on a future operation
- // context.
- let sts_ctx = ctx.clone().with_http_send(HttpTransporter::default());
- let sts_request_signer = AwsV4Signer::new("sts", ®ion);
- let sts_signer = Signer::new(sts_ctx, provider, sts_request_signer);
- let mut assume_role_provider =
- AssumeRoleCredentialProvider::new(role_arn.clone(), sts_signer)
- .with_region(region.clone())
- .with_regional_sts_endpoint();
-
- if let Some(external_id) = &config.external_id {
- assume_role_provider = assume_role_provider.with_external_id(external_id.clone());
- }
- if let Some(role_session_name) = &config.role_session_name {
- assume_role_provider =
- assume_role_provider.with_role_session_name(role_session_name.clone());
- }
- if let Some(duration_seconds) = config.assume_role_duration_seconds {
- assume_role_provider = assume_role_provider.with_duration_seconds(duration_seconds);
+ // Insert static key if user provided.
+ if let (Some(ak), Some(sk)) = (&config.access_key_id, &config.secret_access_key) {
+ let static_provider = if let Some(token) = config.session_token.as_deref() {
+ StaticCredentialProvider::new(ak, sk).with_session_token(token)
+ } else {
+ StaticCredentialProvider::new(ak, sk)
+ };
+ provider = provider.push_front(static_provider);
}
- if let Some(tags) = &config.assume_role_session_tags {
- assume_role_provider = assume_role_provider
- .with_tags(tags.iter().map(|(k, v)| (k.clone(), v.clone())).collect());
+
+ // Insert assume role provider if user provided.
+ if let Some(role_arn) = &config.role_arn {
+ // The assume-role provider owns its STS signer, so give it a
+ // concrete HTTP sender instead of relying on a future operation
+ // context.
+ let sts_ctx = ctx.clone().with_http_send(HttpTransporter::default());
+ let sts_request_signer = AwsV4Signer::new("sts", ®ion);
+ let sts_signer = Signer::new(sts_ctx, provider, sts_request_signer);
+ let mut assume_role_provider =
+ AssumeRoleCredentialProvider::new(role_arn.clone(), sts_signer)
+ .with_region(region.clone())
+ .with_regional_sts_endpoint();
+
+ if let Some(external_id) = &config.external_id {
+ assume_role_provider =
+ assume_role_provider.with_external_id(external_id.clone());
+ }
+ if let Some(role_session_name) = &config.role_session_name {
+ assume_role_provider =
+ assume_role_provider.with_role_session_name(role_session_name.clone());
+ }
+ if let Some(duration_seconds) = config.assume_role_duration_seconds {
+ assume_role_provider =
+ assume_role_provider.with_duration_seconds(duration_seconds);
+ }
+ if let Some(tags) = &config.assume_role_session_tags {
+ assume_role_provider = assume_role_provider
+ .with_tags(tags.iter().map(|(k, v)| (k.clone(), v.clone())).collect());
+ }
+ provider = ProvideCredentialChain::new().push(assume_role_provider);
}
- provider = ProvideCredentialChain::new().push(assume_role_provider);
- }
- // Replace provider if user provide their own.
- let provider = if let Some(credential_providers) = credential_providers {
- credential_providers
- } else {
provider
};
@@ -916,7 +935,7 @@ impl Builder for S3Builder {
Ok(S3Backend {
core: Arc::new(S3Core {
- info: ServiceInfo::new(S3_SCHEME, &root, bucket),
+ info: ServiceInfo::new(scheme, &root, bucket),
capability: Capability {
stat: true,
stat_with_if_match: true,
diff --git a/core/services/s3/src/compatible_services.md b/core/services/s3/src/compatible_services.md
index 328c13d57..9238372f6 100644
--- a/core/services/s3/src/compatible_services.md
+++ b/core/services/s3/src/compatible_services.md
@@ -1,6 +1,15 @@
-
## Compatible Services
+OpenDAL provides provider presets for services with a maintained, narrow
+configuration contract:
+
+- Use [`crate::R2`] or the `r2` scheme for Cloudflare R2.
+- Use [`crate::Minio`] or the `minio` scheme for MinIO deployments.
+
+The presets own provider defaults and validation. Use this generic [`crate::S3`]
+builder for AWS S3, for compatible services without a built-in preset, or when
+an application requires S3 options outside a preset's contract.
+
### AWS S3
[AWS S3](https://aws.amazon.com/s3/) is the default implementations of s3 services. Only `bucket` is required.
@@ -41,22 +50,6 @@ builder.bucket("<bucket_name>");
builder.enable_virtual_host_style();
```
-### Minio
-
-[minio](https://min.io/) is an open-source s3 compatible services.
-
-To connect to minio, we need to set:
-
-- `endpoint`: The endpoint of minio, for example: `http://127.0.0.1:9000`
-- `region`: The region of minio. If you don't care about it, just set it to "auto", it will be ignored.
-- `bucket`: The bucket name of minio.
-
-```rust,ignore
-builder.endpoint("http://127.0.0.1:9000");
-builder.region("<region>");
-builder.bucket("<bucket_name>");
-```
-
### QingStor Object Storage
[QingStor Object Storage](https://www.qingcloud.com/products/qingstor) is a S3-compatible service provided by [QingCloud](https://www.qingcloud.com/).
@@ -122,25 +115,6 @@ To connect to wasabi, we need to set:
> Refer to [What are the service URLs for Wasabi's different storage regions?](https://wasabi-support.zendesk.com/hc/en-us/articles/360015106031) for more details.
-### Cloudflare R2
-
-[Cloudflare R2](https://developers.cloudflare.com/r2/) provides s3 compatible API.
-
-> Cloudflare R2 Storage allows developers to store large amounts of unstructured data without the costly egress bandwidth fees associated with typical cloud storage services.
-
-
-To connect to r2, we need to set:
-
-- `endpoint`: The endpoint of r2, for example: `https://<account_id>.r2.cloudflarestorage.com`
-- `bucket`: The bucket name of r2.
-- `region`: When you create a new bucket, the data location is set to Automatic by default. So please use `auto` for region.
-- `enable_exact_buf_write`: R2 requires the non-tailing parts size to be exactly the same. Please enable this option to avoid the error `All non-trailing parts must have the same length`.
-
-R2 has the following capability differences from S3:
-
-- `delete_max_size`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please override `delete_max_size` to `700`.
-- `stat_with_override_cache_control`, `stat_with_override_content_disposition`, `stat_with_override_content_type`: R2 doesn't support stat with response override queries. Please override them to `false`.
-
### Google Cloud Storage XML API
[Google Cloud Storage XML API](https://cloud.google.com/storage/docs/xml-api/overview) provides s3 compatible API.
- `endpoint`: The endpoint of Google Cloud Storage XML API, for example: `https://storage.googleapis.com`
diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md
index 66dfe5ab0..feb4e98af 100644
--- a/core/services/s3/src/docs.md
+++ b/core/services/s3/src/docs.md
@@ -79,7 +79,7 @@ async fn main() -> Result<()> {
.root("/path/to/dir")
// Set the bucket name. This is required.
.bucket("test")
- // Set the region. This is required for some services, if you don't care about it, for example Minio service, just set it to "auto", it will be ignored.
+ // Set the signing region. Compatible services define their own region rules.
.region("us-east-1")
// Set the endpoint.
//
diff --git a/core/services/s3/src/lib.rs b/core/services/s3/src/lib.rs
index 8e34b7dd1..684132a2a 100644
--- a/core/services/s3/src/lib.rs
+++ b/core/services/s3/src/lib.rs
@@ -26,16 +26,29 @@ mod copier;
mod core;
mod deleter;
mod lister;
+mod minio;
+mod minio_config;
+mod preset;
+mod r2;
+mod r2_config;
mod reader;
mod writer;
pub use backend::S3Builder as S3;
pub use config::S3Config;
+pub use minio::MinioBuilder as Minio;
+pub use minio_config::MinioConfig;
+pub use r2::R2Builder as R2;
+pub use r2_config::R2Config;
/// URI scheme used for service registration and scheme-driven construction.
pub const S3_SCHEME: &str = "s3";
+/// URI scheme for the Cloudflare R2 preset.
+pub const R2_SCHEME: &str = "r2";
+/// URI scheme for the MinIO preset.
+pub const MINIO_SCHEME: &str = "minio";
-/// Register this service's URI scheme or schemes with an operator registry.
+/// Register the Amazon S3 URI scheme with an operator registry.
///
/// Registration enables scheme-driven construction through
/// [`opendal_core::Operator::from_uri`] and
@@ -44,3 +57,54 @@ pub const S3_SCHEME: &str = "s3";
pub fn register_s3_service(registry: &opendal_core::OperatorRegistry) {
registry.register::<S3>(S3_SCHEME);
}
+
+/// Register the Cloudflare R2 URI scheme with an operator registry.
+///
+/// Registration enables scheme-driven construction through
+/// [`opendal_core::Operator::from_uri`] and
+/// [`opendal_core::Operator::via_iter`]. Direct construction through
+/// [`opendal_core::Operator::new`] does not require registration.
+pub fn register_r2_service(registry: &opendal_core::OperatorRegistry) {
+ registry.register::<R2>(R2_SCHEME);
+}
+
+/// Register the MinIO URI scheme with an operator registry.
+///
+/// Registration enables scheme-driven construction through
+/// [`opendal_core::Operator::from_uri`] and
+/// [`opendal_core::Operator::via_iter`]. Direct construction through
+/// [`opendal_core::Operator::new`] does not require registration.
+pub fn register_minio_service(registry: &opendal_core::OperatorRegistry) {
+ registry.register::<Minio>(MINIO_SCHEME);
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn registers_s3_and_provider_schemes() {
+ let registry = opendal_core::OperatorRegistry::get();
+ register_s3_service(registry);
+ register_r2_service(registry);
+ register_minio_service(registry);
+
+ let schemes = registry.schemes();
+ assert!(schemes.contains(S3_SCHEME));
+ assert!(schemes.contains(R2_SCHEME));
+ assert!(schemes.contains(MINIO_SCHEME));
+
+ let r2 = registry
+ .load(("r2://bucket/root", [("account_id", "example-account")]))
+ .unwrap();
+ assert_eq!(r2.info().scheme(), R2_SCHEME);
+
+ let minio = registry
+ .load((
+ "minio://bucket/root",
+ [("endpoint", "http://127.0.0.1:9000")],
+ ))
+ .unwrap();
+ assert_eq!(minio.info().scheme(), MINIO_SCHEME);
+ }
+}
diff --git a/core/services/s3/src/minio.md b/core/services/s3/src/minio.md
new file mode 100644
index 000000000..be0cb6e84
--- /dev/null
+++ b/core/services/s3/src/minio.md
@@ -0,0 +1,51 @@
+## Configuration
+
+Use [`crate::MinioConfig`] for serializable configuration or this builder for
+direct construction. The MinIO preset accepts only deployment connection and
+credential settings. Use [`crate::S3`] when an application needs the complete
+S3 configuration surface. Applications that use the `opendal` facade enable
+the `services-minio` feature.
+
+Every MinIO deployment must provide an `endpoint`. The signing `region`
+defaults to `auto`; set it explicitly when the deployment requires a configured
+region.
+
+The preset loads credentials from direct settings, standard AWS environment
+variables, or static credentials in the shared AWS credential files. It does
+not use AWS SSO, web identity, credential processes, ECS, EC2 metadata, or
+AssumeRole. Call [`MinioBuilder::skip_signature`] for a deployment that accepts
+anonymous requests.
+
+## Examples
+
+Build an operator for a local MinIO deployment:
+
+```rust
+use opendal_core::{Operator, Result};
+use opendal_service_s3::Minio;
+
+fn build() -> Result<Operator> {
+ Operator::new(
+ Minio::default()
+ .bucket("data")
+ .endpoint("http://127.0.0.1:9000")
+ .access_key_id("minioadmin")
+ .secret_access_key("minioadmin"),
+ )
+}
+```
+
+After registering the service, construct it from a `minio://` URI:
+
+```rust
+use opendal_core::{Operator, OperatorRegistry, Result};
+use opendal_service_s3::register_minio_service;
+
+fn build() -> Result<Operator> {
+ register_minio_service(OperatorRegistry::get());
+ Operator::from_uri((
+ "minio://data/root",
+ [("endpoint", "http://127.0.0.1:9000")],
+ ))
+}
+```
diff --git a/core/services/s3/src/minio.rs b/core/services/s3/src/minio.rs
new file mode 100644
index 000000000..2422fa6d8
--- /dev/null
+++ b/core/services/s3/src/minio.rs
@@ -0,0 +1,257 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use opendal_core::Builder;
+use opendal_core::Result;
+use opendal_core::raw::Service;
+use reqsign_aws_v4::Credential;
+use reqsign_core::ProvideCredentialChain;
+
+use crate::MINIO_SCHEME;
+use crate::backend::S3Builder;
+use crate::config::S3Config;
+use crate::minio_config::MinioConfig;
+use crate::preset;
+
+/// Builds a MinIO service with provider-specific configuration.
+#[doc = include_str!("minio.md")]
+#[derive(Default)]
+pub struct MinioBuilder {
+ config: MinioConfig,
+}
+
+impl Debug for MinioBuilder {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("MinioBuilder")
+ .field("config", &self.config)
+ .finish_non_exhaustive()
+ }
+}
+
+impl MinioBuilder {
+ pub(crate) fn from_config(config: MinioConfig) -> Self {
+ Self { config }
+ }
+
+ /// Set the root within the bucket.
+ pub fn root(mut self, root: &str) -> Self {
+ self.config.root = if root.is_empty() {
+ None
+ } else {
+ Some(root.to_string())
+ };
+ self
+ }
+
+ /// Set the bucket name.
+ pub fn bucket(mut self, bucket: &str) -> Self {
+ self.config.bucket = bucket.to_string();
+ self
+ }
+
+ /// Set the MinIO endpoint.
+ pub fn endpoint(mut self, endpoint: &str) -> Self {
+ self.config.endpoint = Some(endpoint.to_string());
+ self
+ }
+
+ /// Set the signing region.
+ pub fn region(mut self, region: &str) -> Self {
+ self.config.region = Some(region.to_string());
+ self
+ }
+
+ /// Set the access key ID.
+ pub fn access_key_id(mut self, access_key_id: &str) -> Self {
+ self.config.access_key_id = Some(access_key_id.to_string());
+ self
+ }
+
+ /// Set the secret access key.
+ pub fn secret_access_key(mut self, secret_access_key: &str) -> Self {
+ self.config.secret_access_key = Some(secret_access_key.to_string());
+ self
+ }
+
+ /// Set the session token for temporary credentials.
+ pub fn session_token(mut self, session_token: &str) -> Self {
+ self.config.session_token = Some(session_token.to_string());
+ self
+ }
+
+ /// Send requests without signing them.
+ pub fn skip_signature(mut self) -> Self {
+ self.config.skip_signature = true;
+ self
+ }
+
+ fn into_s3_config(self) -> Result<(S3Config, ProvideCredentialChain<Credential>)> {
+ let MinioConfig {
+ root,
+ bucket,
+ endpoint,
+ region,
+ access_key_id,
+ secret_access_key,
+ session_token,
+ skip_signature,
+ } = self.config;
+
+ preset::validate_required(&bucket, MINIO_SCHEME, "bucket must not be empty")?;
+ preset::validate_optional(
+ endpoint.as_deref(),
+ MINIO_SCHEME,
+ "endpoint must not be empty when set",
+ )?;
+ preset::validate_optional(
+ region.as_deref(),
+ MINIO_SCHEME,
+ "region must not be empty when set",
+ )?;
+
+ let endpoint = endpoint
+ .ok_or_else(|| preset::config_error(MINIO_SCHEME, "endpoint is required for MinIO"))?;
+
+ if skip_signature
+ && (access_key_id.is_some() || secret_access_key.is_some() || session_token.is_some())
+ {
+ return Err(preset::config_error(
+ MINIO_SCHEME,
+ "skip_signature cannot be combined with direct credentials",
+ ));
+ }
+
+ preset::validate_credentials(
+ access_key_id.as_deref(),
+ secret_access_key.as_deref(),
+ session_token.as_deref(),
+ MINIO_SCHEME,
+ )?;
+
+ let credential_providers = preset::credential_chain(
+ access_key_id.as_deref(),
+ secret_access_key.as_deref(),
+ session_token.as_deref(),
+ );
+
+ let config = S3Config {
+ root,
+ bucket,
+ endpoint: Some(endpoint),
+ region: Some(region.unwrap_or_else(|| "auto".to_string())),
+ access_key_id,
+ secret_access_key,
+ session_token,
+ disable_ec2_metadata: true,
+ skip_signature,
+ ..Default::default()
+ };
+
+ Ok((config, credential_providers))
+ }
+}
+
+impl Builder for MinioBuilder {
+ type Config = MinioConfig;
+
+ fn build(self) -> Result<impl Service> {
+ let (config, credential_providers) = self.into_s3_config()?;
+ S3Builder::from_provider_config(config, credential_providers)
+ .build_with_scheme(MINIO_SCHEME)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use opendal_core::ErrorKind;
+ use opendal_core::Operator;
+
+ use super::*;
+
+ fn build_config(builder: MinioBuilder) -> S3Config {
+ builder.into_s3_config().unwrap().0
+ }
+
+ #[test]
+ fn defaults_region_to_auto() {
+ let config = build_config(
+ MinioBuilder::default()
+ .bucket("bucket")
+ .endpoint("http://127.0.0.1:9000"),
+ );
+ assert_eq!(config.region.as_deref(), Some("auto"));
+ }
+
+ #[test]
+ fn accepts_explicit_region() {
+ let config = build_config(
+ MinioBuilder::default()
+ .bucket("bucket")
+ .endpoint("http://127.0.0.1:9000")
+ .region("us-east-1"),
+ );
+ assert_eq!(config.region.as_deref(), Some("us-east-1"));
+ }
+
+ #[test]
+ fn requires_endpoint() {
+ let err = MinioBuilder::default()
+ .bucket("bucket")
+ .into_s3_config()
+ .unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ConfigInvalid);
+ }
+
+ #[test]
+ fn rejects_incomplete_credentials() {
+ let err = MinioBuilder::default()
+ .bucket("bucket")
+ .endpoint("http://127.0.0.1:9000")
+ .session_token("token")
+ .into_s3_config()
+ .unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ConfigInvalid);
+ }
+
+ #[test]
+ fn rejects_credentials_in_anonymous_mode() {
+ let err = MinioBuilder::default()
+ .bucket("bucket")
+ .endpoint("http://127.0.0.1:9000")
+ .access_key_id("access")
+ .secret_access_key("secret")
+ .skip_signature()
+ .into_s3_config()
+ .unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ConfigInvalid);
+ }
+
+ #[test]
+ fn reports_minio_scheme() {
+ let operator = Operator::new(
+ MinioBuilder::default()
+ .bucket("bucket")
+ .endpoint("http://127.0.0.1:9000")
+ .access_key_id("access")
+ .secret_access_key("secret"),
+ )
+ .unwrap();
+ assert_eq!(operator.info().scheme(), MINIO_SCHEME);
+ }
+}
diff --git a/core/services/s3/src/minio_config.rs b/core/services/s3/src/minio_config.rs
new file mode 100644
index 000000000..3053f8f5d
--- /dev/null
+++ b/core/services/s3/src/minio_config.rs
@@ -0,0 +1,161 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use opendal_core::Configurator;
+use opendal_core::OperatorUri;
+use opendal_core::Result;
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::minio::MinioBuilder;
+use crate::preset;
+
+/// Configuration for a MinIO deployment.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default, deny_unknown_fields)]
+#[non_exhaustive]
+pub struct MinioConfig {
+ /// Root within the bucket.
+ ///
+ /// All operations happen under this root. The default is `/`.
+ ///
+ /// <!-- @group General -->
+ /// <!-- @default / -->
+ pub root: Option<String>,
+ /// Bucket name.
+ ///
+ /// This field is required.
+ ///
+ /// <!-- @group General -->
+ /// <!-- @example my-bucket -->
+ pub bucket: String,
+ /// MinIO endpoint.
+ ///
+ /// This field is required because MinIO deployments do not share a
+ /// universal endpoint.
+ ///
+ /// <!-- @group General -->
+ /// <!-- @example http://127.0.0.1:9000 -->
+ /// <!-- @minimal true -->
+ /// <!-- @required true -->
+ pub endpoint: Option<String>,
+ /// Signing region.
+ ///
+ /// The default is `auto`. Set this field when the deployment requires a
+ /// configured region.
+ ///
+ /// <!-- @group General -->
+ /// <!-- @default auto -->
+ pub region: Option<String>,
+ /// Access key ID.
+ ///
+ /// Set this field together with `secret_access_key`.
+ ///
+ /// <!-- @group Credentials -->
+ pub access_key_id: Option<String>,
+ /// Secret access key.
+ ///
+ /// Set this field together with `access_key_id`.
+ ///
+ /// <!-- @group Credentials -->
+ pub secret_access_key: Option<String>,
+ /// Session token for temporary credentials.
+ ///
+ /// This field requires `access_key_id` and `secret_access_key`.
+ ///
+ /// <!-- @group Credentials -->
+ pub session_token: Option<String>,
+ /// Send requests without signing them.
+ ///
+ /// This option cannot be combined with direct credentials.
+ ///
+ /// <!-- @group Credentials -->
+ /// <!-- @default false -->
+ pub skip_signature: bool,
+}
+
+impl Debug for MinioConfig {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("MinioConfig")
+ .field("root", &self.root)
+ .field("bucket", &self.bucket)
+ .field("endpoint", &self.endpoint)
+ .field("region", &self.region)
+ .field("skip_signature", &self.skip_signature)
+ .finish_non_exhaustive()
+ }
+}
+
+impl Configurator for MinioConfig {
+ type Builder = MinioBuilder;
+
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
+ preset::from_uri(uri)
+ }
+
+ fn into_builder(self) -> Self::Builder {
+ MinioBuilder::from_config(self)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::iter;
+
+ use opendal_core::ErrorKind;
+
+ use super::*;
+
+ #[test]
+ fn from_uri_extracts_bucket_root_and_options() {
+ let uri = OperatorUri::new(
+ "minio://example-bucket/path/to/root?endpoint=http%3A%2F%2F127.0.0.1%3A9000",
+ iter::empty(),
+ )
+ .unwrap();
+ let config = MinioConfig::from_uri(&uri).unwrap();
+
+ assert_eq!(config.bucket, "example-bucket");
+ assert_eq!(config.root.as_deref(), Some("path/to/root"));
+ assert_eq!(config.endpoint.as_deref(), Some("http://127.0.0.1:9000"));
+ }
+
+ #[test]
+ fn rejects_unknown_fields() {
+ let err =
+ MinioConfig::from_iter([("role_arn".to_string(), "role".to_string())]).unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ConfigInvalid);
+ }
+
+ #[test]
+ fn debug_redacts_credentials() {
+ let config = MinioConfig {
+ bucket: "bucket".to_string(),
+ access_key_id: Some("access-value".to_string()),
+ secret_access_key: Some("secret-value".to_string()),
+ session_token: Some("token-value".to_string()),
+ ..Default::default()
+ };
+
+ let output = format!("{config:?}");
+ assert!(!output.contains("access-value"));
+ assert!(!output.contains("secret-value"));
+ assert!(!output.contains("token-value"));
+ }
+}
diff --git a/core/services/s3/src/preset.rs b/core/services/s3/src/preset.rs
new file mode 100644
index 000000000..39f73ebb1
--- /dev/null
+++ b/core/services/s3/src/preset.rs
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use opendal_core::Configurator;
+use opendal_core::Error;
+use opendal_core::ErrorKind;
+use opendal_core::OperatorUri;
+use opendal_core::Result;
+use reqsign_aws_v4::Credential;
+use reqsign_aws_v4::EnvCredentialProvider;
+use reqsign_aws_v4::ProfileCredentialProvider;
+use reqsign_aws_v4::StaticCredentialProvider;
+use reqsign_core::ProvideCredentialChain;
+
+pub(crate) fn from_uri<C: Configurator>(uri: &OperatorUri) -> Result<C> {
+ let mut map = uri.options().clone();
+
+ if let Some(name) = uri.name() {
+ map.insert("bucket".to_string(), name.to_string());
+ }
+
+ if let Some(root) = uri.root() {
+ map.insert("root".to_string(), root.to_string());
+ }
+
+ C::from_iter(map)
+}
+
+pub(crate) fn config_error(service: &'static str, message: &'static str) -> Error {
+ Error::new(ErrorKind::ConfigInvalid, message)
+ .with_operation("Builder::build")
+ .with_context("service", service)
+}
+
+pub(crate) fn validate_required(
+ value: &str,
+ service: &'static str,
+ message: &'static str,
+) -> Result<()> {
+ if value.trim().is_empty() {
+ return Err(config_error(service, message));
+ }
+ Ok(())
+}
+
+pub(crate) fn validate_optional(
+ value: Option<&str>,
+ service: &'static str,
+ message: &'static str,
+) -> Result<()> {
+ if value.is_some_and(|value| value.trim().is_empty()) {
+ return Err(config_error(service, message));
+ }
+ Ok(())
+}
+
+pub(crate) fn validate_credentials(
+ access_key_id: Option<&str>,
+ secret_access_key: Option<&str>,
+ session_token: Option<&str>,
+ service: &'static str,
+) -> Result<()> {
+ validate_optional(
+ access_key_id,
+ service,
+ "access_key_id must not be empty when set",
+ )?;
+ validate_optional(
+ secret_access_key,
+ service,
+ "secret_access_key must not be empty when set",
+ )?;
+ validate_optional(
+ session_token,
+ service,
+ "session_token must not be empty when set",
+ )?;
+
+ match (access_key_id, secret_access_key, session_token) {
+ (None, None, None) | (Some(_), Some(_), None | Some(_)) => Ok(()),
+ _ => Err(config_error(
+ service,
+ "access_key_id and secret_access_key must be provided together; session_token requires both",
+ )),
+ }
+}
+
+pub(crate) fn credential_chain(
+ access_key_id: Option<&str>,
+ secret_access_key: Option<&str>,
+ session_token: Option<&str>,
+) -> ProvideCredentialChain<Credential> {
+ let mut chain = ProvideCredentialChain::new()
+ .push(EnvCredentialProvider::new())
+ .push(ProfileCredentialProvider::new());
+
+ if let (Some(access_key_id), Some(secret_access_key)) = (access_key_id, secret_access_key) {
+ let provider = if let Some(session_token) = session_token {
+ StaticCredentialProvider::new(access_key_id, secret_access_key)
+ .with_session_token(session_token)
+ } else {
+ StaticCredentialProvider::new(access_key_id, secret_access_key)
+ };
+ chain = chain.push_front(provider);
+ }
+
+ chain
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashMap;
+
+ use reqsign_core::Context;
+ use reqsign_core::ProvideCredential;
+ use reqsign_core::StaticEnv;
+
+ use super::*;
+
+ #[test]
+ fn credential_chain_contains_only_declared_sources() {
+ assert_eq!(credential_chain(None, None, None).len(), 2);
+ assert_eq!(
+ credential_chain(Some("access"), Some("secret"), Some("token")).len(),
+ 3
+ );
+ }
+
+ #[tokio::test]
+ async fn direct_credentials_take_priority_over_environment() {
+ let context = Context::new().with_env(StaticEnv {
+ home_dir: None,
+ envs: HashMap::from([
+ ("AWS_ACCESS_KEY_ID".to_string(), "env-access".to_string()),
+ (
+ "AWS_SECRET_ACCESS_KEY".to_string(),
+ "env-secret".to_string(),
+ ),
+ ]),
+ });
+ let chain = credential_chain(Some("direct-access"), Some("direct-secret"), None);
+
+ let credential = chain.provide_credential(&context).await.unwrap().unwrap();
+ assert_eq!(credential.access_key_id, "direct-access");
+ assert_eq!(credential.secret_access_key, "direct-secret");
+ }
+
+ #[tokio::test]
+ async fn loads_credentials_from_standard_environment() {
+ let context = Context::new().with_env(StaticEnv {
+ home_dir: None,
+ envs: HashMap::from([
+ ("AWS_ACCESS_KEY_ID".to_string(), "env-access".to_string()),
+ (
+ "AWS_SECRET_ACCESS_KEY".to_string(),
+ "env-secret".to_string(),
+ ),
+ ]),
+ });
+ let chain = credential_chain(None, None, None);
+
+ let credential = chain.provide_credential(&context).await.unwrap().unwrap();
+ assert_eq!(credential.access_key_id, "env-access");
+ assert_eq!(credential.secret_access_key, "env-secret");
+ }
+}
diff --git a/core/services/s3/src/r2.md b/core/services/s3/src/r2.md
new file mode 100644
index 000000000..e099f0d2c
--- /dev/null
+++ b/core/services/s3/src/r2.md
@@ -0,0 +1,53 @@
+## Configuration
+
+Use [`crate::R2Config`] for serializable configuration or this builder for
+direct construction. The R2 preset accepts only R2 connection and credential
+settings. Use [`crate::S3`] when an application needs the complete S3
+configuration surface. Applications that use the `opendal` facade enable the
+`services-r2` feature.
+
+Set exactly one endpoint source:
+
+- Set `account_id` to derive
+ `https://<ACCOUNT_ID>.r2.cloudflarestorage.com`.
+- Set `account_id` and `jurisdiction` to derive a jurisdictional endpoint.
+- Set `endpoint` for a proxy, gateway, or test server.
+
+The preset uses `auto` as the signing region. It loads credentials from direct
+settings, standard AWS environment variables, or static credentials in the
+shared AWS credential files. It does not use AWS SSO, web identity, credential
+processes, ECS, EC2 metadata, or AssumeRole.
+
+## Examples
+
+Build an operator from an account ID:
+
+```rust
+use opendal_core::{Operator, Result};
+use opendal_service_s3::R2;
+
+fn build() -> Result<Operator> {
+ Operator::new(
+ R2::default()
+ .bucket("data")
+ .account_id("example-account")
+ .access_key_id("example-access-key")
+ .secret_access_key("example-secret-key"),
+ )
+}
+```
+
+After registering the service, construct it from an `r2://` URI:
+
+```rust
+use opendal_core::{Operator, OperatorRegistry, Result};
+use opendal_service_s3::register_r2_service;
+
+fn build() -> Result<Operator> {
+ register_r2_service(OperatorRegistry::get());
+ Operator::from_uri((
+ "r2://data/root",
+ [("account_id", "example-account")],
+ ))
+}
+```
diff --git a/core/services/s3/src/r2.rs b/core/services/s3/src/r2.rs
new file mode 100644
index 000000000..a77ab3d44
--- /dev/null
+++ b/core/services/s3/src/r2.rs
@@ -0,0 +1,298 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use opendal_core::Builder;
+use opendal_core::Result;
+use opendal_core::raw::Service;
+use reqsign_aws_v4::Credential;
+use reqsign_core::ProvideCredentialChain;
+
+use crate::R2_SCHEME;
+use crate::backend::S3Builder;
+use crate::config::S3Config;
+use crate::preset;
+use crate::r2_config::R2Config;
+
+/// Builds a Cloudflare R2 service with provider-specific configuration.
+#[doc = include_str!("r2.md")]
+#[derive(Default)]
+pub struct R2Builder {
+ config: R2Config,
+}
+
+impl Debug for R2Builder {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("R2Builder")
+ .field("config", &self.config)
+ .finish_non_exhaustive()
+ }
+}
+
+impl R2Builder {
+ pub(crate) fn from_config(config: R2Config) -> Self {
+ Self { config }
+ }
+
+ /// Set the root within the bucket.
+ pub fn root(mut self, root: &str) -> Self {
+ self.config.root = if root.is_empty() {
+ None
+ } else {
+ Some(root.to_string())
+ };
+ self
+ }
+
+ /// Set the bucket name.
+ pub fn bucket(mut self, bucket: &str) -> Self {
+ self.config.bucket = bucket.to_string();
+ self
+ }
+
+ /// Set the Cloudflare account ID used to derive the endpoint.
+ pub fn account_id(mut self, account_id: &str) -> Self {
+ self.config.account_id = Some(account_id.to_string());
+ self
+ }
+
+ /// Set the R2 jurisdiction to `eu` or `fedramp`.
+ pub fn jurisdiction(mut self, jurisdiction: &str) -> Self {
+ self.config.jurisdiction = Some(jurisdiction.to_string());
+ self
+ }
+
+ /// Set an explicit R2-compatible endpoint.
+ pub fn endpoint(mut self, endpoint: &str) -> Self {
+ self.config.endpoint = Some(endpoint.to_string());
+ self
+ }
+
+ /// Set the access key ID.
+ pub fn access_key_id(mut self, access_key_id: &str) -> Self {
+ self.config.access_key_id = Some(access_key_id.to_string());
+ self
+ }
+
+ /// Set the secret access key.
+ pub fn secret_access_key(mut self, secret_access_key: &str) -> Self {
+ self.config.secret_access_key = Some(secret_access_key.to_string());
+ self
+ }
+
+ /// Set the session token for temporary credentials.
+ pub fn session_token(mut self, session_token: &str) -> Self {
+ self.config.session_token = Some(session_token.to_string());
+ self
+ }
+
+ fn into_s3_config(self) -> Result<(S3Config, ProvideCredentialChain<Credential>)> {
+ let R2Config {
+ root,
+ bucket,
+ account_id,
+ jurisdiction,
+ endpoint,
+ access_key_id,
+ secret_access_key,
+ session_token,
+ } = self.config;
+
+ preset::validate_required(&bucket, R2_SCHEME, "bucket must not be empty")?;
+ preset::validate_optional(
+ account_id.as_deref(),
+ R2_SCHEME,
+ "account_id must not be empty when set",
+ )?;
+ preset::validate_optional(
+ jurisdiction.as_deref(),
+ R2_SCHEME,
+ "jurisdiction must not be empty when set",
+ )?;
+ preset::validate_optional(
+ endpoint.as_deref(),
+ R2_SCHEME,
+ "endpoint must not be empty when set",
+ )?;
+ preset::validate_credentials(
+ access_key_id.as_deref(),
+ secret_access_key.as_deref(),
+ session_token.as_deref(),
+ R2_SCHEME,
+ )?;
+
+ let endpoint = match (account_id.as_deref(), endpoint.as_deref()) {
+ (Some(account_id), None) => match jurisdiction.as_deref() {
+ None => format!("https://{account_id}.r2.cloudflarestorage.com"),
+ Some(jurisdiction @ ("eu" | "fedramp")) => {
+ format!("https://{account_id}.{jurisdiction}.r2.cloudflarestorage.com")
+ }
+ Some(_) => {
+ return Err(preset::config_error(
+ R2_SCHEME,
+ "jurisdiction must be either eu or fedramp",
+ ));
+ }
+ },
+ (None, Some(endpoint)) if jurisdiction.is_none() => endpoint.to_string(),
+ (None, Some(_)) => {
+ return Err(preset::config_error(
+ R2_SCHEME,
+ "jurisdiction requires account_id and cannot be used with endpoint",
+ ));
+ }
+ (Some(_), Some(_)) => {
+ return Err(preset::config_error(
+ R2_SCHEME,
+ "account_id and endpoint are mutually exclusive",
+ ));
+ }
+ (None, None) => {
+ return Err(preset::config_error(
+ R2_SCHEME,
+ "exactly one of account_id and endpoint is required",
+ ));
+ }
+ };
+
+ let credential_providers = preset::credential_chain(
+ access_key_id.as_deref(),
+ secret_access_key.as_deref(),
+ session_token.as_deref(),
+ );
+
+ let config = S3Config {
+ root,
+ bucket,
+ endpoint: Some(endpoint),
+ region: Some("auto".to_string()),
+ access_key_id,
+ secret_access_key,
+ session_token,
+ disable_ec2_metadata: true,
+ ..Default::default()
+ };
+
+ Ok((config, credential_providers))
+ }
+}
+
+impl Builder for R2Builder {
+ type Config = R2Config;
+
+ fn build(self) -> Result<impl Service> {
+ let (config, credential_providers) = self.into_s3_config()?;
+ S3Builder::from_provider_config(config, credential_providers).build_with_scheme(R2_SCHEME)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use opendal_core::ErrorKind;
+ use opendal_core::Operator;
+
+ use super::*;
+
+ fn build_config(builder: R2Builder) -> S3Config {
+ builder.into_s3_config().unwrap().0
+ }
+
+ #[test]
+ fn derives_default_endpoint() {
+ let config = build_config(R2Builder::default().bucket("bucket").account_id("account"));
+ assert_eq!(
+ config.endpoint.as_deref(),
+ Some("https://account.r2.cloudflarestorage.com")
+ );
+ assert_eq!(config.region.as_deref(), Some("auto"));
+ }
+
+ #[test]
+ fn derives_jurisdiction_endpoints() {
+ for jurisdiction in ["eu", "fedramp"] {
+ let config = build_config(
+ R2Builder::default()
+ .bucket("bucket")
+ .account_id("account")
+ .jurisdiction(jurisdiction),
+ );
+ let expected = format!("https://account.{jurisdiction}.r2.cloudflarestorage.com");
+ assert_eq!(config.endpoint.as_deref(), Some(expected.as_str()));
+ }
+ }
+
+ #[test]
+ fn accepts_explicit_endpoint() {
+ let config = build_config(
+ R2Builder::default()
+ .bucket("bucket")
+ .endpoint("http://127.0.0.1:9000"),
+ );
+ assert_eq!(config.endpoint.as_deref(), Some("http://127.0.0.1:9000"));
+ }
+
+ #[test]
+ fn rejects_invalid_endpoint_and_jurisdiction_combinations() {
+ let cases = [
+ R2Builder::default().bucket("bucket"),
+ R2Builder::default()
+ .bucket("bucket")
+ .account_id("account")
+ .endpoint("https://example.com"),
+ R2Builder::default()
+ .bucket("bucket")
+ .endpoint("https://example.com")
+ .jurisdiction("eu"),
+ R2Builder::default()
+ .bucket("bucket")
+ .account_id("account")
+ .jurisdiction("invalid"),
+ ];
+
+ for builder in cases {
+ assert_eq!(
+ builder.into_s3_config().unwrap_err().kind(),
+ ErrorKind::ConfigInvalid
+ );
+ }
+ }
+
+ #[test]
+ fn rejects_incomplete_credentials() {
+ let err = R2Builder::default()
+ .bucket("bucket")
+ .account_id("account")
+ .session_token("token")
+ .into_s3_config()
+ .unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ConfigInvalid);
+ }
+
+ #[test]
+ fn reports_r2_scheme() {
+ let operator = Operator::new(
+ R2Builder::default()
+ .bucket("bucket")
+ .account_id("account")
+ .access_key_id("access")
+ .secret_access_key("secret"),
+ )
+ .unwrap();
+ assert_eq!(operator.info().scheme(), R2_SCHEME);
+ }
+}
diff --git a/core/services/s3/src/r2_config.rs b/core/services/s3/src/r2_config.rs
new file mode 100644
index 000000000..95b924a7a
--- /dev/null
+++ b/core/services/s3/src/r2_config.rs
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use opendal_core::Configurator;
+use opendal_core::OperatorUri;
+use opendal_core::Result;
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::preset;
+use crate::r2::R2Builder;
+
+/// Configuration for Cloudflare R2.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default, deny_unknown_fields)]
+#[non_exhaustive]
+pub struct R2Config {
+ /// Root within the bucket.
+ ///
+ /// All operations happen under this root. The default is `/`.
+ ///
+ /// <!-- @group General -->
+ /// <!-- @default / -->
+ pub root: Option<String>,
+ /// Bucket name.
+ ///
+ /// This field is required.
+ ///
+ /// <!-- @group General -->
+ /// <!-- @example my-bucket -->
+ pub bucket: String,
+ /// Cloudflare account ID used to derive the R2 endpoint.
+ ///
+ /// Set exactly one of `account_id` and `endpoint`.
+ ///
+ /// <!-- @group General -->
+ /// <!-- @example example-account -->
+ /// <!-- @minimal true -->
+ pub account_id: Option<String>,
+ /// R2 jurisdiction.
+ ///
+ /// Supported values are `eu` and `fedramp`. This field requires
+ /// `account_id` and cannot be used with `endpoint`.
+ ///
+ /// <!-- @group General -->
+ pub jurisdiction: Option<String>,
+ /// Explicit R2-compatible endpoint.
+ ///
+ /// Use this field for a proxy, gateway, or test server. Set exactly one of
+ /// `endpoint` and `account_id`.
+ ///
+ /// <!-- @group General -->
+ /// <!-- @example https://example.r2.cloudflarestorage.com -->
+ pub endpoint: Option<String>,
+ /// Access key ID.
+ ///
+ /// Set this field together with `secret_access_key`.
+ ///
+ /// <!-- @group Credentials -->
+ pub access_key_id: Option<String>,
+ /// Secret access key.
+ ///
+ /// Set this field together with `access_key_id`.
+ ///
+ /// <!-- @group Credentials -->
+ pub secret_access_key: Option<String>,
+ /// Session token for temporary credentials.
+ ///
+ /// This field requires `access_key_id` and `secret_access_key`.
+ ///
+ /// <!-- @group Credentials -->
+ pub session_token: Option<String>,
+}
+
+impl Debug for R2Config {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("R2Config")
+ .field("root", &self.root)
+ .field("bucket", &self.bucket)
+ .field("account_id", &self.account_id)
+ .field("jurisdiction", &self.jurisdiction)
+ .field("endpoint", &self.endpoint)
+ .finish_non_exhaustive()
+ }
+}
+
+impl Configurator for R2Config {
+ type Builder = R2Builder;
+
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
+ preset::from_uri(uri)
+ }
+
+ fn into_builder(self) -> Self::Builder {
+ R2Builder::from_config(self)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::iter;
+
+ use opendal_core::ErrorKind;
+
+ use super::*;
+
+ #[test]
+ fn from_uri_extracts_bucket_root_and_options() {
+ let uri = OperatorUri::new(
+ "r2://example-bucket/path/to/root?account_id=example-account",
+ iter::empty(),
+ )
+ .unwrap();
+ let config = R2Config::from_uri(&uri).unwrap();
+
+ assert_eq!(config.bucket, "example-bucket");
+ assert_eq!(config.root.as_deref(), Some("path/to/root"));
+ assert_eq!(config.account_id.as_deref(), Some("example-account"));
+ }
+
+ #[test]
+ fn rejects_unknown_fields() {
+ let err = R2Config::from_iter([("role_arn".to_string(), "role".to_string())]).unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::ConfigInvalid);
+ }
+
+ #[test]
+ fn debug_redacts_credentials() {
+ let config = R2Config {
+ bucket: "bucket".to_string(),
+ access_key_id: Some("access-value".to_string()),
+ secret_access_key: Some("secret-value".to_string()),
+ session_token: Some("token-value".to_string()),
+ ..Default::default()
+ };
+
+ let output = format!("{config:?}");
+ assert!(!output.contains("access-value"));
+ assert!(!output.contains("secret-value"));
+ assert!(!output.contains("token-value"));
+ }
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 197bf54f5..21c6ed827 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -215,6 +215,12 @@ fn init_default_registry_inner(registry: &OperatorRegistry) {
#[cfg(feature = "services-s3")]
opendal_service_s3::register_s3_service(registry);
+ #[cfg(feature = "services-r2")]
+ opendal_service_s3::register_r2_service(registry);
+
+ #[cfg(feature = "services-minio")]
+ opendal_service_s3::register_minio_service(registry);
+
#[cfg(feature = "services-seafile")]
opendal_service_seafile::register_seafile_service(registry);
@@ -274,7 +280,7 @@ fn register_default_operator_registry() {
///
/// [`Memory`](services::Memory) is always available. Other builders are
/// re-exported when their matching facade feature is enabled; for example,
-/// `services-s3` enables [`S3`](services::S3).
+/// `services-r2` enables the `R2` builder.
///
/// Pass a configured builder to [`Operator::new`]. The facade automatically
/// registers enabled services for [`Operator::from_uri`] and
@@ -379,8 +385,12 @@ pub mod services {
pub use opendal_service_redis::*;
#[cfg(feature = "services-rocksdb")]
pub use opendal_service_rocksdb::*;
+ #[cfg(feature = "services-minio")]
+ pub use opendal_service_s3::{MINIO_SCHEME, Minio, MinioConfig, register_minio_service};
+ #[cfg(feature = "services-r2")]
+ pub use opendal_service_s3::{R2, R2_SCHEME, R2Config, register_r2_service};
#[cfg(feature = "services-s3")]
- pub use opendal_service_s3::*;
+ pub use opendal_service_s3::{S3, S3_SCHEME, S3Config, register_s3_service};
#[cfg(feature = "services-seafile")]
pub use opendal_service_seafile::*;
#[cfg(feature = "services-sftp")]
@@ -411,6 +421,43 @@ pub mod services {
pub use opendal_service_yandex_disk::*;
}
+#[cfg(test)]
+mod service_feature_tests {
+ use super::*;
+
+ #[test]
+ fn registers_enabled_s3_provider_schemes() {
+ init_default_registry();
+ let schemes = OperatorRegistry::get().schemes();
+
+ assert!(schemes.contains("memory"));
+
+ #[cfg(feature = "services-s3")]
+ {
+ let _ = services::S3::default();
+ assert!(schemes.contains("s3"));
+ }
+ #[cfg(not(feature = "services-s3"))]
+ assert!(!schemes.contains("s3"));
+
+ #[cfg(feature = "services-r2")]
+ {
+ let _ = services::R2::default();
+ assert!(schemes.contains("r2"));
+ }
+ #[cfg(not(feature = "services-r2"))]
+ assert!(!schemes.contains("r2"));
+
+ #[cfg(feature = "services-minio")]
+ {
+ let _ = services::Minio::default();
+ assert!(schemes.contains("minio"));
+ }
+ #[cfg(not(feature = "services-minio"))]
+ assert!(!schemes.contains("minio"));
+ }
+}
+
/// Layers enabled through `layers-*` Cargo features.
///
/// A layer adds cross-service behavior to an [`Operator`]. Enable the matching
diff --git a/dev/src/generate/docs.rs b/dev/src/generate/docs.rs
index 9bcd49ae0..0df031607 100644
--- a/dev/src/generate/docs.rs
+++ b/dev/src/generate/docs.rs
@@ -26,7 +26,7 @@
//! all expect. Adding a new binding is a matter of adding an entry to
//! [`BINDINGS`] plus a snippet renderer in [`render_examples`].
-use crate::generate::parser::{AttrDeprecated, Config, ConfigType, Services};
+use crate::generate::parser::{AttrDeprecated, Config, ConfigType, S3_PROVIDER_PRESETS, Services};
use anyhow::{Context, Result};
use regex::Regex;
use serde::Serialize;
@@ -198,14 +198,19 @@ pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> {
.map(Field::from)
.collect();
let grouped = group_fields(&fields, &groups);
- let required: Vec<&Field> = fields.iter().filter(|f| f.required).collect();
+ let required: Vec<&Field> = fields.iter().filter(|f| f.required || f.minimal).collect();
let mut examples = Vec::new();
for (binding, support) in BINDINGS.iter().zip(support.iter()) {
- let enabled = match support {
- None => true,
- Some(set) => set.contains(&scheme),
- };
+ // Go loads one dynamic package per scheme, while these presets share
+ // the Rust S3 crate and don't have standalone Go service packages.
+ let enabled =
+ !matches!(binding.id, "go") || !S3_PROVIDER_PRESETS.contains(&scheme.as_str());
+ let enabled = enabled
+ && match support {
+ None => true,
+ Some(set) => set.contains(&scheme),
+ };
if !enabled {
continue;
}
@@ -286,6 +291,7 @@ struct Field<'a> {
name: &'a str,
value: ConfigType,
required: bool,
+ minimal: bool,
comments: &'a str,
group: &'a str,
default: Option<&'a str>,
@@ -298,6 +304,7 @@ impl<'a> From<&'a Config> for Field<'a> {
name: &c.name,
value: c.value,
required: !c.optional,
+ minimal: c.minimal,
comments: &c.comments,
group: c.group.as_deref().unwrap_or(DEFAULT_GROUP),
default: c.default_value.as_deref(),
@@ -388,8 +395,9 @@ struct Syntax {
///
/// `minimal` lists only the required fields. `full` walks every group in order,
/// emitting a group header (when there is more than one group), each field's
-/// doc comment, and the assignment — uncommented when required, commented out
-/// otherwise so the block is a copy-and-trim reference.
+/// doc comment, and the assignment. Required fields and optional fields chosen
+/// for the minimal example stay uncommented so both snippets are valid;
+/// everything else is commented out as a copy-and-trim reference.
fn build(required: &[&Field], grouped: &[(&str, Vec<&Field>)], s: &Syntax) -> (String, String) {
let doc_prefix = format!("{}{}", s.indent, s.comment);
@@ -423,7 +431,7 @@ fn build(required: &[&Field], grouped: &[(&str, Vec<&Field>)], s: &Syntax) -> (S
comment_block(field.comments, &mut field_comment);
let assign = (s.assign)(field);
// field have block indententation
- if field.required {
+ if field.required || field.minimal {
config_options.push(format!("{field_comment}{}{assign}", s.indent));
} else {
config_options.push(format!("{field_comment}{}{}{assign}", s.indent, s.comment));
@@ -528,3 +536,31 @@ fn render_examples(
build(required, grouped, &syntax)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn minimal_optional_fields_stay_active_in_full_examples() {
+ let field = Field {
+ name: "account_id",
+ value: ConfigType::String,
+ required: false,
+ minimal: true,
+ comments: "Cloudflare account ID.",
+ group: DEFAULT_GROUP,
+ default: None,
+ example: Some("example-account"),
+ };
+ let required = vec![&field];
+ let grouped = vec![(DEFAULT_GROUP, vec![&field])];
+
+ let (minimal, full) = render_examples("rust", "r2", &required, &grouped);
+
+ let assignment = "(\"account_id\".to_string(), \"example-account\".to_string()),";
+ assert!(minimal.contains(assignment));
+ assert!(full.contains(&format!(" {assignment}")));
+ assert!(!full.contains(&format!("// {assignment}")));
+ }
+}
diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs
index f46eaf697..954976b44 100644
--- a/dev/src/generate/parser.rs
+++ b/dev/src/generate/parser.rs
@@ -21,6 +21,7 @@ use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::fs;
use std::fs::read_dir;
+use std::path::Path;
use std::str::FromStr;
use syn::{
Expr, ExprLit, Field, GenericArgument, Item, Lit, LitStr, Meta, PathArguments, Type, TypePath,
@@ -29,6 +30,12 @@ use syn::{
pub type Services = HashMap<String, Service>;
+pub const S3_PROVIDER_PRESETS: &[&str] = &["minio", "r2"];
+
+pub fn service_feature(service: &str) -> String {
+ format!("services-{}", service.replace('_', "-"))
+}
+
pub fn sorted_services(services: Services, test: fn(&str) -> bool) -> Services {
let mut srvs = Services::new();
for (k, srv) in services.into_iter() {
@@ -59,7 +66,8 @@ pub struct Config {
pub name: String,
/// The value type this config.
pub value: ConfigType,
- /// If given config is optional or not.
+ /// Whether this config is optional in public configuration. An
+ /// `@required true` marker can make an `Option<T>` field required.
pub optional: bool,
/// if this field is deprecated, a deprecated message will be provided.
pub deprecated: Option<AttrDeprecated>,
@@ -75,6 +83,9 @@ pub struct Config {
pub default_value: Option<String>,
/// An example value, parsed from a `@example <value>` doc marker.
pub example: Option<String>,
+ /// Whether generated minimal examples should include this optional field,
+ /// parsed from a `@minimal true` doc marker.
+ pub minimal: bool,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
@@ -159,13 +170,15 @@ pub struct AttrDeprecated {
pub note: String,
}
-/// Structured markers extracted from a field's doc comment, e.g. `@group`,
-/// `@default` and `@example`.
+/// Structured markers extracted from a field's doc comment, such as `@group`,
+/// `@default`, `@example`, `@minimal`, and `@required`.
#[derive(Debug, Default, Clone, Eq, PartialEq)]
struct DocMarkers {
group: Option<String>,
default_value: Option<String>,
example: Option<String>,
+ minimal: bool,
+ required: bool,
}
/// List and parse given path to a `Services` struct.
@@ -200,29 +213,67 @@ pub fn parse(path: &str) -> Result<Services> {
if dir.file_type()?.is_file() {
continue;
}
- let path = dir.path().join("config.rs");
+ let service_dir = dir.path();
+ let path = service_dir.join("config.rs");
// Try the old layout (config.rs directly) first, then the new layout (src/config.rs)
let path = if path.exists() {
path
} else {
- dir.path().join("src/config.rs")
+ service_dir.join("src/config.rs")
};
- if !path.exists() {
+ if path.exists() {
+ insert_service(
+ &mut map,
+ dir.file_name().to_string_lossy().to_string(),
+ &path,
+ )?;
+ }
+
+ let src_dir = service_dir.join("src");
+ if !src_dir.exists() {
continue;
}
- let content = fs::read_to_string(&path)?;
- let parser = ServiceParser {
- service: dir.file_name().to_string_lossy().to_string(),
- path: path.to_string_lossy().to_string(),
- content,
- };
- let service = parser.parse().context(format!("path: {path:?}"))?;
- map.insert(parser.service, service);
+ // A service crate can expose additional schemes through
+ // `<scheme>_config.rs` files next to its primary `config.rs`.
+ for entry in read_dir(src_dir)? {
+ let entry = entry?;
+ if !entry.file_type()?.is_file() {
+ continue;
+ }
+ let Some(service) = supplemental_service_name(&entry.path()) else {
+ continue;
+ };
+ insert_service(&mut map, service, &entry.path())?;
+ }
}
Ok(map)
}
+fn supplemental_service_name(path: &Path) -> Option<String> {
+ path.file_name()?
+ .to_str()?
+ .strip_suffix("_config.rs")
+ .filter(|name| !name.is_empty())
+ .map(ToString::to_string)
+}
+
+fn insert_service(map: &mut Services, service: String, path: &Path) -> Result<()> {
+ if map.contains_key(&service) {
+ bail!("duplicate config for service {service}");
+ }
+
+ let content = fs::read_to_string(path)?;
+ let parser = ServiceParser {
+ service: service.clone(),
+ path: path.to_string_lossy().to_string(),
+ content,
+ };
+ let parsed = parser.parse().context(format!("path: {path:?}"))?;
+ map.insert(service, parsed);
+ Ok(())
+}
+
/// ServiceParser is used to parse a service config file.
pub struct ServiceParser {
service: String,
@@ -334,6 +385,7 @@ impl ServiceParser {
}
v => return Err(anyhow!("unsupported config type {v:?}")),
};
+ let optional = optional && !markers.required;
Ok(Config {
name: name.to_string(),
@@ -344,6 +396,7 @@ impl ServiceParser {
group: markers.group,
default_value: markers.default_value,
example: markers.example,
+ minimal: markers.minimal,
})
}
@@ -388,6 +441,8 @@ impl ServiceParser {
"group" => Some(("group", value)),
"default" => Some(("default", value)),
"example" => Some(("example", value)),
+ "minimal" => Some(("minimal", value)),
+ "required" => Some(("required", value)),
_ => None,
}
}
@@ -419,6 +474,8 @@ impl ServiceParser {
"group" => markers.group = Some(value),
"default" => markers.default_value = Some(value),
"example" => markers.example = Some(value),
+ "minimal" => markers.minimal = value == "true",
+ "required" => markers.required = value == "true",
_ => {}
}
}
@@ -499,6 +556,7 @@ mod tests {
group: None,
default_value: None,
example: None,
+ minimal: false,
},
),
(
@@ -512,6 +570,7 @@ mod tests {
group: None,
default_value: None,
example: None,
+ minimal: false,
},
),
];
@@ -793,6 +852,7 @@ Please tune this value based on services' document."
group: None,
default_value: None,
example: None,
+ minimal: false,
},
);
assert_eq!(
@@ -808,6 +868,7 @@ For example, Ceph RADOS S3 doesn't support write with if match.".to_string(),
group: None,
default_value: None,
example: None,
+ minimal: false,
},
);
}
@@ -823,7 +884,9 @@ For example, Ceph RADOS S3 doesn't support write with if match.".to_string(),
/// <!-- @group General -->
/// <!-- @default my-bucket -->
/// @example example-bucket
- pub bucket: String
+ /// <!-- @minimal true -->
+ /// <!-- @required true -->
+ pub bucket: Option<String>
}
"#;
let x: ItemStruct = syn::parse_str(input).unwrap();
@@ -835,9 +898,19 @@ For example, Ceph RADOS S3 doesn't support write with if match.".to_string(),
assert_eq!(actual.group.as_deref(), Some("General"));
assert_eq!(actual.default_value.as_deref(), Some("my-bucket"));
assert_eq!(actual.example.as_deref(), Some("example-bucket"));
+ assert!(actual.minimal);
+ assert!(!actual.optional);
assert_eq!(actual.comments, "bucket name of this backend.\n\nrequired.");
}
+ #[test]
+ fn test_service_feature() {
+ assert_eq!(service_feature("s3"), "services-s3");
+ assert_eq!(service_feature("r2"), "services-r2");
+ assert_eq!(service_feature("minio"), "services-minio");
+ assert_eq!(service_feature("mini_moka"), "services-mini-moka");
+ }
+
#[test]
fn test_parse() {
let path = workspace_dir()
@@ -847,5 +920,14 @@ For example, Ceph RADOS S3 doesn't support write with if match.".to_string(),
// Parse should just pass.
let _ = parse(&path.to_string_lossy()).unwrap();
+
+ let path = workspace_dir()
+ .join("core/services")
+ .canonicalize()
+ .unwrap();
+ let services = parse(&path.to_string_lossy()).unwrap();
+ assert!(services.contains_key("s3"));
+ assert!(services.contains_key("r2"));
+ assert!(services.contains_key("minio"));
}
}
diff --git a/dev/src/generate/python.rs b/dev/src/generate/python.rs
index 0763c066a..a47a6f3d2 100644
--- a/dev/src/generate/python.rs
+++ b/dev/src/generate/python.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::generate::parser::{Config, ConfigType, Services, sorted_services};
+use crate::generate::parser::{Config, ConfigType, Services, service_feature, sorted_services};
use anyhow::Result;
use minijinja::value::ViaDeserialize;
use minijinja::{Environment, context};
@@ -38,7 +38,7 @@ pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> {
env.add_template("python", include_str!("python.j2"))?;
env.add_template("python_config", include_str!("python_config.j2"))?;
env.add_function("snake_to_kebab_case", snake_to_kebab_case);
- env.add_function("service_to_feature", service_to_feature);
+ env.add_function("service_to_feature", service_feature);
env.add_function("service_to_pascal", service_to_pascal);
env.add_function("make_python_type", make_python_type);
env.add_function("make_pydoc_param_header", make_pydoc_param_header);
@@ -72,10 +72,6 @@ fn snake_to_kebab_case(str: &str) -> String {
str.replace('_', "-")
}
-fn service_to_feature(service: &str) -> String {
- format!("services-{}", snake_to_kebab_case(service))
-}
-
fn service_to_pascal(service: &str) -> String {
let mut result = String::with_capacity(service.len());
let mut capitalize = true;
diff --git a/justfile b/justfile
index 956577df4..91c8b9c9a 100644
--- a/justfile
+++ b/justfile
@@ -17,7 +17,7 @@
# Generate code for language
#
-# Available languages: python, java
+# Available languages: python, java, docs
generate language:
cargo run --quiet --manifest-path=dev/Cargo.toml -- generate -l {{language}}
diff --git a/website/data/services.json b/website/data/services.json
index f9600f471..c0f9c29c4 100644
--- a/website/data/services.json
+++ b/website/data/services.json
@@ -2769,6 +2769,109 @@
}
]
},
+ {
+ "name": "minio",
+ "scheme": "minio",
+ "groups": [
+ "General",
+ "Credentials"
+ ],
+ "configs": [
+ {
+ "name": "root",
+ "type": "string",
+ "required": false,
+ "group": "General",
+ "default": "/",
+ "comments": "Root within the bucket.\n\nAll operations happen under this root. The default is `/`."
+ },
+ {
+ "name": "bucket",
+ "type": "string",
+ "required": true,
+ "group": "General",
+ "example": "my-bucket",
+ "comments": "Bucket name.\n\nThis field is required."
+ },
+ {
+ "name": "endpoint",
+ "type": "string",
+ "required": true,
+ "group": "General",
+ "example": "http://127.0.0.1:9000",
+ "comments": "MinIO endpoint.\n\nThis field is required because MinIO deployments do not share a\nuniversal endpoint."
+ },
+ {
+ "name": "region",
+ "type": "string",
+ "required": false,
+ "group": "General",
+ "default": "auto",
+ "comments": "Signing region.\n\nThe default is `auto`. Set this field when the deployment requires a\nconfigured region."
+ },
+ {
+ "name": "access_key_id",
+ "type": "string",
+ "required": false,
+ "group": "Credentials",
+ "comments": "Access key ID.\n\nSet this field together with `secret_access_key`."
+ },
+ {
+ "name": "secret_access_key",
+ "type": "string",
+ "required": false,
+ "group": "Credentials",
+ "comments": "Secret access key.\n\nSet this field together with `access_key_id`."
+ },
+ {
+ "name": "session_token",
+ "type": "string",
+ "required": false,
+ "group": "Credentials",
+ "comments": "Session token for temporary credentials.\n\nThis field requires `access_key_id` and `secret_access_key`."
+ },
+ {
+ "name": "skip_signature",
+ "type": "bool",
+ "required": false,
+ "group": "Credentials",
+ "default": "false",
+ "comments": "Send requests without signing them.\n\nThis option cannot be combined with direct credentials."
+ }
+ ],
+ "examples": [
+ {
+ "binding": "rust",
+ "language": "rust",
+ "minimal": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"minio\", [\n (\"bucket\".to_string(), \"my-bucket\".to_string()),\n (\"endpoint\".to_string(), \"http://127.0.0.1:9000\".to_string()),\n])?;",
+ "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"minio\", [\n // --- General ---\n // Root within the bucket.\n //\n // All operations happen under this root. The default is `/`.\n // (\"root\".to_string(), \"/\".to_string()),\n\n // Bucket name.\n //\n // This field is required.\n (\"bucket\".to_string(), \"my-bucket\".to_string()),\n\n // MinIO endpoint.\n //\n // This field is required because MinIO deployments do not [...]
+ },
+ {
+ "binding": "java",
+ "language": "java",
+ "minimal": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap<String, String> config = new HashMap<>();\nconfig.put(\"bucket\", \"my-bucket\");\nconfig.put(\"endpoint\", \"http://127.0.0.1:9000\");\nOperator operator = Operator.of(\"minio\", config);",
+ "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap<String, String> config = new HashMap<>();\n// --- General ---\n// Root within the bucket.\n//\n// All operations happen under this root. The default is `/`.\n// config.put(\"root\", \"/\");\n\n// Bucket name.\n//\n// This field is required.\nconfig.put(\"bucket\", \"my-bucket\");\n\n// MinIO endpoint.\n//\n// This field is required because MinIO deployments do not share a\n// u [...]
+ },
+ {
+ "binding": "python",
+ "language": "python",
+ "minimal": "import opendal\n\noperator = opendal.Operator(\n \"minio\",\n bucket=\"my-bucket\",\n endpoint=\"http://127.0.0.1:9000\",\n)",
+ "full": "import opendal\n\noperator = opendal.Operator(\n \"minio\",\n # --- General ---\n # Root within the bucket.\n #\n # All operations happen under this root. The default is `/`.\n # root=\"/\",\n\n # Bucket name.\n #\n # This field is required.\n bucket=\"my-bucket\",\n\n # MinIO endpoint.\n #\n # This field is required because MinIO deployments do not share a\n # universal endpoint.\n endpoint=\"http://127.0.0.1:9000\",\n\n [...]
+ },
+ {
+ "binding": "nodejs",
+ "language": "javascript",
+ "minimal": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"minio\", {\n bucket: \"my-bucket\",\n endpoint: \"http://127.0.0.1:9000\",\n});",
+ "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"minio\", {\n // --- General ---\n // Root within the bucket.\n //\n // All operations happen under this root. The default is `/`.\n // root: \"/\",\n\n // Bucket name.\n //\n // This field is required.\n bucket: \"my-bucket\",\n\n // MinIO endpoint.\n //\n // This field is required because MinIO deployments do not share a\n // universal endpoint.\n endpoint: \"http://127.0.0.1:9000\" [...]
+ },
+ {
+ "binding": "ruby",
+ "language": "ruby",
+ "minimal": "require \"opendal\"\n\noperator = OpenDal::Operator.new(\"minio\", {\n \"bucket\" => \"my-bucket\",\n \"endpoint\" => \"http://127.0.0.1:9000\",\n})",
+ "full": "require \"opendal\"\n\noperator = OpenDal::Operator.new(\"minio\", {\n # --- General ---\n # Root within the bucket.\n #\n # All operations happen under this root. The default is `/`.\n # \"root\" => \"/\",\n\n # Bucket name.\n #\n # This field is required.\n \"bucket\" => \"my-bucket\",\n\n # MinIO endpoint.\n #\n # This field is required because MinIO deployments do not share a\n # universal endpoint.\n \"endpoint\" => \"http://127.0.0.1:9000\",\n\n # [...]
+ }
+ ]
+ },
{
"name": "moka",
"scheme": "moka",
@@ -3674,6 +3777,108 @@
}
]
},
+ {
+ "name": "r2",
+ "scheme": "r2",
+ "groups": [
+ "General",
+ "Credentials"
+ ],
+ "configs": [
+ {
+ "name": "root",
+ "type": "string",
+ "required": false,
+ "group": "General",
+ "default": "/",
+ "comments": "Root within the bucket.\n\nAll operations happen under this root. The default is `/`."
+ },
+ {
+ "name": "bucket",
+ "type": "string",
+ "required": true,
+ "group": "General",
+ "example": "my-bucket",
+ "comments": "Bucket name.\n\nThis field is required."
+ },
+ {
+ "name": "account_id",
+ "type": "string",
+ "required": false,
+ "group": "General",
+ "example": "example-account",
+ "comments": "Cloudflare account ID used to derive the R2 endpoint.\n\nSet exactly one of `account_id` and `endpoint`."
+ },
+ {
+ "name": "jurisdiction",
+ "type": "string",
+ "required": false,
+ "group": "General",
+ "comments": "R2 jurisdiction.\n\nSupported values are `eu` and `fedramp`. This field requires\n`account_id` and cannot be used with `endpoint`."
+ },
+ {
+ "name": "endpoint",
+ "type": "string",
+ "required": false,
+ "group": "General",
+ "example": "https://example.r2.cloudflarestorage.com",
+ "comments": "Explicit R2-compatible endpoint.\n\nUse this field for a proxy, gateway, or test server. Set exactly one of\n`endpoint` and `account_id`."
+ },
+ {
+ "name": "access_key_id",
+ "type": "string",
+ "required": false,
+ "group": "Credentials",
+ "comments": "Access key ID.\n\nSet this field together with `secret_access_key`."
+ },
+ {
+ "name": "secret_access_key",
+ "type": "string",
+ "required": false,
+ "group": "Credentials",
+ "comments": "Secret access key.\n\nSet this field together with `access_key_id`."
+ },
+ {
+ "name": "session_token",
+ "type": "string",
+ "required": false,
+ "group": "Credentials",
+ "comments": "Session token for temporary credentials.\n\nThis field requires `access_key_id` and `secret_access_key`."
+ }
+ ],
+ "examples": [
+ {
+ "binding": "rust",
+ "language": "rust",
+ "minimal": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"r2\", [\n (\"bucket\".to_string(), \"my-bucket\".to_string()),\n (\"account_id\".to_string(), \"example-account\".to_string()),\n])?;",
+ "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"r2\", [\n // --- General ---\n // Root within the bucket.\n //\n // All operations happen under this root. The default is `/`.\n // (\"root\".to_string(), \"/\".to_string()),\n\n // Bucket name.\n //\n // This field is required.\n (\"bucket\".to_string(), \"my-bucket\".to_string()),\n\n // Cloudflare account ID used to derive the R2 endpoint.\n //\n // Set exactly one of ` [...]
+ },
+ {
+ "binding": "java",
+ "language": "java",
+ "minimal": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap<String, String> config = new HashMap<>();\nconfig.put(\"bucket\", \"my-bucket\");\nconfig.put(\"account_id\", \"example-account\");\nOperator operator = Operator.of(\"r2\", config);",
+ "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap<String, String> config = new HashMap<>();\n// --- General ---\n// Root within the bucket.\n//\n// All operations happen under this root. The default is `/`.\n// config.put(\"root\", \"/\");\n\n// Bucket name.\n//\n// This field is required.\nconfig.put(\"bucket\", \"my-bucket\");\n\n// Cloudflare account ID used to derive the R2 endpoint.\n//\n// Set exactly one of `account_id` [...]
+ },
+ {
+ "binding": "python",
+ "language": "python",
+ "minimal": "import opendal\n\noperator = opendal.Operator(\n \"r2\",\n bucket=\"my-bucket\",\n account_id=\"example-account\",\n)",
+ "full": "import opendal\n\noperator = opendal.Operator(\n \"r2\",\n # --- General ---\n # Root within the bucket.\n #\n # All operations happen under this root. The default is `/`.\n # root=\"/\",\n\n # Bucket name.\n #\n # This field is required.\n bucket=\"my-bucket\",\n\n # Cloudflare account ID used to derive the R2 endpoint.\n #\n # Set exactly one of `account_id` and `endpoint`.\n account_id=\"example-account\",\n\n # R2 jurisd [...]
+ },
+ {
+ "binding": "nodejs",
+ "language": "javascript",
+ "minimal": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"r2\", {\n bucket: \"my-bucket\",\n account_id: \"example-account\",\n});",
+ "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"r2\", {\n // --- General ---\n // Root within the bucket.\n //\n // All operations happen under this root. The default is `/`.\n // root: \"/\",\n\n // Bucket name.\n //\n // This field is required.\n bucket: \"my-bucket\",\n\n // Cloudflare account ID used to derive the R2 endpoint.\n //\n // Set exactly one of `account_id` and `endpoint`.\n account_id: \"example-account\",\n\n // R [...]
+ },
+ {
+ "binding": "ruby",
+ "language": "ruby",
+ "minimal": "require \"opendal\"\n\noperator = OpenDal::Operator.new(\"r2\", {\n \"bucket\" => \"my-bucket\",\n \"account_id\" => \"example-account\",\n})",
+ "full": "require \"opendal\"\n\noperator = OpenDal::Operator.new(\"r2\", {\n # --- General ---\n # Root within the bucket.\n #\n # All operations happen under this root. The default is `/`.\n # \"root\" => \"/\",\n\n # Bucket name.\n #\n # This field is required.\n \"bucket\" => \"my-bucket\",\n\n # Cloudflare account ID used to derive the R2 endpoint.\n #\n # Set exactly one of `account_id` and `endpoint`.\n \"account_id\" => \"example-account\",\n\n # R2 jurisdi [...]
+ }
+ ]
+ },
{
"name": "redb",
"scheme": "redb",