You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/02/27 03:35:31 UTC
(opendal) branch codegen created (now 36f375858b)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a change to branch codegen
in repository https://gitbox.apache.org/repos/asf/opendal.git
at 36f375858b feat: Codegen for rust config
This branch includes the following new commits:
new 36f375858b feat: Codegen for rust config
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opendal) 01/01: feat: Codegen for rust config
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch codegen
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 36f375858bd81da3bfb147dda6dd18832734738a
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Feb 27 11:35:12 2024 +0800
feat: Codegen for rust config
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
.taplo.toml | 5 +
core/src/raw/mod.rs | 3 +
.../{services/s3/mod.rs => raw/service_util.rs} | 21 +-
core/src/services/s3/backend.rs | 156 +--------------
core/src/services/s3/generated.rs | 222 +++++++++++++++++++++
core/src/services/s3/mod.rs | 3 +-
scripts/specs.py | 183 +++++++++++++++++
specs/README.md | 9 +
specs/s3.toml | 212 ++++++++++++++++++++
specs/schema.json | 68 +++++++
10 files changed, 719 insertions(+), 163 deletions(-)
diff --git a/.taplo.toml b/.taplo.toml
index 0a6b2a1376..e54a090680 100644
--- a/.taplo.toml
+++ b/.taplo.toml
@@ -45,3 +45,8 @@ reorder_keys = true
allowed_blank_lines = 1
# Use CRLF for line endings.
crlf = false
+
+[[rule]]
+include = ["specs/*.toml"]
+[rule.schema]
+path = "specs/schema.json"
diff --git a/core/src/raw/mod.rs b/core/src/raw/mod.rs
index 2783dddf9a..6e120c8adc 100644
--- a/core/src/raw/mod.rs
+++ b/core/src/raw/mod.rs
@@ -76,6 +76,9 @@ pub use futures_util::ConcurrentFutures;
mod enum_utils;
pub use enum_utils::*;
+mod service_util;
+pub use service_util::*;
+
// Expose as a pub mod to avoid confusing.
pub mod adapters;
pub mod oio;
diff --git a/core/src/services/s3/mod.rs b/core/src/raw/service_util.rs
similarity index 62%
copy from core/src/services/s3/mod.rs
copy to core/src/raw/service_util.rs
index 4e2f283cb2..60cefa9b6d 100644
--- a/core/src/services/s3/mod.rs
+++ b/core/src/raw/service_util.rs
@@ -15,11 +15,18 @@
// specific language governing permissions and limitations
// under the License.
-mod backend;
-pub use backend::S3Builder as S3;
-pub use backend::S3Config;
+/// A helper function to mask secret string.
+pub fn mask_secret(s: &str) -> String {
+ // Always use 3 stars to mask the secret if length is low.
+ //
+ // # NOTE
+ //
+ // It's by design to use 10 instead of 6. Attackers could brute force the secrets
+ // if the length is too short.
+ if s.len() <= 10 {
+ return "***".to_string();
+ }
-mod core;
-mod error;
-mod lister;
-mod writer;
+ // Keep the first & end three chars visible for easier debugging.
+ format!("{}***{}", &s[..3], &s[s.len() - 3..])
+}
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index bb7b3949c3..dc29d7a91e 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -42,6 +42,7 @@ use serde::Deserialize;
use super::core::*;
use super::error::parse_error;
use super::error::parse_s3_error_code;
+use super::generated::S3Config;
use super::lister::S3Lister;
use super::writer::S3Writer;
use super::writer::S3Writers;
@@ -61,161 +62,6 @@ static ENDPOINT_TEMPLATES: Lazy<HashMap<&'static str, &'static str>> = Lazy::new
const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000;
-/// Config for Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support.
-#[derive(Default, Deserialize)]
-#[serde(default)]
-#[non_exhaustive]
-pub struct S3Config {
- /// root of this backend.
- ///
- /// All operations will happen under this root.
- ///
- /// default to `/` if not set.
- pub root: Option<String>,
- /// bucket name of this backend.
- ///
- /// required.
- pub bucket: String,
- /// endpoint of this backend.
- ///
- /// Endpoint must be full uri, e.g.
- ///
- /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com`
- /// - Cloudflare R2: `https://<ACCOUNT_ID>.r2.cloudflarestorage.com`
- /// - Aliyun OSS: `https://{region}.aliyuncs.com`
- /// - Tencent COS: `https://cos.{region}.myqcloud.com`
- /// - Minio: `http://127.0.0.1:9000`
- ///
- /// If user inputs endpoint without scheme like "s3.amazonaws.com", we
- /// will prepend "https://" before it.
- ///
- /// default to `https://s3.amazonaws.com` if not set.
- pub endpoint: Option<String>,
- /// Region represent the signing region of this endpoint. This is required
- /// if you are using the default AWS S3 endpoint.
- ///
- /// If using a custom endpoint,
- /// - If region is set, we will take user's input first.
- /// - If not, we will try to load it from environment.
- pub region: Option<String>,
-
- /// access_key_id of this backend.
- ///
- /// - If access_key_id is set, we will take user's input first.
- /// - If not, we will try to load it from environment.
- pub access_key_id: Option<String>,
- /// secret_access_key of this backend.
- ///
- /// - If secret_access_key is set, we will take user's input first.
- /// - If not, we will try to load it from environment.
- pub secret_access_key: Option<String>,
- /// security_token (aka, session token) of this backend.
- ///
- /// This token will expire after sometime, it's recommended to set security_token
- /// by hand.
- pub security_token: Option<String>,
- /// role_arn for this backend.
- ///
- /// If `role_arn` is set, we will use already known config as source
- /// credential to assume role with `role_arn`.
- pub role_arn: Option<String>,
- /// external_id for this backend.
- pub external_id: Option<String>,
- /// Disable config load so that opendal will not load config from
- /// environment.
- ///
- /// For examples:
- ///
- /// - envs like `AWS_ACCESS_KEY_ID`
- /// - files like `~/.aws/config`
- pub disable_config_load: bool,
- /// Disable load credential from ec2 metadata.
- ///
- /// This option is used to disable the default behavior of opendal
- /// to load credential from ec2 metadata, a.k.a, IMDSv2
- pub disable_ec2_metadata: bool,
- /// Allow anonymous will allow opendal to send request without signing
- /// when credential is not loaded.
- pub allow_anonymous: bool,
- /// server_side_encryption for this backend.
- ///
- /// Available values: `AES256`, `aws:kms`.
- pub server_side_encryption: Option<String>,
- /// server_side_encryption_aws_kms_key_id for this backend
- ///
- /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
- /// is not set, S3 will use aws managed kms key to encrypt data.
- /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
- /// is a valid kms key id, S3 will use the provided kms key to encrypt data.
- /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be
- /// returned.
- /// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id`
- /// is a noop.
- pub server_side_encryption_aws_kms_key_id: Option<String>,
- /// server_side_encryption_customer_algorithm for this backend.
- ///
- /// Available values: `AES256`.
- pub server_side_encryption_customer_algorithm: Option<String>,
- /// server_side_encryption_customer_key for this backend.
- ///
- /// # Value
- ///
- /// base64 encoded key that matches algorithm specified in
- /// `server_side_encryption_customer_algorithm`.
- pub server_side_encryption_customer_key: Option<String>,
- /// Set server_side_encryption_customer_key_md5 for this backend.
- ///
- /// # Value
- ///
- /// MD5 digest of key specified in `server_side_encryption_customer_key`.
- pub server_side_encryption_customer_key_md5: Option<String>,
- /// default storage_class for this backend.
- ///
- /// Available values:
- /// - `DEEP_ARCHIVE`
- /// - `GLACIER`
- /// - `GLACIER_IR`
- /// - `INTELLIGENT_TIERING`
- /// - `ONEZONE_IA`
- /// - `OUTPOSTS`
- /// - `REDUCED_REDUNDANCY`
- /// - `STANDARD`
- /// - `STANDARD_IA`
- ///
- /// S3 compatible services don't support all of them
- pub default_storage_class: Option<String>,
- /// Enable virtual host style so that opendal will send API requests
- /// in virtual host style instead of path style.
- ///
- /// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name`
- /// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com`
- pub enable_virtual_host_style: bool,
- /// Set maximum batch operations of this backend.
- ///
- /// Some compatible services have a limit on the number of operations in a batch request.
- /// For example, R2 could return `Internal Error` while batch delete 1000 files.
- ///
- /// Please tune this value based on services' document.
- pub batch_max_operations: Option<usize>,
- /// Disable stat with override so that opendal will not send stat request with override queries.
- ///
- /// For example, R2 doesn't support stat with `response_content_type` query.
- pub disable_stat_with_override: bool,
-}
-
-impl Debug for S3Config {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- let mut d = f.debug_struct("S3Config");
-
- d.field("root", &self.root)
- .field("bucket", &self.bucket)
- .field("endpoint", &self.endpoint)
- .field("region", &self.region);
-
- d.finish_non_exhaustive()
- }
-}
-
/// Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support.
/// For more information about s3-compatible services, refer to [Compatible Services](#compatible-services).
#[doc = include_str!("docs.md")]
diff --git a/core/src/services/s3/generated.rs b/core/src/services/s3/generated.rs
new file mode 100644
index 0000000000..03fd620d9e
--- /dev/null
+++ b/core/src/services/s3/generated.rs
@@ -0,0 +1,222 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use serde::Deserialize;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+/// Config for Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct S3Config {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+
+ /// bucket name of this backend.
+ pub bucket: String,
+
+ /// endpoint of this backend.
+ ///
+ /// Endpoint must be full uri, e.g.
+ ///
+ /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com`
+ /// - Cloudflare R2: `https://<ACCOUNT_ID>.r2.cloudflarestorage.com`
+ /// - Aliyun OSS: `https://{region}.aliyuncs.com`
+ /// - Tencent COS: `https://cos.{region}.myqcloud.com`
+ /// - Minio: `http://127.0.0.1:9000`
+ ///
+ /// If user inputs endpoint without scheme like "s3.amazonaws.com", we will prepend "https://" before it.
+ pub endpoint: Option<String>,
+
+ /// Region represent the signing region of this endpoint.
+ /// This is required if you are using the default AWS S3 endpoint.
+ ///
+ /// If using a custom endpoint,
+ /// - If region is set, we will take user's input first.
+ /// - If not, we will try to load it from environment.
+ pub region: Option<String>,
+
+ /// access_key_id of this backend.
+ pub access_key_id: Option<String>,
+
+ /// secret_access_key of this backend.
+ pub secret_access_key: Option<String>,
+
+ /// security_token (aka, session token) of this backend.
+ ///
+ /// This token will expire after sometime, it's recommended not to set security_token by hand.
+ pub security_token: Option<String>,
+
+ /// role_arn of this backend.
+ ///
+ /// If `role_arn` is set, we will use already known config as source to assume role.
+ pub role_arn: Option<String>,
+
+ /// external_id of this backend.
+ ///
+ /// external_id is a unique identifier that is used by third parties to assume a role in their customers' accounts.
+ pub external_id: Option<String>,
+
+ /// Use this flag to make sure opendal don't load config from the environment.
+ ///
+ /// - envs like `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+ /// - files like `~/.aws/config`
+ pub disable_config_load: bool,
+
+ /// Use this flag to make sure opendal don't load credentials from the EC2 instance metadata service.
+ pub disable_ec2_metadata: bool,
+
+ /// Use this flag to allow anonymous access to the bucket.
+ ///
+ /// OpenDAL returns an permission error if credentials are not loaded by default. This flag allows users to send requests without signing.
+ pub allow_anonymous: bool,
+
+ /// server_side_encryption for this backend.
+ pub server_side_encryption: Option<String>,
+
+ /// server_side_encryption_aws_kms_key_id for this backend
+ ///
+ /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` is not set, S3 will use aws managed kms key to encrypt data.
+ /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` is a valid kms key id, S3 will use the provided kms key to encrypt data.
+ /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be returned.
+ /// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id` is a noop.
+ pub server_side_encryption_aws_kms_key_id: Option<String>,
+
+ /// server_side_encryption_customer_algorithm for this backend.
+ pub server_side_encryption_customer_algorithm: Option<String>,
+
+ /// server_side_encryption_customer_key for this backend.
+ ///
+ /// # Value
+ ///
+ /// base64 encoded key that matches algorithm specified in `server_side_encryption_customer_algorithm`.
+ pub server_side_encryption_customer_key: Option<String>,
+
+ /// server_side_encryption_customer_key_md5 for this backend.
+ ///
+ /// # Value
+ ///
+ /// MD5 digest of key specified in `server_side_encryption_customer_key`.
+ pub server_side_encryption_customer_key_md5: Option<String>,
+
+ /// default storage class for this backend.
+ pub default_storage_class: Option<String>,
+
+ /// Enable virtual host style so that opendal will send API requests in virtual host style instead of path style.
+ ///
+ /// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name`
+ /// - After enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com`
+ ///
+ /// Some s3 compatible services like `oss` require virtual host style.
+ pub enable_virtual_host_style: bool,
+
+ /// Maximum batch operations of this backend.
+ ///
+ /// Some compatible services have a limit on the number of operations in a batch request.
+ /// For example, R2 could return `Internal Error` while batch delete 1000 files.
+ ///
+ /// Please tune this value based on services' document.
+ pub batch_max_operations: Option<usize>,
+
+ /// Disable stat with override so that opendal will not send stat request with override queries.
+ ///
+ /// For example, R2 doesn't support stat with `response_content_type` query.
+ pub disable_stat_with_override: bool,
+}
+impl Debug for S3Config {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut d = f.debug_struct("S3Config");
+
+ d.field("root", &self.root);
+
+ d.field("bucket", &self.bucket);
+
+ d.field("endpoint", &self.endpoint);
+
+ d.field("region", &self.region);
+
+ d.field(
+ "access_key_id",
+ &self.access_key_id.as_deref().map(mask_secret),
+ );
+
+ d.field(
+ "secret_access_key",
+ &self.secret_access_key.as_deref().map(mask_secret),
+ );
+
+ d.field(
+ "security_token",
+ &self.security_token.as_deref().map(mask_secret),
+ );
+
+ d.field("role_arn", &self.role_arn);
+
+ d.field("external_id", &self.external_id);
+
+ d.field("disable_config_load", &self.disable_config_load);
+
+ d.field("disable_ec2_metadata", &self.disable_ec2_metadata);
+
+ d.field("allow_anonymous", &self.allow_anonymous);
+
+ d.field("server_side_encryption", &self.server_side_encryption);
+
+ d.field(
+ "server_side_encryption_aws_kms_key_id",
+ &self.server_side_encryption_aws_kms_key_id,
+ );
+
+ d.field(
+ "server_side_encryption_customer_algorithm",
+ &self.server_side_encryption_customer_algorithm,
+ );
+
+ d.field(
+ "server_side_encryption_customer_key",
+ &self
+ .server_side_encryption_customer_key
+ .as_deref()
+ .map(mask_secret),
+ );
+
+ d.field(
+ "server_side_encryption_customer_key_md5",
+ &self
+ .server_side_encryption_customer_key_md5
+ .as_deref()
+ .map(mask_secret),
+ );
+
+ d.field("default_storage_class", &self.default_storage_class);
+
+ d.field("enable_virtual_host_style", &self.enable_virtual_host_style);
+
+ d.field("batch_max_operations", &self.batch_max_operations);
+
+ d.field(
+ "disable_stat_with_override",
+ &self.disable_stat_with_override,
+ );
+
+ d.finish()
+ }
+}
diff --git a/core/src/services/s3/mod.rs b/core/src/services/s3/mod.rs
index 4e2f283cb2..7b1ee6361a 100644
--- a/core/src/services/s3/mod.rs
+++ b/core/src/services/s3/mod.rs
@@ -17,9 +17,10 @@
mod backend;
pub use backend::S3Builder as S3;
-pub use backend::S3Config;
+pub use generated::S3Config;
mod core;
mod error;
+mod generated;
mod lister;
mod writer;
diff --git a/scripts/specs.py b/scripts/specs.py
new file mode 100755
index 0000000000..2b06e1ad92
--- /dev/null
+++ b/scripts/specs.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Mapping
+import tomllib
+import subprocess
+
+ROOT_DIR = Path(__file__).parent.parent
+
+
+@dataclass
+class Config:
+ description: str
+ type: str
+ required: bool = False
+ sensitive: bool = False
+ default: str = ""
+ available: List[str] = field(default_factory=list)
+ example: List[str] = field(default_factory=list)
+
+ @property
+ def rust_type(self):
+ type_mapping = {
+ "string": "String",
+ "usize": "usize",
+ "bool": "bool",
+ }
+
+ try:
+ value_type = type_mapping[self.type]
+ except KeyError:
+ raise ValueError(f"Not supported type {self.type}, must be bug")
+
+ if self.required:
+ return value_type
+ # Always return bool without Option
+ elif value_type == "bool":
+ return f"bool"
+ else:
+ return f"Option<{value_type}>"
+
+
+@dataclass
+class Spec:
+ # name is parsed from the spec file name.
+ name: str
+ description: str
+ config: Mapping[str, Config]
+
+ # Convert the name from snake_case to CamelCase
+ @property
+ def camel_case_name(self):
+ return to_camel_case(self.name)
+
+
+def list_specs() -> List[Spec]:
+ specs = []
+ for path in (ROOT_DIR / "specs").glob("*.toml"):
+ with open(path, "rb") as f:
+ data = tomllib.load(f)
+
+ spec = Spec(
+ # Parse and set the spec name
+ name=path.stem,
+ description=data["description"],
+ config={
+ item: Config(**data["config"][item]) for item in data["config"]
+ },
+ )
+ specs.append(spec)
+ return specs
+
+
+# Convert from snake_case to CamelCase
+def to_camel_case(name: str) -> str:
+ return name.replace("_", " ").title().replace(" ", "")
+
+
+# Convert from plain string to rust comments
+def to_comments(v: str) -> str:
+ return "\n".join(f"/// {line}" for line in v.strip().splitlines())
+
+
+def generate_rust(spec: Spec):
+ # Header
+ content = """
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use serde::Deserialize;
+use crate::raw::*;
+"""
+
+ # Config struct Start
+ content += f"""
+/// Config for {spec.description.strip()}
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct {spec.camel_case_name}Config {{
+"""
+
+ # Config fields
+ for name, cfg in spec.config.items():
+ content += f"""
+{to_comments(cfg.description)}
+pub {name}: {cfg.rust_type},
+"""
+
+ # Config struct End
+ content += "}"
+
+ # Impl Debug for Config Start
+ content += f"""
+impl Debug for {spec.camel_case_name}Config {{
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {{
+ let mut d = f.debug_struct("{spec.camel_case_name}Config");
+"""
+
+ # Config fields
+ for name, cfg in spec.config.items():
+ if cfg.sensitive:
+ content += f"""
+d.field("{name}", &self.{name}.as_deref().map(mask_secret));
+"""
+ else:
+ content += f"""
+d.field("{name}", &self.{name});
+"""
+
+ # Impl Debug for Config End
+ content += f"""
+ d.finish()
+}}
+}}
+"""
+
+ return content
+
+
+if __name__ == "__main__":
+ for spec in list_specs():
+ content = generate_rust(spec)
+ with open(ROOT_DIR / "core/src/services" / spec.name / "generated.rs", "w") as f:
+ f.write(content)
+ subprocess.run(
+ ["cargo", "fmt"],
+ cwd=ROOT_DIR / "core",
+ check=True,
+ )
diff --git a/specs/README.md b/specs/README.md
new file mode 100644
index 0000000000..867877f4c8
--- /dev/null
+++ b/specs/README.md
@@ -0,0 +1,9 @@
+# OpenDAL Services Specs
+
+`specs` is the single truth of the opendal service's specification, which is used to generate the service's code and documentation.
+
+## Validate
+
+```shell
+taplo check specs/*.toml
+```
diff --git a/specs/s3.toml b/specs/s3.toml
new file mode 100644
index 0000000000..e221736fe1
--- /dev/null
+++ b/specs/s3.toml
@@ -0,0 +1,212 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+description = """
+Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support.
+"""
+
+[config.root]
+default = "/"
+description = """
+root of this backend.
+
+All operations will happen under this root.
+"""
+required = false
+type = "string"
+
+[config.bucket]
+description = """
+bucket name of this backend.
+"""
+required = true
+type = "string"
+
+[config.endpoint]
+default = "https://s3.amazonaws.com"
+description = """
+endpoint of this backend.
+
+Endpoint must be full uri, e.g.
+
+- AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com`
+- Cloudflare R2: `https://<ACCOUNT_ID>.r2.cloudflarestorage.com`
+- Aliyun OSS: `https://{region}.aliyuncs.com`
+- Tencent COS: `https://cos.{region}.myqcloud.com`
+- Minio: `http://127.0.0.1:9000`
+
+If user inputs endpoint without scheme like "s3.amazonaws.com", we will prepend "https://" before it.
+"""
+type = "string"
+
+[config.region]
+description = """
+Region represent the signing region of this endpoint.
+This is required if you are using the default AWS S3 endpoint.
+
+If using a custom endpoint,
+- If region is set, we will take user's input first.
+- If not, we will try to load it from environment.
+"""
+type = "string"
+
+[config.access_key_id]
+description = """
+access_key_id of this backend.
+"""
+sensitive = true
+type = "string"
+
+[config.secret_access_key]
+description = """
+secret_access_key of this backend.
+"""
+sensitive = true
+type = "string"
+
+[config.security_token]
+description = """
+security_token (aka, session token) of this backend.
+
+This token will expire after sometime, it's recommended not to set security_token by hand.
+"""
+sensitive = true
+type = "string"
+
+[config.role_arn]
+description = """
+role_arn of this backend.
+
+If `role_arn` is set, we will use already known config as source to assume role.
+"""
+type = "string"
+
+[config.external_id]
+description = """
+external_id of this backend.
+
+external_id is a unique identifier that is used by third parties to assume a role in their customers' accounts.
+"""
+type = "string"
+
+[config.disable_config_load]
+description = """
+Use this flag to make sure opendal don't load config from the environment.
+
+- envs like `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+- files like `~/.aws/config`
+"""
+type = "bool"
+
+[config.disable_ec2_metadata]
+description = """
+Use this flag to make sure opendal don't load credentials from the EC2 instance metadata service.
+"""
+type = "bool"
+
+[config.allow_anonymous]
+description = """
+Use this flag to allow anonymous access to the bucket.
+
+OpenDAL returns an permission error if credentials are not loaded by default. This flag allows users to send requests without signing.
+"""
+type = "bool"
+
+[config.server_side_encryption]
+available = ["AES256", "aws:kms"]
+description = """
+server_side_encryption for this backend.
+"""
+type = "string"
+
+[config.server_side_encryption_aws_kms_key_id]
+description = """
+server_side_encryption_aws_kms_key_id for this backend
+
+- If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` is not set, S3 will use aws managed kms key to encrypt data.
+- If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` is a valid kms key id, S3 will use the provided kms key to encrypt data.
+- If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be returned.
+- If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id` is a noop.
+"""
+type = "string"
+
+[config.server_side_encryption_customer_algorithm]
+available = ["AES256"]
+description = """
+server_side_encryption_customer_algorithm for this backend.
+"""
+type = "string"
+
+[config.server_side_encryption_customer_key]
+description = """
+server_side_encryption_customer_key for this backend.
+
+# Value
+
+base64 encoded key that matches algorithm specified in `server_side_encryption_customer_algorithm`.
+"""
+sensitive = true
+type = "string"
+
+[config.server_side_encryption_customer_key_md5]
+description = """
+server_side_encryption_customer_key_md5 for this backend.
+
+# Value
+
+MD5 digest of key specified in `server_side_encryption_customer_key`.
+"""
+sensitive = true
+type = "string"
+
+[config.default_storage_class]
+description = """
+default storage class for this backend.
+"""
+example = ["STANDARD", "STANDARD_IA"]
+type = "string"
+
+[config.enable_virtual_host_style]
+description = """
+Enable virtual host style so that opendal will send API requests in virtual host style instead of path style.
+
+- By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name`
+- After enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com`
+
+Some s3 compatible services like `oss` require virtual host style.
+"""
+type = "bool"
+
+[config.batch_max_operations]
+description = """
+Maximum batch operations of this backend.
+
+Some compatible services have a limit on the number of operations in a batch request.
+For example, R2 could return `Internal Error` while batch delete 1000 files.
+
+Please tune this value based on services' document.
+"""
+type = "usize"
+
+[config.disable_stat_with_override]
+description = """
+Disable stat with override so that opendal will not send stat request with override queries.
+
+For example, R2 doesn't support stat with `response_content_type` query.
+"""
+type = "bool"
diff --git a/specs/schema.json b/specs/schema.json
new file mode 100644
index 0000000000..f2221d2f39
--- /dev/null
+++ b/specs/schema.json
@@ -0,0 +1,68 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "description": "A Schema for opendal service",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "description": {
+ "type": "string",
+ "description": "A description of the service"
+ },
+ "config": {
+ "type": "object",
+ "description": "The config entries for the service",
+ "additionalProperties": {
+ "$ref": "#/definitions/Config"
+ }
+ }
+ },
+ "definitions": {
+ "Config": {
+ "type": "object",
+ "title": "Config",
+ "description": "Config Entry for the service",
+ "additionalProperties": false,
+ "properties": {
+ "description": {
+ "type": "string",
+ "description": "A description of the config entry"
+ },
+ "type": {
+ "type": "string",
+ "description": "The value type of the config entry",
+ "enum": [
+ "string",
+ "usize",
+ "bool"
+ ]
+ },
+ "required": {
+ "type": "boolean",
+ "description": "Whether the config entry is required"
+ },
+ "sensitive": {
+ "type": "boolean",
+ "description": "Whether the config entry is sensitive"
+ },
+ "default": {
+ "type": "string",
+ "description": "The default value of the config entry"
+ },
+ "available": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "available values of the config entry"
+ },
+ "example": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "example values of the config entry"
+ }
+ }
+ }
+ }
+}