You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by er...@apache.org on 2026/08/04 17:32:00 UTC
(opendal) branch main updated: fix(bindings/python): annotate `**kwargs` types in generated stubs (#8012)
This is an automated email from the ASF dual-hosted git repository.
erickguan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new a88bc8480 fix(bindings/python): annotate `**kwargs` types in generated stubs (#8012)
a88bc8480 is described below
commit a88bc848010959e97c708a6bba7bd1b01f0615ac
Author: Trim21 <i...@trim21.me>
AuthorDate: Wed Aug 5 01:31:53 2026 +0800
fix(bindings/python): annotate `**kwargs` types in generated stubs (#8012)
---
bindings/python/python/opendal/_opendal.pyi | 15 +-
bindings/python/python/opendal/operator.pyi | 17 +-
bindings/python/python/opendal/options.py | 168 ++++++++++++++++++++
bindings/python/scripts/postprocess_stubs.py | 2 +
bindings/python/src/operator.rs | 12 +-
bindings/python/src/options.rs | 47 ++++++
dev/src/generate/mod.rs | 1 +
dev/src/generate/options.rs | 229 +++++++++++++++++++++++++++
dev/src/generate/python.rs | 151 ++++++++++++++++++
9 files changed, 625 insertions(+), 17 deletions(-)
diff --git a/bindings/python/python/opendal/_opendal.pyi b/bindings/python/python/opendal/_opendal.pyi
index 76695653e..de16b6ad6 100644
--- a/bindings/python/python/opendal/_opendal.pyi
+++ b/bindings/python/python/opendal/_opendal.pyi
@@ -24,19 +24,24 @@ from .operator import AsyncOperator, Operator
__version__: Final[str]
@final
-class DeleteOptions: ...
+class DeleteOptions:
+ """Options for `delete` operations."""
@final
-class ListOptions: ...
+class ListOptions:
+ """Options for `list` operations."""
@final
-class ReadOptions: ...
+class ReadOptions:
+ """Options for `read` operations."""
@final
-class StatOptions: ...
+class StatOptions:
+ """Options for `stat` operations."""
@final
-class WriteOptions: ...
+class WriteOptions:
+ """Options for `write` operations."""
def _reconstruct_async_operator(scheme: str, map: dict[str, str]) -> AsyncOperator:
"""
diff --git a/bindings/python/python/opendal/operator.pyi b/bindings/python/python/opendal/operator.pyi
index a2ad4402a..7062a21e7 100644
--- a/bindings/python/python/opendal/operator.pyi
+++ b/bindings/python/python/opendal/operator.pyi
@@ -20,10 +20,13 @@ from datetime import datetime
from os import PathLike
from typing import Any, final
+from typing_extensions import Unpack
+
from .capability import Capability
from .config import ServiceConfig
from .file import AsyncFile, File
from .layers import Layer
+from .options import OpenKwargs
from .services import Scheme
from .types import Entry, Metadata, PresignedRequest
@@ -39,7 +42,7 @@ class AsyncOperator:
Operator
"""
- def __new__(cls, /, scheme: str | Scheme, **kwargs) -> AsyncOperator:
+ def __new__(cls, /, scheme: str | Scheme, **kwargs: str) -> AsyncOperator:
"""
Create a new `AsyncOperator`.
@@ -195,7 +198,7 @@ class AsyncOperator:
```
"""
@classmethod
- def from_uri(cls, /, uri: str, **kwargs) -> AsyncOperator:
+ def from_uri(cls, /, uri: str, **kwargs: str) -> AsyncOperator:
"""
Create a new `AsyncOperator` from a URI string.
@@ -279,7 +282,7 @@ class AsyncOperator:
An awaitable that returns an async iterator over the entries.
"""
def open(
- self, /, path: str | PathLike[str], mode: str, **kwargs
+ self, /, path: str | PathLike[str], mode: str, **kwargs: Unpack[OpenKwargs]
) -> collections.abc.Awaitable[AsyncFile]:
"""
Open an async file-like object for the given path.
@@ -716,7 +719,7 @@ class Operator:
AsyncOperator
"""
- def __new__(cls, /, scheme: str | Scheme, **kwargs) -> Operator:
+ def __new__(cls, /, scheme: str | Scheme, **kwargs: str) -> Operator:
"""
Create a new blocking `Operator`.
@@ -848,7 +851,7 @@ class Operator:
```
"""
@classmethod
- def from_uri(cls, /, uri: str, **kwargs) -> Operator:
+ def from_uri(cls, /, uri: str, **kwargs: str) -> Operator:
"""
Create a new blocking `Operator` from a URI string.
@@ -929,7 +932,9 @@ class Operator:
BlockingLister
An iterator over the entries in the directory.
"""
- def open(self, /, path: str | PathLike[str], mode: str, **kwargs) -> File:
+ def open(
+ self, /, path: str | PathLike[str], mode: str, **kwargs: Unpack[OpenKwargs]
+ ) -> File:
"""
Open a file-like object for the given path.
diff --git a/bindings/python/python/opendal/options.py b/bindings/python/python/opendal/options.py
new file mode 100644
index 000000000..2011ed2cc
--- /dev/null
+++ b/bindings/python/python/opendal/options.py
@@ -0,0 +1,168 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/generate/python.rs.
+
+"""Typed options for operator methods.
+
+Each `*Kwargs` is a `TypedDict` describing the keyword arguments accepted by
+the corresponding `Operator` / `AsyncOperator` method, e.g. `ReadKwargs` for
+`read`. Every key is optional; an unset key is equivalent to the method's
+default.
+
+Build an options dict, then unpack it into the call:
+
+```python
+import opendal
+
+opts: opendal.options.ReadKwargs = {
+ "offset": 1024,
+ "size": 2048,
+}
+op.read("path/to/file", **opts)
+```
+
+A static type checker validates the keys and their value types against the
+method's keyword arguments.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import TypedDict
+
+
+class ReadKwargs(TypedDict, total=False):
+ """Options for `read` operations."""
+
+ version: str
+ """The version of the file."""
+ concurrent: int
+ """The number of concurrent readers."""
+ chunk: int
+ """The size of each chunk."""
+ gap: int
+ """The gap between each chunk."""
+ offset: int
+ """The offset of the file."""
+ prefetch: int
+ """The number of bytes to prefetch."""
+ size: int
+ """The size of the file."""
+ if_match: str
+ """The ETag of the file."""
+ if_none_match: str
+ """The ETag of the file."""
+ if_modified_since: datetime
+ """The last modified time of the file."""
+ if_unmodified_since: datetime
+ """The last modified time of the file."""
+ content_type: str
+ """The content type of the file."""
+ cache_control: str
+ """The cache control of the file."""
+ content_disposition: str
+ """The content disposition of the file."""
+
+
+class WriteKwargs(TypedDict, total=False):
+ """Options for `write` operations."""
+
+ append: bool
+ """Whether to append to the file instead of overwriting it."""
+ chunk: int
+ """The chunk size to use when writing the file."""
+ concurrent: int
+ """The number of concurrent requests to make when writing the file."""
+ cache_control: str
+ """The cache control header to set on the file."""
+ content_type: str
+ """The content type header to set on the file."""
+ content_disposition: str
+ """The content disposition header to set on the file."""
+ content_encoding: str
+ """The content encoding header to set on the file."""
+ if_match: str
+ """The ETag to match when writing the file."""
+ if_none_match: str
+ """The ETag to not match when writing the file."""
+ if_not_exists: bool
+ """Whether to fail if the file already exists."""
+ user_metadata: dict[str, str]
+ """The user metadata to set on the file."""
+
+
+class ListKwargs(TypedDict, total=False):
+ """Options for `list` operations."""
+
+ limit: int
+ """The maximum number of entries to return."""
+ start_after: str
+ """The entry to start after."""
+ recursive: bool
+ """Whether to list recursively."""
+ versions: bool
+ """Whether to list versions."""
+ deleted: bool
+ """Whether to list deleted entries."""
+
+
+class StatKwargs(TypedDict, total=False):
+ """Options for `stat` operations."""
+
+ version: str
+ """The version of the file."""
+ if_match: str
+ """The ETag of the file."""
+ if_none_match: str
+ """The ETag of the file."""
+ if_modified_since: datetime
+ """The last modified time of the file."""
+ if_unmodified_since: datetime
+ """The last modified time of the file."""
+ content_type: str
+ """The content type of the file."""
+ cache_control: str
+ """The cache control of the file."""
+ content_disposition: str
+ """The content disposition of the file."""
+
+
+class DeleteKwargs(TypedDict, total=False):
+ """Options for `delete` operations."""
+
+ version: str
+ """The version of the file to delete. Only supported on version-aware backends."""
+ recursive: bool
+ """If True, delete the path recursively.
+
+ Only supported on backends that support recursive delete.
+ """
+
+
+class OpenKwargs(ReadKwargs, WriteKwargs, total=False):
+ """Keyword arguments for `Operator.open`."""
+
+
+__all__ = [
+ "ReadKwargs",
+ "WriteKwargs",
+ "ListKwargs",
+ "StatKwargs",
+ "DeleteKwargs",
+ "OpenKwargs",
+]
diff --git a/bindings/python/scripts/postprocess_stubs.py b/bindings/python/scripts/postprocess_stubs.py
index 32aa56912..2fa209193 100644
--- a/bindings/python/scripts/postprocess_stubs.py
+++ b/bindings/python/scripts/postprocess_stubs.py
@@ -54,8 +54,10 @@ IMPORTS = {
"import collections.abc\n"
"from .config import ServiceConfig\n"
"from .file import AsyncFile\n"
+ "from .options import OpenKwargs\n"
"from .services import Scheme\n"
"from .types import Entry, PresignedRequest\n"
+ "from typing_extensions import Unpack\n"
),
"file": "import collections.abc\nimport types\nimport typing_extensions\n",
}
diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs
index d4aea9e52..560eaacc5 100644
--- a/bindings/python/src/operator.rs
+++ b/bindings/python/src/operator.rs
@@ -190,7 +190,7 @@ impl Operator {
/// Operator
/// The new operator.
#[new]
- #[pyo3(signature = (scheme: "str | Scheme", *, **kwargs))]
+ #[pyo3(signature = (scheme: "str | Scheme", *, **kwargs: "str"))]
pub fn new(scheme: Bound<PyAny>, kwargs: Option<HashMap<String, String>>) -> PyResult<Self> {
let scheme = if let Ok(scheme_str) = scheme.extract::<&str>() {
scheme_str.to_string()
@@ -242,7 +242,7 @@ impl Operator {
/// op = opendal.Operator.from_uri(f"s3://bucket/path?{query}")
/// ```
#[classmethod]
- #[pyo3(signature = (uri, **kwargs))]
+ #[pyo3(signature = (uri, **kwargs: "str"))]
pub fn from_uri(
_cls: &Bound<PyType>,
uri: &str,
@@ -337,7 +337,7 @@ impl Operator {
/// -------
/// File
/// A file-like object.
- #[pyo3(signature = (path, mode, *, **kwargs))]
+ #[pyo3(signature = (path, mode, *, **kwargs: "Unpack[OpenKwargs]"))]
pub fn open(
&self,
path: PathBuf,
@@ -922,7 +922,7 @@ impl AsyncOperator {
/// AsyncOperator
/// The new async operator.
#[new]
- #[pyo3(signature = (scheme: "str | Scheme", * ,**kwargs))]
+ #[pyo3(signature = (scheme: "str | Scheme", * ,**kwargs: "str"))]
pub fn new(scheme: Bound<PyAny>, kwargs: Option<HashMap<String, String>>) -> PyResult<Self> {
let scheme = if let Ok(scheme_str) = scheme.extract::<&str>() {
scheme_str.to_string()
@@ -975,7 +975,7 @@ impl AsyncOperator {
/// op = opendal.AsyncOperator.from_uri(f"s3://bucket/path?{query}")
/// ```
#[classmethod]
- #[pyo3(signature = (uri, **kwargs))]
+ #[pyo3(signature = (uri, **kwargs: "str"))]
pub fn from_uri(
_cls: &Bound<PyType>,
uri: &str,
@@ -1066,7 +1066,7 @@ impl AsyncOperator {
/// -------
/// coroutine
/// An awaitable that returns a file-like object.
- #[pyo3(signature = (path, mode, *, **kwargs) -> "collections.abc.Awaitable[AsyncFile]")]
+ #[pyo3(signature = (path, mode, *, **kwargs: "Unpack[OpenKwargs]") -> "collections.abc.Awaitable[AsyncFile]")]
pub fn open<'p>(
&'p self,
py: Python<'p>,
diff --git a/bindings/python/src/options.rs b/bindings/python/src/options.rs
index 8d222b636..1a578d53a 100644
--- a/bindings/python/src/options.rs
+++ b/bindings/python/src/options.rs
@@ -29,22 +29,37 @@ use pyo3::types::PyDict;
use pyo3::types::PyDictMethods;
use std::collections::HashMap;
+/// Options for `read` operations.
#[pyclass(module = "opendal")]
#[derive(Default)]
pub struct ReadOptions {
+ /// The version of the file.
pub version: Option<String>,
+ /// The number of concurrent readers.
pub concurrent: Option<usize>,
+ /// The size of each chunk.
pub chunk: Option<usize>,
+ /// The gap between each chunk.
pub gap: Option<usize>,
+ /// The offset of the file.
pub offset: Option<usize>,
+ /// The number of bytes to prefetch.
pub prefetch: Option<usize>,
+ /// The size of the file.
pub size: Option<usize>,
+ /// The ETag of the file.
pub if_match: Option<String>,
+ /// The ETag of the file.
pub if_none_match: Option<String>,
+ /// The last modified time of the file.
pub if_modified_since: Option<jiff::Timestamp>,
+ /// The last modified time of the file.
pub if_unmodified_since: Option<jiff::Timestamp>,
+ /// The content type of the file.
pub content_type: Option<String>,
+ /// The cache control of the file.
pub cache_control: Option<String>,
+ /// The content disposition of the file.
pub content_disposition: Option<String>,
}
@@ -106,19 +121,31 @@ impl ReadOptions {
}
}
+/// Options for `write` operations.
#[pyclass(module = "opendal")]
#[derive(Default)]
pub struct WriteOptions {
+ /// Whether to append to the file instead of overwriting it.
pub append: Option<bool>,
+ /// The chunk size to use when writing the file.
pub chunk: Option<usize>,
+ /// The number of concurrent requests to make when writing the file.
pub concurrent: Option<usize>,
+ /// The cache control header to set on the file.
pub cache_control: Option<String>,
+ /// The content type header to set on the file.
pub content_type: Option<String>,
+ /// The content disposition header to set on the file.
pub content_disposition: Option<String>,
+ /// The content encoding header to set on the file.
pub content_encoding: Option<String>,
+ /// The ETag to match when writing the file.
pub if_match: Option<String>,
+ /// The ETag to not match when writing the file.
pub if_none_match: Option<String>,
+ /// Whether to fail if the file already exists.
pub if_not_exists: Option<bool>,
+ /// The user metadata to set on the file.
pub user_metadata: Option<HashMap<String, String>>,
}
@@ -199,13 +226,19 @@ impl From<WriteOptions> for ocore::options::WriteOptions {
}
}
+/// Options for `list` operations.
#[pyclass(module = "opendal")]
#[derive(Default, Debug)]
pub struct ListOptions {
+ /// The maximum number of entries to return.
pub limit: Option<usize>,
+ /// The entry to start after.
pub start_after: Option<String>,
+ /// Whether to list recursively.
pub recursive: Option<bool>,
+ /// Whether to list versions.
pub versions: Option<bool>,
+ /// Whether to list deleted entries.
pub deleted: Option<bool>,
}
@@ -237,16 +270,25 @@ impl From<ListOptions> for ocore::options::ListOptions {
}
}
+/// Options for `stat` operations.
#[pyclass(module = "opendal")]
#[derive(Default, Debug)]
pub struct StatOptions {
+ /// The version of the file.
pub version: Option<String>,
+ /// The ETag of the file.
pub if_match: Option<String>,
+ /// The ETag of the file.
pub if_none_match: Option<String>,
+ /// The last modified time of the file.
pub if_modified_since: Option<jiff::Timestamp>,
+ /// The last modified time of the file.
pub if_unmodified_since: Option<jiff::Timestamp>,
+ /// The content type of the file.
pub content_type: Option<String>,
+ /// The cache control of the file.
pub cache_control: Option<String>,
+ /// The content disposition of the file.
pub content_disposition: Option<String>,
}
@@ -284,10 +326,15 @@ impl From<StatOptions> for ocore::options::StatOptions {
}
}
+/// Options for `delete` operations.
#[pyclass(module = "opendal")]
#[derive(Default, Debug)]
pub struct DeleteOptions {
+ /// The version of the file to delete. Only supported on version-aware backends.
pub version: Option<String>,
+ /// If True, delete the path recursively.
+ ///
+ /// Only supported on backends that support recursive delete.
pub recursive: Option<bool>,
}
diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs
index 75e70e393..e5830e525 100644
--- a/dev/src/generate/mod.rs
+++ b/dev/src/generate/mod.rs
@@ -17,6 +17,7 @@
mod docs;
mod java;
+mod options;
mod parser;
mod python;
diff --git a/dev/src/generate/options.rs b/dev/src/generate/options.rs
new file mode 100644
index 000000000..16c7031cc
--- /dev/null
+++ b/dev/src/generate/options.rs
@@ -0,0 +1,229 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use anyhow::{Context, Result, anyhow, bail};
+use std::fs;
+use std::path::Path;
+use syn::{
+ Expr, ExprLit, Field, GenericArgument, Item, ItemStruct, Lit, Meta, PathArguments, Type,
+ TypePath,
+};
+
+/// The Python type an operator option field maps to.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum OptionType {
+ /// `Option<String>`
+ Str,
+ /// `Option<usize>`
+ Int,
+ /// `Option<bool>`
+ Bool,
+ /// `Option<jiff::Timestamp>`
+ DateTime,
+ /// `Option<HashMap<String, String>>`
+ Dict,
+}
+
+/// One keyword option of an operator method.
+#[derive(Debug, Clone)]
+pub struct OptionField {
+ /// The option name, e.g. `if_match`.
+ pub name: String,
+ /// The value type of this option.
+ pub ty: OptionType,
+ /// The doc comment of the field, trimmed.
+ pub doc: String,
+}
+
+/// One option type, parsed from a `#[pyclass]` struct in
+/// `bindings/python/src/options.rs`, e.g. `ReadOptions`.
+#[derive(Debug, Clone)]
+pub struct Options {
+ /// The struct name, e.g. `ReadOptions`.
+ pub name: String,
+ /// The doc comment of the struct, trimmed.
+ pub doc: String,
+ /// The fields of the struct.
+ pub fields: Vec<OptionField>,
+}
+
+/// Parse all `#[pyclass]` option structs from `bindings/python/src/options.rs`.
+///
+/// A struct is treated as an option type when it is annotated with
+/// `#[pyclass]` and its name ends with `Options`.
+pub fn parse(path: &Path) -> Result<Vec<Options>> {
+ let content =
+ fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?;
+ let ast = syn::parse_file(&content)?;
+
+ let mut options = Vec::new();
+ for item in ast.items {
+ if let Item::Struct(struct_) = item
+ && is_option_struct(&struct_)
+ {
+ options.push(parse_struct(&struct_)?);
+ }
+ }
+ Ok(options)
+}
+
+/// Whether a struct maps to a generated `*OptionsKwargs` TypedDict.
+fn is_option_struct(struct_: &ItemStruct) -> bool {
+ struct_.ident.to_string().ends_with("Options")
+ && struct_
+ .attrs
+ .iter()
+ .any(|attr| attr.path().is_ident("pyclass"))
+}
+
+fn parse_struct(struct_: &ItemStruct) -> Result<Options> {
+ let name = struct_.ident.to_string();
+ let doc = doc_lines(&struct_.attrs).join("\n").trim().to_string();
+
+ let fields = match &struct_.fields {
+ syn::Fields::Named(named) => &named.named,
+ _ => bail!("option struct {name} must have named fields"),
+ };
+
+ let mut parsed = Vec::with_capacity(fields.len());
+ for field in fields {
+ parsed.push(parse_field(field)?);
+ }
+
+ Ok(Options {
+ name,
+ doc,
+ fields: parsed,
+ })
+}
+
+fn parse_field(field: &Field) -> Result<OptionField> {
+ let name = field
+ .ident
+ .as_ref()
+ .ok_or_else(|| anyhow!("option field is unnamed: {field:?}"))?
+ .to_string();
+ let ty = parse_type(&field.ty)?;
+ let doc = doc_lines(&field.attrs).join("\n").trim().to_string();
+
+ Ok(OptionField { name, ty, doc })
+}
+
+/// Parse the type of an option field, which must be `Option<T>`.
+fn parse_type(ty: &Type) -> Result<OptionType> {
+ let Type::Path(TypePath { path, .. }) = ty else {
+ bail!("unsupported option field type: {ty:?}");
+ };
+ let segment = path
+ .segments
+ .last()
+ .ok_or_else(|| anyhow!("option type path is empty: {ty:?}"))?;
+ if segment.ident != "Option" {
+ bail!("option field must be Option<T>, found {segment:?}");
+ }
+ let PathArguments::AngleBracketed(args) = &segment.arguments else {
+ bail!("Option must have angle bracketed arguments");
+ };
+ let Some(GenericArgument::Type(Type::Path(inner))) = args.args.first() else {
+ bail!("Option must contain a type argument");
+ };
+ let inner_segment = inner
+ .path
+ .segments
+ .last()
+ .ok_or_else(|| anyhow!("option inner type path is empty: {inner:?}"))?;
+
+ Ok(match inner_segment.ident.to_string().as_str() {
+ "String" => OptionType::Str,
+ "usize" => OptionType::Int,
+ "bool" => OptionType::Bool,
+ // jiff::Timestamp
+ "Timestamp" => OptionType::DateTime,
+ // HashMap<String, String>
+ "HashMap" => OptionType::Dict,
+ other => bail!("unsupported option type: {other}"),
+ })
+}
+
+/// Collect the doc comment of a struct or field as trimmed lines.
+fn doc_lines(attrs: &[syn::Attribute]) -> Vec<String> {
+ attrs
+ .iter()
+ .filter(|attr| attr.path().is_ident("doc"))
+ .filter_map(|attr| {
+ if let Meta::NameValue(meta) = &attr.meta
+ && let Expr::Lit(ExprLit {
+ lit: Lit::Str(s), ..
+ }) = &meta.value
+ {
+ return Some(s.value().trim().to_string());
+ }
+
+ None
+ })
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_parse_options() {
+ let path = crate::workspace_dir().join("bindings/python/src/options.rs");
+ let options = parse(&path).unwrap();
+
+ let names: Vec<&str> = options.iter().map(|o| o.name.as_str()).collect();
+ assert_eq!(
+ names,
+ [
+ "ReadOptions",
+ "WriteOptions",
+ "ListOptions",
+ "StatOptions",
+ "DeleteOptions"
+ ]
+ );
+
+ let read = &options[0];
+ assert_eq!(read.doc, "Options for `read` operations.");
+ assert_eq!(read.fields.len(), 14);
+ assert_eq!(read.fields[0].name, "version");
+ assert_eq!(read.fields[0].ty, OptionType::Str);
+ assert_eq!(read.fields[4].name, "offset");
+ assert_eq!(read.fields[4].ty, OptionType::Int);
+ assert_eq!(read.fields[9].name, "if_modified_since");
+ assert_eq!(read.fields[9].ty, OptionType::DateTime);
+ assert_eq!(read.fields[0].doc, "The version of the file.");
+
+ let write = &options[1];
+ assert_eq!(write.fields.len(), 11);
+ assert_eq!(write.fields[10].name, "user_metadata");
+ assert_eq!(write.fields[10].ty, OptionType::Dict);
+ assert_eq!(
+ write.fields[10].doc,
+ "The user metadata to set on the file."
+ );
+
+ let delete = &options[4];
+ assert_eq!(delete.fields.len(), 2);
+ assert_eq!(
+ delete.fields[1].doc,
+ "If True, delete the path recursively.\n\nOnly supported on backends that support recursive delete."
+ );
+ }
+}
diff --git a/dev/src/generate/python.rs b/dev/src/generate/python.rs
index 0763c066a..b866349ff 100644
--- a/dev/src/generate/python.rs
+++ b/dev/src/generate/python.rs
@@ -15,11 +15,13 @@
// specific language governing permissions and limitations
// under the License.
+use crate::generate::options;
use crate::generate::parser::{Config, ConfigType, Services, sorted_services};
use anyhow::Result;
use minijinja::value::ViaDeserialize;
use minijinja::{Environment, context};
use std::fs;
+use std::path::Path;
use std::path::PathBuf;
fn enabled_service(srv: &str) -> bool {
@@ -65,6 +67,9 @@ pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> {
}
fs::write(output, rendered)?;
+ // Generate the typed operator options TypedDicts (Python).
+ generate_options_py(&workspace_dir)?;
+
Ok(())
}
@@ -352,3 +357,149 @@ fn make_python_type(ty: ViaDeserialize<ConfigType>) -> Result<String, minijinja:
}
.to_string())
}
+
+/// License header shared by generated Python files.
+const PYTHON_LICENSE_HEADER: &str = "\
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# \"License\"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+";
+
+/// Python type of an operator option field, parsed from
+/// `bindings/python/src/options.rs`.
+fn make_option_type(ty: options::OptionType) -> &'static str {
+ match ty {
+ options::OptionType::Str => "str",
+ options::OptionType::Int => "int",
+ options::OptionType::Bool => "bool",
+ options::OptionType::DateTime => "datetime",
+ options::OptionType::Dict => "dict[str, str]",
+ }
+}
+
+/// Generate `bindings/python/python/opendal/options.py`, the typed keyword
+/// arguments accepted by the operator methods.
+///
+/// The option fields are parsed from the `#[pyclass]` structs in
+/// `bindings/python/src/options.rs`, so the generated file stays in sync with
+/// the Rust option types.
+fn generate_options_py(workspace_dir: &Path) -> Result<()> {
+ let source = workspace_dir.join("bindings/python/src/options.rs");
+ let options = options::parse(&source)?;
+ if options.is_empty() {
+ anyhow::bail!("no option structs parsed from {}", source.display());
+ }
+
+ let mut out = String::new();
+ out.push_str(PYTHON_LICENSE_HEADER);
+ out.push_str("\n# DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/generate/python.rs.\n\n");
+ out.push_str(OPTIONS_MODULE_DOC);
+ out.push_str(
+ "\nfrom __future__ import annotations\n\nfrom datetime import datetime\nfrom typing import TypedDict\n",
+ );
+
+ let mut kwargs_names = Vec::new();
+ for opts in &options {
+ // `ReadOptions` maps to the `ReadKwargs` TypedDict.
+ let kwargs = format!(
+ "{}Kwargs",
+ opts.name.strip_suffix("Options").unwrap_or(&opts.name)
+ );
+ kwargs_names.push(kwargs.clone());
+ let doc = if opts.doc.is_empty() {
+ "Keyword arguments for the corresponding `Operator` / `AsyncOperator` method."
+ .to_string()
+ } else {
+ opts.doc.clone()
+ };
+ out.push_str(&format!(
+ "\n\nclass {kwargs}(TypedDict, total=False):\n \"\"\"{doc}\"\"\"\n\n"
+ ));
+ for field in &opts.fields {
+ out.push_str(&format!(
+ " {}: {}\n",
+ field.name,
+ make_option_type(field.ty)
+ ));
+ push_field_doc(&mut out, &field.doc);
+ }
+ }
+
+ // `open` accepts the union of the read and write options, so its kwargs
+ // TypedDict inherits both.
+ kwargs_names.push("OpenKwargs".to_string());
+ out.push_str(
+ "\n\nclass OpenKwargs(ReadKwargs, WriteKwargs, total=False):\n \"\"\"Keyword arguments for `Operator.open`.\"\"\"\n",
+ );
+
+ out.push_str("\n\n__all__ = [\n");
+ for name in &kwargs_names {
+ out.push_str(&format!(" \"{name}\",\n"));
+ }
+ out.push_str("]\n");
+
+ let output = workspace_dir.join("bindings/python/python/opendal/options.py");
+ fs::write(output, out)?;
+ Ok(())
+}
+
+/// Module docstring for the generated `options.py`.
+const OPTIONS_MODULE_DOC: &str = "\
+\"\"\"Typed options for operator methods.
+
+Each `*Kwargs` is a `TypedDict` describing the keyword arguments accepted by
+the corresponding `Operator` / `AsyncOperator` method, e.g. `ReadKwargs` for
+`read`. Every key is optional; an unset key is equivalent to the method's
+default.
+
+Build an options dict, then unpack it into the call:
+
+```python
+import opendal
+
+opts: opendal.options.ReadKwargs = {
+ \"offset\": 1024,
+ \"size\": 2048,
+}
+op.read(\"path/to/file\", **opts)
+```
+
+A static type checker validates the keys and their value types against the
+method's keyword arguments.
+\"\"\"
+";
+
+/// Render a field docstring as a `"""..."""` block, one line per source line.
+fn push_field_doc(out: &mut String, doc: &str) {
+ let lines: Vec<&str> = doc.lines().collect();
+ if lines.is_empty() {
+ return;
+ }
+ out.push_str(" \"\"\"");
+ out.push_str(lines[0]);
+ if lines.len() == 1 {
+ out.push_str("\"\"\"\n");
+ } else {
+ for line in &lines[1..] {
+ out.push('\n');
+ if !line.is_empty() {
+ out.push_str(" ");
+ out.push_str(line);
+ }
+ }
+ out.push_str("\n \"\"\"\n");
+ }
+}