You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/03/18 05:51:19 UTC
[incubator-opendal] branch main updated: feat(bindings/python): add auto-generated api docs (#1613)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 0ac2b78a feat(bindings/python): add auto-generated api docs (#1613)
0ac2b78a is described below
commit 0ac2b78a04a40459eed85f29ffbb03cdcdfcb79f
Author: messense <me...@icloud.com>
AuthorDate: Sat Mar 18 13:51:14 2023 +0800
feat(bindings/python): add auto-generated api docs (#1613)
* fix(bindings/python): add workaround to make `from opendal.layers import XXX` work
* feat(bindings/python): add docsstrings
* feat(bindings/python): build docs on CI
---
.github/workflows/docs.yml | 21 ++++++++++
bindings/python/src/asyncio.rs | 41 +++++++++++++++++++
bindings/python/src/lib.rs | 89 +++++++++++++++++++++++++++++++++++++++++-
website/docusaurus.config.js | 4 ++
4 files changed, 153 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 6c524763..8b1a1217 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -83,6 +83,27 @@ jobs:
- name: Copy docs into build
run: |
cp -r ./bindings/nodejs/docs ./website/static/docs/nodejs
+
+ # Setup python environment ----------------------------------------
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '3.11'
+
+ # Build bindings/python docs --------------------------------------
+ - name: Install dependencies
+ working-directory: bindings/python
+ run: |
+ set -e
+ python -m pip install -e .
+ python -m pip install pdoc
+
+ - name: Build bindings/python Docs
+ working-directory: bindings/python
+ run: pdoc --output-dir ./docs opendal
+
+ - name: Copy docs into build
+ run: |
+ cp -r ./bindings/python/docs ./website/static/docs/python
# Build website ---------------------------------------------------
- name: Install Dependencies
diff --git a/bindings/python/src/asyncio.rs b/bindings/python/src/asyncio.rs
index b57c6607..dcfbbbed 100644
--- a/bindings/python/src/asyncio.rs
+++ b/bindings/python/src/asyncio.rs
@@ -36,6 +36,9 @@ use tokio::sync::Mutex;
use crate::{build_operator, format_pyerr, layers, Entry, Metadata};
+/// `AsyncOperator` is the entry for all public async APIs
+///
+/// Create a new `AsyncOperator` with the given `scheme` and options(`**kwargs`).
#[pyclass(module = "opendal")]
pub struct AsyncOperator(od::Operator);
@@ -59,6 +62,7 @@ impl AsyncOperator {
Ok(AsyncOperator(build_operator(scheme, map, layers)?))
}
+ /// Read the whole path into bytes.
pub fn read<'p>(&'p self, py: Python<'p>, path: String) -> PyResult<&'p PyAny> {
let this = self.0.clone();
future_into_py(py, async move {
@@ -68,6 +72,7 @@ impl AsyncOperator {
})
}
+ /// Open a file-like reader for the given path.
pub fn open_reader(&self, path: String) -> PyResult<AsyncReader> {
Ok(AsyncReader::new(ReaderState::Init {
operator: self.0.clone(),
@@ -75,6 +80,7 @@ impl AsyncOperator {
}))
}
+ /// Write bytes into given path.
pub fn write<'p>(&'p self, py: Python<'p>, path: String, bs: Vec<u8>) -> PyResult<&'p PyAny> {
let this = self.0.clone();
future_into_py(py, async move {
@@ -82,6 +88,7 @@ impl AsyncOperator {
})
}
+ /// Get current path's metadata **without cache** directly.
pub fn stat<'p>(&'p self, py: Python<'p>, path: String) -> PyResult<&'p PyAny> {
let this = self.0.clone();
future_into_py(py, async move {
@@ -91,6 +98,18 @@ impl AsyncOperator {
})
}
+ /// Create a dir at given path.
+ ///
+ /// # Notes
+ ///
+ /// To indicate that a path is a directory, it is compulsory to include
+ /// a trailing / in the path. Failure to do so may result in
+ /// `NotADirectory` error being returned by OpenDAL.
+ ///
+ /// # Behavior
+ ///
+ /// - Create on existing dir will succeed.
+ /// - Create dir is always recursive, works like `mkdir -p`
pub fn create_dir<'p>(&'p self, py: Python<'p>, path: String) -> PyResult<&'p PyAny> {
let this = self.0.clone();
future_into_py(py, async move {
@@ -98,6 +117,11 @@ impl AsyncOperator {
})
}
+ /// Delete given path.
+ ///
+ /// # Notes
+ ///
+ /// - Delete not existing error won't return errors.
pub fn delete<'p>(&'p self, py: Python<'p>, path: String) -> PyResult<&'p PyAny> {
let this = self.0.clone();
future_into_py(
@@ -106,6 +130,7 @@ impl AsyncOperator {
)
}
+ /// List current dir path.
pub fn list<'p>(&'p self, py: Python<'p>, path: String) -> PyResult<&'p PyAny> {
let this = self.0.clone();
future_into_py(py, async move {
@@ -115,6 +140,7 @@ impl AsyncOperator {
})
}
+ /// List dir in flat way.
pub fn scan<'p>(&'p self, py: Python<'p>, path: String) -> PyResult<&'p PyAny> {
let this = self.0.clone();
future_into_py(py, async move {
@@ -159,6 +185,8 @@ impl ReaderState {
}
}
+/// A file-like async reader.
+/// Can be used as an async context manager.
#[pyclass(module = "opendal")]
pub struct AsyncReader(Arc<Mutex<ReaderState>>);
@@ -170,6 +198,7 @@ impl AsyncReader {
#[pymethods]
impl AsyncReader {
+ /// Read and return size bytes, or if size is not given, until EOF.
pub fn read<'p>(&'p self, py: Python<'p>, size: Option<usize>) -> PyResult<&'p PyAny> {
let reader = self.0.clone();
future_into_py(py, async move {
@@ -198,6 +227,8 @@ impl AsyncReader {
})
}
+ /// `AsyncReader` doesn't support write.
+ /// Raises a `NotImplementedError` if called.
pub fn write<'p>(&'p mut self, py: Python<'p>, _bs: &'p [u8]) -> PyResult<&'p PyAny> {
future_into_py::<_, PyObject>(py, async move {
Err(PyNotImplementedError::new_err(
@@ -206,6 +237,15 @@ impl AsyncReader {
})
}
+ /// Change the stream position to the given byte offset.
+ /// offset is interpreted relative to the position indicated by `whence`.
+ /// The default value for whence is `SEEK_SET`. Values for `whence` are:
+ ///
+ /// * `SEEK_SET` or `0` – start of the stream (the default); offset should be zero or positive
+ /// * `SEEK_CUR` or `1` – current stream position; offset may be negative
+ /// * `SEEK_END` or `2` – end of the stream; offset is usually negative
+ ///
+ /// Return the new absolute position.
#[pyo3(signature = (pos, whence = 0))]
pub fn seek<'p>(&'p mut self, py: Python<'p>, pos: i64, whence: u8) -> PyResult<&'p PyAny> {
let whence = match whence {
@@ -226,6 +266,7 @@ impl AsyncReader {
})
}
+ /// Return the current stream position.
pub fn tell<'p>(&'p mut self, py: Python<'p>) -> PyResult<&'p PyAny> {
let reader = self.0.clone();
future_into_py(py, async move {
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index d5dfe4a8..14ffddc4 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -40,7 +40,7 @@ mod layers;
use crate::asyncio::*;
-create_exception!(opendal, Error, PyException);
+create_exception!(opendal, Error, PyException, "OpenDAL related errors");
fn add_layers(mut op: od::Operator, layers: Vec<layers::Layer>) -> PyResult<od::Operator> {
for layer in layers {
@@ -112,6 +112,9 @@ fn build_operator(
add_layers(op, layers)
}
+/// `Operator` is the entry for all public blocking APIs
+///
+/// Create a new blocking `Operator` with the given `scheme` and options(`**kwargs`).
#[pyclass(module = "opendal")]
struct Operator(od::BlockingOperator);
@@ -135,6 +138,7 @@ impl Operator {
Ok(Operator(build_operator(scheme, map, layers)?.blocking()))
}
+ /// Read the whole path into bytes.
pub fn read<'p>(&'p self, py: Python<'p>, path: &str) -> PyResult<&'p PyAny> {
self.0
.read(path)
@@ -142,6 +146,7 @@ impl Operator {
.map(|res| PyBytes::new(py, &res).into())
}
+ /// Open a file-like reader for the given path.
pub fn open_reader(&self, path: &str) -> PyResult<Reader> {
self.0
.reader(path)
@@ -149,31 +154,54 @@ impl Operator {
.map_err(format_pyerr)
}
+ /// Write bytes into given path.
pub fn write(&self, path: &str, bs: Vec<u8>) -> PyResult<()> {
self.0.write(path, bs).map_err(format_pyerr)
}
+ /// Get current path's metadata **without cache** directly.
pub fn stat(&self, path: &str) -> PyResult<Metadata> {
self.0.stat(path).map_err(format_pyerr).map(Metadata)
}
+ /// Create a dir at given path.
+ ///
+ /// # Notes
+ ///
+ /// To indicate that a path is a directory, it is compulsory to include
+ /// a trailing / in the path. Failure to do so may result in
+ /// `NotADirectory` error being returned by OpenDAL.
+ ///
+ /// # Behavior
+ ///
+ /// - Create on existing dir will succeed.
+ /// - Create dir is always recursive, works like `mkdir -p`
pub fn create_dir(&self, path: &str) -> PyResult<()> {
self.0.create_dir(path).map_err(format_pyerr)
}
+ /// Delete given path.
+ ///
+ /// # Notes
+ ///
+ /// - Delete not existing error won't return errors.
pub fn delete(&self, path: &str) -> PyResult<()> {
self.0.delete(path).map_err(format_pyerr)
}
+ /// List current dir path.
pub fn list(&self, path: &str) -> PyResult<BlockingLister> {
Ok(BlockingLister(self.0.list(path).map_err(format_pyerr)?))
}
+ /// List dir in flat way.
pub fn scan(&self, path: &str) -> PyResult<BlockingLister> {
Ok(BlockingLister(self.0.scan(path).map_err(format_pyerr)?))
}
}
+/// A file-like blocking reader.
+/// Can be used as a context manager.
#[pyclass(module = "opendal")]
struct Reader(Option<od::BlockingReader>);
@@ -189,6 +217,7 @@ impl Reader {
#[pymethods]
impl Reader {
+ /// Read and return size bytes, or if size is not given, until EOF.
#[pyo3(signature = (size=None,))]
pub fn read<'p>(&'p mut self, py: Python<'p>, size: Option<usize>) -> PyResult<&'p PyAny> {
let reader = self.as_mut()?;
@@ -211,12 +240,23 @@ impl Reader {
Ok(PyBytes::new(py, &buffer).into())
}
+ /// `Reader` doesn't support write.
+ /// Raises a `NotImplementedError` if called.
pub fn write(&mut self, _bs: &[u8]) -> PyResult<()> {
Err(PyNotImplementedError::new_err(
"Reader does not support write",
))
}
+ /// Change the stream position to the given byte offset.
+ /// offset is interpreted relative to the position indicated by `whence`.
+ /// The default value for whence is `SEEK_SET`. Values for `whence` are:
+ ///
+ /// * `SEEK_SET` or `0` – start of the stream (the default); offset should be zero or positive
+ /// * `SEEK_CUR` or `1` – current stream position; offset may be negative
+ /// * `SEEK_END` or `2` – end of the stream; offset is usually negative
+ ///
+ /// Return the new absolute position.
#[pyo3(signature = (pos, whence = 0))]
pub fn seek(&mut self, pos: i64, whence: u8) -> PyResult<u64> {
let whence = match whence {
@@ -231,6 +271,7 @@ impl Reader {
.map_err(|err| PyIOError::new_err(err.to_string()))
}
+ /// Return the current stream position.
pub fn tell(&mut self) -> PyResult<u64> {
let reader = self.as_mut()?;
reader
@@ -272,6 +313,7 @@ struct Entry(od::Entry);
#[pymethods]
impl Entry {
+ /// Path of entry. Path is relative to operator's root.
#[getter]
pub fn path(&self) -> &str {
self.0.path()
@@ -296,26 +338,31 @@ impl Metadata {
self.0.content_disposition()
}
+ /// Content length of this entry.
#[getter]
pub fn content_length(&self) -> u64 {
self.0.content_length()
}
+ /// Content MD5 of this entry.
#[getter]
pub fn content_md5(&self) -> Option<&str> {
self.0.content_md5()
}
+ /// Content Type of this entry.
#[getter]
pub fn content_type(&self) -> Option<&str> {
self.0.content_type()
}
+ /// ETag of this entry.
#[getter]
pub fn etag(&self) -> Option<&str> {
self.0.etag()
}
+ /// mode represent this entry's mode.
#[getter]
pub fn mode(&self) -> EntryMode {
EntryMode(self.0.mode())
@@ -327,10 +374,12 @@ struct EntryMode(od::EntryMode);
#[pymethods]
impl EntryMode {
+ /// Returns `True` if this is a file.
pub fn is_file(&self) -> bool {
self.0.is_file()
}
+ /// Returns `True` if this is a directory.
pub fn is_dir(&self) -> bool {
self.0.is_dir()
}
@@ -352,6 +401,37 @@ fn format_pyerr(err: od::Error) -> PyErr {
}
}
+/// OpenDAL Python binding
+///
+/// ## Installation
+///
+/// ```bash
+/// pip install opendal
+/// ```
+///
+/// ## Usage
+///
+/// ```python
+/// import opendal
+///
+/// op = opendal.Operator("fs", root="/tmp")
+/// op.write("test.txt", b"Hello World")
+/// print(op.read("test.txt"))
+/// print(op.stat("test.txt").content_length)
+/// ```
+///
+/// Or using the async API:
+///
+/// ```python
+/// import asyncio
+///
+/// async def main():
+/// op = opendal.AsyncOperator("fs", root="/tmp")
+/// await op.write("test.txt", b"Hello World")
+/// print(await op.read("test.txt"))
+///
+/// asyncio.run(main())
+/// ```
#[pymodule]
fn opendal(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<Operator>()?;
@@ -363,6 +443,11 @@ fn opendal(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<Metadata>()?;
m.add("Error", py.get_type::<Error>())?;
- m.add_submodule(layers::create_submodule(py)?)?;
+ let layers = layers::create_submodule(py)?;
+ m.add_submodule(layers)?;
+ py.import("sys")?
+ .getattr("modules")?
+ .set_item("opendal.layers", layers)?;
+
Ok(())
}
diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
index 9e3c874f..f2fb5ffa 100644
--- a/website/docusaurus.config.js
+++ b/website/docusaurus.config.js
@@ -98,6 +98,10 @@ const config = {
type: 'html',
value: '<a class="dropdown__link" href="/docs/nodejs/">Node.js</a>'
},
+ {
+ type: 'html',
+ value: '<a class="dropdown__link" href="/docs/python/">Python</a>'
+ },
]
},
{