(opendal) branch main updated: perf(bindings/python): avoid an extra copy when reading bytes (#8160)

[xu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 053739c76 perf(bindings/python): avoid an extra copy when reading bytes (#8160)
053739c76 is described below

commit 053739c76bfa8b127e62cdb8efb52b86130a059f
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Aug 26 23:00:52 2026 +0800

    perf(bindings/python): avoid an extra copy when reading bytes (#8160)
    
    perf(bindings/python): avoid extra read copy
---
 bindings/python/src/operator.rs | 10 +++-----
 bindings/python/src/utils.rs    | 56 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs
index 7f017f81c..ebdcea662 100644
--- a/bindings/python/src/operator.rs
+++ b/bindings/python/src/operator.rs
@@ -472,10 +472,9 @@ impl Operator {
         let buffer = self
             .core
             .read_options(&path, opts.into())
-            .map_err(format_pyerr)?
-            .to_vec();
+            .map_err(format_pyerr)?;
 
-        Buffer::new(buffer).into_bytes_ref(py)
+        Ok(buffer_into_py_bytes(py, buffer)?.into_any())
     }
 
     /// Write bytes to a file at the given path.
@@ -1219,9 +1218,8 @@ impl AsyncOperator {
                 .map_err(format_pyerr)?
                 .read(range.to_range())
                 .await
-                .map_err(format_pyerr)?
-                .to_vec();
-            Python::attach(|py| Buffer::new(res).into_bytes(py))
+                .map_err(format_pyerr)?;
+            Python::attach(|py| buffer_into_py_bytes(py, res).map(Bound::unbind))
         })
     }
 
diff --git a/bindings/python/src/utils.rs b/bindings/python/src/utils.rs
index dd86ce358..bbd0d3563 100644
--- a/bindings/python/src/utils.rs
+++ b/bindings/python/src/utils.rs
@@ -17,9 +17,65 @@
 
 use std::os::raw::c_int;
 
+use bytes::Buf;
 use pyo3::IntoPyObjectExt;
+use pyo3::exceptions::PyOverflowError;
 use pyo3::ffi;
 use pyo3::prelude::*;
+use pyo3::types::PyBytes;
+
+use crate::ocore;
+
+/// Copy an OpenDAL buffer directly into a new Python `bytes` object.
+pub fn buffer_into_py_bytes<'py>(
+    py: Python<'py>,
+    mut buffer: ocore::Buffer,
+) -> PyResult<Bound<'py, PyBytes>> {
+    let len = buffer.remaining();
+    let py_len = len
+        .try_into()
+        .map_err(|_| PyOverflowError::new_err("buffer is too large for Python bytes"))?;
+
+    unsafe {
+        let bytes = Bound::from_owned_ptr_or_err(
+            py,
+            ffi::PyBytes_FromStringAndSize(std::ptr::null(), py_len),
+        )?
+        .cast_into_unchecked::<PyBytes>();
+
+        // PyBytes_FromStringAndSize with a null source allocates an uninitialized
+        // buffer. `buffer` has exactly `len` remaining bytes, so copy_to_slice
+        // initializes the entire Python object before it becomes observable.
+        let dst =
+            std::slice::from_raw_parts_mut(ffi::PyBytes_AsString(bytes.as_ptr()).cast::<u8>(), len);
+        buffer.copy_to_slice(dst);
+
+        Ok(bytes)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use bytes::Bytes;
+    use pyo3::types::PyBytesMethods;
+
+    use super::*;
+
+    #[test]
+    fn test_buffer_into_py_bytes() {
+        Python::initialize();
+        Python::attach(|py| {
+            let buffer = [Bytes::from_static(b"hello, "), Bytes::from_static(b"world")]
+                .into_iter()
+                .collect();
+            let result = buffer_into_py_bytes(py, buffer).unwrap();
+            assert_eq!(result.as_bytes(), b"hello, world");
+
+            let result = buffer_into_py_bytes(py, ocore::Buffer::new()).unwrap();
+            assert!(result.as_bytes().is_empty());
+        });
+    }
+}
 
 /// A bytes-like object that implements buffer protocol.
 #[pyclass(module = "opendal")]