You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/03/12 11:34:06 UTC
(opendal) 05/05: Address python
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch align-api
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 667707c82a12bfb781ad0303413db5979809fed9
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Mar 12 19:33:27 2024 +0800
Address python
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
bin/ofs/Cargo.lock | 20 ++-----
bin/oli/Cargo.lock | 107 ++++++++++++++++++++++++++++++++--
bindings/nodejs/src/lib.rs | 6 +-
bindings/ocaml/src/operator/reader.rs | 6 +-
bindings/python/src/file.rs | 8 +--
core/src/types/reader.rs | 46 +++++++++++++++
6 files changed, 162 insertions(+), 31 deletions(-)
diff --git a/bin/ofs/Cargo.lock b/bin/ofs/Cargo.lock
index 591f06532f..a3d40c91b6 100644
--- a/bin/ofs/Cargo.lock
+++ b/bin/ofs/Cargo.lock
@@ -114,9 +114,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "backon"
-version = "0.4.1"
+version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9"
+checksum = "c491fa80d69c03084223a4e73c378dd9f9a1e612eb54051213f88b2d5249b458"
dependencies = [
"fastrand",
"futures-core",
@@ -364,12 +364,9 @@ dependencies = [
[[package]]
name = "fastrand"
-version = "1.9.0"
+version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"
-dependencies = [
- "instant",
-]
+checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
[[package]]
name = "flagset"
@@ -686,15 +683,6 @@ dependencies = [
"hashbrown",
]
-[[package]]
-name = "instant"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
-dependencies = [
- "cfg-if",
-]
-
[[package]]
name = "ipnet"
version = "2.9.0"
diff --git a/bin/oli/Cargo.lock b/bin/oli/Cargo.lock
index a8f048c0cd..f28a894261 100644
--- a/bin/oli/Cargo.lock
+++ b/bin/oli/Cargo.lock
@@ -17,6 +17,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+[[package]]
+name = "aes"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures",
+]
+
[[package]]
name = "aho-corasick"
version = "1.1.2"
@@ -365,11 +376,11 @@ dependencies = [
[[package]]
name = "backon"
-version = "0.4.1"
+version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9"
+checksum = "c491fa80d69c03084223a4e73c378dd9f9a1e612eb54051213f88b2d5249b458"
dependencies = [
- "fastrand 1.9.0",
+ "fastrand 2.0.1",
"futures-core",
"pin-project",
"tokio",
@@ -457,6 +468,15 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "block-padding"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
+dependencies = [
+ "generic-array",
+]
+
[[package]]
name = "blocking"
version = "1.5.1"
@@ -550,6 +570,15 @@ dependencies = [
"serde_json",
]
+[[package]]
+name = "cbc"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
+dependencies = [
+ "cipher",
+]
+
[[package]]
name = "cc"
version = "1.0.83"
@@ -589,6 +618,16 @@ dependencies = [
"windows-targets 0.52.0",
]
+[[package]]
+name = "cipher"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
+dependencies = [
+ "crypto-common",
+ "inout",
+]
+
[[package]]
name = "clang-sys"
version = "1.7.0"
@@ -1423,6 +1462,16 @@ dependencies = [
"hashbrown 0.14.3",
]
+[[package]]
+name = "inout"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
+dependencies = [
+ "block-padding",
+ "generic-array",
+]
+
[[package]]
name = "instant"
version = "0.1.12"
@@ -1993,6 +2042,16 @@ dependencies = [
"windows-targets 0.48.5",
]
+[[package]]
+name = "pbkdf2"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
+dependencies = [
+ "digest",
+ "hmac",
+]
+
[[package]]
name = "peeking_take_while"
version = "0.1.2"
@@ -2088,6 +2147,21 @@ dependencies = [
"spki",
]
+[[package]]
+name = "pkcs5"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6"
+dependencies = [
+ "aes",
+ "cbc",
+ "der",
+ "pbkdf2",
+ "scrypt",
+ "sha2",
+ "spki",
+]
+
[[package]]
name = "pkcs8"
version = "0.10.2"
@@ -2095,6 +2169,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
dependencies = [
"der",
+ "pkcs5",
+ "rand_core",
"spki",
]
@@ -2446,9 +2522,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
[[package]]
name = "reqsign"
-version = "0.14.7"
+version = "0.14.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed08ac3aa0676637644b1b892202f1ae789c28c15ebfa906128d111ae8086062"
+checksum = "43e319d9de9ff4d941abf4ac718897118b0fe04577ea3f8e0f5788971784eef5"
dependencies = [
"anyhow",
"async-trait",
@@ -2556,6 +2632,7 @@ dependencies = [
"pkcs1",
"pkcs8",
"rand_core",
+ "sha2",
"signature",
"spki",
"subtle",
@@ -2675,6 +2752,15 @@ version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
+[[package]]
+name = "salsa20"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213"
+dependencies = [
+ "cipher",
+]
+
[[package]]
name = "same-file"
version = "1.0.6"
@@ -2699,6 +2785,17 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "scrypt"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f"
+dependencies = [
+ "pbkdf2",
+ "salsa20",
+ "sha2",
+]
+
[[package]]
name = "sct"
version = "0.7.1"
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
index 4a0a9e188d..fec882b879 100644
--- a/bindings/nodejs/src/lib.rs
+++ b/bindings/nodejs/src/lib.rs
@@ -26,7 +26,6 @@ use std::time::Duration;
use futures::TryStreamExt;
use napi::bindgen_prelude::*;
-use opendal::raw::oio::BlockingRead;
#[napi]
pub struct Operator(opendal::Operator);
@@ -648,7 +647,10 @@ pub struct BlockingReader(opendal::BlockingReader);
impl BlockingReader {
#[napi]
pub fn read(&mut self, mut buf: Buffer) -> Result<usize> {
- self.0.read(buf.as_mut()).map_err(format_napi_error)
+ let buf = buf.as_mut();
+ let bs = self.0.read(buf.len()).map_err(format_napi_error)?;
+ buf[..bs.len()].copy_from_slice(&bs);
+ Ok(bs.len())
}
}
diff --git a/bindings/ocaml/src/operator/reader.rs b/bindings/ocaml/src/operator/reader.rs
index b3bc7bc587..8ae7b4583b 100644
--- a/bindings/ocaml/src/operator/reader.rs
+++ b/bindings/ocaml/src/operator/reader.rs
@@ -17,14 +17,14 @@
use std::io;
-use opendal::raw::oio::BlockingRead;
-
use super::*;
#[ocaml::func]
#[ocaml::sig("reader -> bytes -> (int, string) Result.t ")]
pub fn reader_read(reader: &mut Reader, buf: &mut [u8]) -> Result<usize, String> {
- map_res_error(reader.0.read(buf))
+ let bs = map_res_error(reader.0.read(buf.len()))?;
+ buf[..bs.len()].copy_from_slice(&bs);
+ Ok(bs.len())
}
#[ocaml::func]
diff --git a/bindings/python/src/file.rs b/bindings/python/src/file.rs
index 806306ca5a..f8a98c8f51 100644
--- a/bindings/python/src/file.rs
+++ b/bindings/python/src/file.rs
@@ -18,7 +18,6 @@
// Remove this `allow` after <https://github.com/rust-lang/rust-clippy/issues/12039> fixed.
#![allow(clippy::unnecessary_fallible_conversions)]
-use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::io::Write;
@@ -77,11 +76,10 @@ impl File {
let buffer = match size {
Some(size) => {
- let mut buffer = vec![0; size];
- reader
- .read_exact(&mut buffer)
+ let bs = reader
+ .read_exact(size)
.map_err(|err| PyIOError::new_err(err.to_string()))?;
- buffer
+ bs.to_vec()
}
None => {
let mut buffer = Vec::new();
diff --git a/core/src/types/reader.rs b/core/src/types/reader.rs
index 6aa068dab5..8db4e9f9f4 100644
--- a/core/src/types/reader.rs
+++ b/core/src/types/reader.rs
@@ -458,6 +458,52 @@ impl BlockingReader {
self.inner.read(limit)
}
+ /// Read exact `size` bytes of data from reader.
+ pub fn read_exact(&mut self, size: usize) -> Result<Bytes> {
+ // Lucky path.
+ let bs1 = self.inner.read(size)?;
+ debug_assert!(
+ bs1.len() <= size,
+ "read should not return more bytes than expected"
+ );
+ if bs1.len() == size {
+ return Ok(bs1);
+ }
+ if bs1.is_empty() {
+ return Err(
+ Error::new(ErrorKind::ContentIncomplete, "reader got too little data")
+ .with_context("expect", size.to_string()),
+ );
+ }
+
+ let mut bs = BytesMut::with_capacity(size);
+ bs.put_slice(&bs1);
+
+ let mut remaining = size - bs.len();
+
+ loop {
+ let tmp = self.inner.read(remaining)?;
+ if tmp.is_empty() {
+ return Err(
+ Error::new(ErrorKind::ContentIncomplete, "reader got too little data")
+ .with_context("expect", size.to_string())
+ .with_context("actual", bs.len().to_string()),
+ );
+ }
+ bs.put_slice(&tmp);
+ debug_assert!(
+ tmp.len() <= remaining,
+ "read should not return more bytes than expected"
+ );
+
+ remaining -= tmp.len();
+ if remaining == 0 {
+ break;
+ }
+ }
+
+ Ok(bs.freeze())
+ }
/// Reads all bytes until EOF in this source, placing them into buf.
pub fn read_to_end(&mut self, buf: &mut Vec<u8>) -> Result<usize> {
let start_len = buf.len();