You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xy...@apache.org on 2023/06/21 08:10:15 UTC
[incubator-opendal] branch main updated: chore: Polish some code details (#2505)
This is an automated email from the ASF dual-hosted git repository.
xyji pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 46dd42a09 chore: Polish some code details (#2505)
46dd42a09 is described below
commit 46dd42a092b0a608c32866b3eb7d1cf26403c601
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Jun 21 16:10:09 2023 +0800
chore: Polish some code details (#2505)
* Format code
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* polish some details
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Use cargo test for supabase
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
.github/workflows/service_test_supabase.yml | 4 +-
bindings/c/src/lib.rs | 11 +++--
bindings/haskell/src/lib.rs | 5 ++-
bindings/haskell/src/result.rs | 7 ++--
bindings/lua/src/lib.rs | 3 +-
core/src/layers/mod.rs | 5 +--
core/src/layers/throttle.rs | 13 ++++--
.../src/raw/oio/into_stream/from_futures_stream.rs | 3 +-
core/src/raw/oio/stream.rs | 4 +-
core/src/services/dropbox/backend.rs | 35 +++++-----------
core/src/services/dropbox/builder.rs | 48 +++++++++++-----------
core/src/services/dropbox/core.rs | 38 ++++++++---------
core/src/services/ftp/backend.rs | 1 -
core/src/services/hdfs/backend.rs | 1 -
core/src/services/ipfs/backend.rs | 1 -
core/src/services/ipmfs/backend.rs | 1 -
core/src/services/memory/backend.rs | 1 -
core/src/services/mod.rs | 1 -
core/src/services/obs/backend.rs | 3 +-
core/src/services/obs/core.rs | 3 +-
core/src/services/redis/backend.rs | 1 -
core/src/services/rocksdb/backend.rs | 1 -
core/src/services/s3/backend.rs | 1 -
core/src/services/sftp/backend.rs | 1 -
core/src/services/sled/backend.rs | 1 -
core/src/services/webhdfs/backend.rs | 1 -
core/src/services/webhdfs/error.rs | 3 +-
27 files changed, 90 insertions(+), 107 deletions(-)
diff --git a/.github/workflows/service_test_supabase.yml b/.github/workflows/service_test_supabase.yml
index 1b529021a..08d449ccb 100644
--- a/.github/workflows/service_test_supabase.yml
+++ b/.github/workflows/service_test_supabase.yml
@@ -44,8 +44,6 @@ jobs:
- uses: actions/checkout@v3
- name: Setup Rust toolchain
uses: ./.github/actions/setup
- with:
- need-nextest: true
- name: Load secret
id: op-load-secret
@@ -62,4 +60,4 @@ jobs:
- name: Test
shell: bash
working-directory: core
- run: cargo nextest run supabase --features=services-supabase
+ run: cargo test supabase --features=services-supabase
diff --git a/bindings/c/src/lib.rs b/bindings/c/src/lib.rs
index 8569a7a01..7c1c88f43 100644
--- a/bindings/c/src/lib.rs
+++ b/bindings/c/src/lib.rs
@@ -28,10 +28,13 @@ use std::str::FromStr;
use ::opendal as od;
use crate::error::opendal_code;
-use crate::result::{opendal_result_is_exist, opendal_result_read, opendal_result_stat};
-use crate::types::{
- opendal_bytes, opendal_metadata, opendal_operator_options, opendal_operator_ptr,
-};
+use crate::result::opendal_result_is_exist;
+use crate::result::opendal_result_read;
+use crate::result::opendal_result_stat;
+use crate::types::opendal_bytes;
+use crate::types::opendal_metadata;
+use crate::types::opendal_operator_options;
+use crate::types::opendal_operator_ptr;
/// \brief Construct an operator based on `scheme` and `options`
///
diff --git a/bindings/haskell/src/lib.rs b/bindings/haskell/src/lib.rs
index 30c5c2df8..812defcc6 100644
--- a/bindings/haskell/src/lib.rs
+++ b/bindings/haskell/src/lib.rs
@@ -18,8 +18,6 @@
mod result;
mod types;
-use ::opendal as od;
-use result::FFIResult;
use std::collections::HashMap;
use std::ffi::CStr;
use std::mem;
@@ -27,6 +25,9 @@ use std::os::raw::c_char;
use std::str::FromStr;
use types::{ByteSlice, Metadata};
+use ::opendal as od;
+use result::FFIResult;
+
/// # Safety
///
/// * The `keys`, `values`, `len` are valid from `HashMap`.
diff --git a/bindings/haskell/src/result.rs b/bindings/haskell/src/result.rs
index 5916c48e7..fd770d8f5 100644
--- a/bindings/haskell/src/result.rs
+++ b/bindings/haskell/src/result.rs
@@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-use std::{
- ffi::{c_char, CString},
- ptr,
-};
+use std::ffi::c_char;
+use std::ffi::CString;
+use std::ptr;
use ::opendal as od;
diff --git a/bindings/lua/src/lib.rs b/bindings/lua/src/lib.rs
index 3a264fa64..1a198602b 100644
--- a/bindings/lua/src/lib.rs
+++ b/bindings/lua/src/lib.rs
@@ -15,7 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-use std::{collections::HashMap, str::FromStr};
+use std::collections::HashMap;
+use std::str::FromStr;
use ::opendal as od;
use mlua::prelude::*;
diff --git a/core/src/layers/mod.rs b/core/src/layers/mod.rs
index 586c37772..85fcdca0a 100644
--- a/core/src/layers/mod.rs
+++ b/core/src/layers/mod.rs
@@ -79,8 +79,7 @@ mod oteltrace;
#[cfg(feature = "layers-throttle")]
mod throttle;
-#[cfg(feature = "layers-throttle")]
-pub use self::throttle::ThrottleLayer;
-
#[cfg(feature = "layers-otel-trace")]
pub use self::oteltrace::OtelTraceLayer;
+#[cfg(feature = "layers-throttle")]
+pub use self::throttle::ThrottleLayer;
diff --git a/core/src/layers/throttle.rs b/core/src/layers/throttle.rs
index acf53daac..871d4ec18 100644
--- a/core/src/layers/throttle.rs
+++ b/core/src/layers/throttle.rs
@@ -18,15 +18,20 @@
use std::io::SeekFrom;
use std::num::NonZeroU32;
use std::sync::Arc;
-use std::task::{Context, Poll};
+use std::task::Context;
+use std::task::Poll;
use std::thread;
use async_trait::async_trait;
use bytes::Bytes;
-use governor::clock::{Clock, DefaultClock};
+use governor::clock::Clock;
+use governor::clock::DefaultClock;
use governor::middleware::NoOpMiddleware;
-use governor::state::{InMemoryState, NotKeyed};
-use governor::{NegativeMultiDecision, Quota, RateLimiter};
+use governor::state::InMemoryState;
+use governor::state::NotKeyed;
+use governor::NegativeMultiDecision;
+use governor::Quota;
+use governor::RateLimiter;
use crate::raw::oio::Streamer;
use crate::raw::*;
diff --git a/core/src/raw/oio/into_stream/from_futures_stream.rs b/core/src/raw/oio/into_stream/from_futures_stream.rs
index cd014cc90..c04379f71 100644
--- a/core/src/raw/oio/into_stream/from_futures_stream.rs
+++ b/core/src/raw/oio/into_stream/from_futures_stream.rs
@@ -15,7 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-use std::task::{Context, Poll};
+use std::task::Context;
+use std::task::Poll;
use bytes::Bytes;
use futures::TryStreamExt;
diff --git a/core/src/raw/oio/stream.rs b/core/src/raw/oio/stream.rs
index 39a1fd1e4..6ae6a732c 100644
--- a/core/src/raw/oio/stream.rs
+++ b/core/src/raw/oio/stream.rs
@@ -15,10 +15,12 @@
// specific language governing permissions and limitations
// under the License.
+use std::task::Context;
+use std::task::Poll;
+
use bytes::Bytes;
use crate::*;
-use std::task::{Context, Poll};
/// Streamer is a type erased [`Stream`].
pub type Streamer = Box<dyn Stream>;
diff --git a/core/src/services/dropbox/backend.rs b/core/src/services/dropbox/backend.rs
index 1c7784e1f..da8e3e690 100644
--- a/core/src/services/dropbox/backend.rs
+++ b/core/src/services/dropbox/backend.rs
@@ -15,36 +15,21 @@
// specific language governing permissions and limitations
// under the License.
+use std::fmt::Debug;
+use std::sync::Arc;
+
use async_trait::async_trait;
use http::StatusCode;
-use std::{fmt::Debug, sync::Arc};
-
-use super::{core::DropboxCore, error::parse_error, writer::DropboxWriter};
-use crate::{
- raw::{
- parse_into_metadata, Accessor, AccessorInfo, HttpClient, IncomingAsyncBody, OpDelete,
- OpRead, OpWrite, RpDelete, RpRead, RpWrite,
- },
- types::Result,
- Capability, Error, ErrorKind,
-};
+use super::core::DropboxCore;
+use super::error::parse_error;
+use super::writer::DropboxWriter;
+use crate::raw::*;
+use crate::*;
#[derive(Clone, Debug)]
pub struct DropboxBackend {
- core: Arc<DropboxCore>,
-}
-
-impl DropboxBackend {
- pub(crate) fn new(root: String, access_token: String, http_client: HttpClient) -> Self {
- DropboxBackend {
- core: Arc::new(DropboxCore {
- token: access_token,
- client: http_client,
- root,
- }),
- }
- }
+ pub core: Arc<DropboxCore>,
}
#[async_trait]
@@ -59,7 +44,7 @@ impl Accessor for DropboxBackend {
fn info(&self) -> AccessorInfo {
let mut ma = AccessorInfo::default();
- ma.set_scheme(crate::Scheme::Dropbox)
+ ma.set_scheme(Scheme::Dropbox)
.set_root(&self.core.root)
.set_capability(Capability {
read: true,
diff --git a/core/src/services/dropbox/builder.rs b/core/src/services/dropbox/builder.rs
index 10bb9cda8..b8fd8efae 100644
--- a/core/src/services/dropbox/builder.rs
+++ b/core/src/services/dropbox/builder.rs
@@ -15,13 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-use std::fmt::{Debug, Formatter};
-
use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
use super::backend::DropboxBackend;
-use crate::raw::{normalize_root, HttpClient};
-use crate::Scheme;
+use super::core::DropboxCore;
+use crate::raw::*;
use crate::*;
/// [Dropbox](https://www.dropbox.com/) backend support.
@@ -54,28 +55,17 @@ use crate::*;
///
/// ```
/// use anyhow::Result;
+/// use opendal::raw::OpWrite;
/// use opendal::services::Dropbox;
/// use opendal::Operator;
-/// use opendal::raw::OpWrite;
///
/// #[tokio::main]
/// async fn main() -> Result<()> {
/// // create backend builder
/// let mut builder = Dropbox::default();
-///
-/// builder.access_token("x").root("/");
+/// builder.access_token("<token>");
///
/// let op: Operator = Operator::new(builder)?.finish();
-/// let content = "who are you";
-///
-///
-/// let write = op.write_with("abc2.txt", content)
-/// .content_type("application/octet-stream")
-/// .content_length(content.len() as u64).await?;
-/// let read = op.read("abc2.txt").await?;
-/// let s = String::from_utf8(read).unwrap();
-/// println!("{}", s);
-/// let delete = op.delete("abc.txt").await?;
/// Ok(())
/// }
/// ```
@@ -138,12 +128,22 @@ impl Builder for DropboxBuilder {
.with_context("service", Scheme::Dropbox)
})?
};
- match self.access_token.clone() {
- Some(access_token) => Ok(DropboxBackend::new(root, access_token, client)),
- None => Err(Error::new(
- ErrorKind::ConfigInvalid,
- "access_token is required",
- )),
- }
+ let token = match self.access_token.clone() {
+ Some(access_token) => access_token,
+ None => {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "access_token is required",
+ ))
+ }
+ };
+
+ Ok(DropboxBackend {
+ core: Arc::new(DropboxCore {
+ root,
+ token,
+ client,
+ }),
+ })
}
}
diff --git a/core/src/services/dropbox/core.rs b/core/src/services/dropbox/core.rs
index cdaa13e4c..31b597d4c 100644
--- a/core/src/services/dropbox/core.rs
+++ b/core/src/services/dropbox/core.rs
@@ -19,17 +19,21 @@ use std::default::Default;
use std::fmt::Debug;
use std::fmt::Formatter;
-use crate::raw::{build_rooted_abs_path, new_json_serialize_error, HttpClient};
-
use bytes::Bytes;
+use http::header;
use http::request::Builder;
-use http::{header, Request, Response};
-use serde::{Deserialize, Serialize};
+use http::Request;
+use http::Response;
+use serde::Deserialize;
+use serde::Serialize;
-use crate::{
- raw::{new_request_build_error, AsyncBody, IncomingAsyncBody},
- types::Result,
-};
+use crate::raw::build_rooted_abs_path;
+use crate::raw::new_json_serialize_error;
+use crate::raw::new_request_build_error;
+use crate::raw::AsyncBody;
+use crate::raw::HttpClient;
+use crate::raw::IncomingAsyncBody;
+use crate::types::Result;
pub struct DropboxCore {
pub token: String,
@@ -50,9 +54,8 @@ impl DropboxCore {
let download_args = DropboxDownloadArgs {
path: build_rooted_abs_path(&self.root, path),
};
- let request_payload = serde_json::to_string(&download_args)
- .map_err(new_json_serialize_error)
- .unwrap();
+ let request_payload =
+ serde_json::to_string(&download_args).map_err(new_json_serialize_error)?;
let request = self
.build_auth_header(Request::post(&url))
.header("Dropbox-API-Arg", request_payload)
@@ -84,9 +87,7 @@ impl DropboxCore {
.build_auth_header(request_builder)
.header(
"Dropbox-API-Arg",
- serde_json::to_string(&args)
- .map_err(new_json_serialize_error)
- .unwrap(),
+ serde_json::to_string(&args).map_err(new_json_serialize_error)?,
)
.body(body)
.map_err(new_request_build_error)?;
@@ -99,14 +100,13 @@ impl DropboxCore {
let args = DropboxDeleteArgs {
path: build_rooted_abs_path(&self.root, path),
};
+
+ let bs = Bytes::from(serde_json::to_string(&args).map_err(new_json_serialize_error)?);
+
let request = self
.build_auth_header(Request::post(&url))
.header(header::CONTENT_TYPE, "application/json")
- .body(AsyncBody::Bytes(Bytes::from(
- serde_json::to_string(&args)
- .map_err(new_json_serialize_error)
- .unwrap(),
- )))
+ .body(AsyncBody::Bytes(bs))
.map_err(new_request_build_error)?;
self.client.send(request).await
}
diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs
index 681857011..b4067305f 100644
--- a/core/src/services/ftp/backend.rs
+++ b/core/src/services/ftp/backend.rs
@@ -45,7 +45,6 @@ use crate::raw::*;
use crate::*;
/// FTP and FTPS services support.
-///
#[doc = include_str!("docs.md")]
#[derive(Default)]
pub struct FtpBuilder {
diff --git a/core/src/services/hdfs/backend.rs b/core/src/services/hdfs/backend.rs
index 4c075715d..40e479074 100644
--- a/core/src/services/hdfs/backend.rs
+++ b/core/src/services/hdfs/backend.rs
@@ -33,7 +33,6 @@ use crate::raw::*;
use crate::*;
/// [Hadoop Distributed File System (HDFS™)](https://hadoop.apache.org/) support.
-///
#[doc = include_str!("docs.md")]
#[derive(Debug, Default)]
pub struct HdfsBuilder {
diff --git a/core/src/services/ipfs/backend.rs b/core/src/services/ipfs/backend.rs
index 8c8e15672..32ee1d0b3 100644
--- a/core/src/services/ipfs/backend.rs
+++ b/core/src/services/ipfs/backend.rs
@@ -33,7 +33,6 @@ use crate::raw::*;
use crate::*;
/// IPFS file system support based on [IPFS HTTP Gateway](https://docs.ipfs.tech/concepts/ipfs-gateway/).
-///
#[doc = include_str!("docs.md")]
#[derive(Default, Clone, Debug)]
pub struct IpfsBuilder {
diff --git a/core/src/services/ipmfs/backend.rs b/core/src/services/ipmfs/backend.rs
index da304e899..8dd05e6fb 100644
--- a/core/src/services/ipmfs/backend.rs
+++ b/core/src/services/ipmfs/backend.rs
@@ -34,7 +34,6 @@ use crate::raw::*;
use crate::*;
/// IPFS Mutable File System (IPMFS) backend.
-///
#[doc = include_str!("docs.md")]
#[derive(Clone)]
pub struct IpmfsBackend {
diff --git a/core/src/services/memory/backend.rs b/core/src/services/memory/backend.rs
index 716874e7c..2c0a08be2 100644
--- a/core/src/services/memory/backend.rs
+++ b/core/src/services/memory/backend.rs
@@ -26,7 +26,6 @@ use crate::raw::adapters::typed_kv;
use crate::*;
/// In memory service support. (BTreeMap Based)
-///
#[doc = include_str!("docs.md")]
#[derive(Default)]
pub struct MemoryBuilder {
diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs
index 37a1bd8a4..d27ec1d41 100644
--- a/core/src/services/mod.rs
+++ b/core/src/services/mod.rs
@@ -161,7 +161,6 @@ pub use gdrive::Gdrive;
mod dropbox;
#[cfg(feature = "services-dropbox")]
pub use dropbox::Dropbox;
-
#[cfg(feature = "services-webhdfs")]
pub use webhdfs::Webhdfs;
diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs
index 78d15d147..f1164ee3b 100644
--- a/core/src/services/obs/backend.rs
+++ b/core/src/services/obs/backend.rs
@@ -27,10 +27,11 @@ use reqsign::HuaweicloudObsConfig;
use reqsign::HuaweicloudObsCredentialLoader;
use reqsign::HuaweicloudObsSigner;
+use super::appender::ObsAppender;
+use super::core::ObsCore;
use super::error::parse_error;
use super::pager::ObsPager;
use super::writer::ObsWriter;
-use super::{appender::ObsAppender, core::ObsCore};
use crate::raw::*;
use crate::*;
diff --git a/core/src/services/obs/core.rs b/core/src/services/obs/core.rs
index 29e2d783b..f41e1599f 100644
--- a/core/src/services/obs/core.rs
+++ b/core/src/services/obs/core.rs
@@ -19,11 +19,12 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::time::Duration;
+use http::header::CACHE_CONTROL;
+use http::header::CONTENT_DISPOSITION;
use http::header::CONTENT_LENGTH;
use http::header::CONTENT_TYPE;
use http::header::IF_MATCH;
use http::header::IF_NONE_MATCH;
-use http::header::{CACHE_CONTROL, CONTENT_DISPOSITION};
use http::Request;
use http::Response;
use reqsign::HuaweicloudObsCredential;
diff --git a/core/src/services/redis/backend.rs b/core/src/services/redis/backend.rs
index dfac88778..e1fde6fdf 100644
--- a/core/src/services/redis/backend.rs
+++ b/core/src/services/redis/backend.rs
@@ -40,7 +40,6 @@ const DEFAULT_REDIS_ENDPOINT: &str = "tcp://127.0.0.1:6379";
const DEFAULT_REDIS_PORT: u16 = 6379;
/// [Redis](https://redis.io/) services support.
-///
#[doc = include_str!("docs.md")]
#[derive(Clone, Default)]
pub struct RedisBuilder {
diff --git a/core/src/services/rocksdb/backend.rs b/core/src/services/rocksdb/backend.rs
index e167b3073..fc1492664 100644
--- a/core/src/services/rocksdb/backend.rs
+++ b/core/src/services/rocksdb/backend.rs
@@ -28,7 +28,6 @@ use crate::Result;
use crate::*;
/// RocksDB service support.
-///
#[doc = include_str!("docs.md")]
#[derive(Clone, Default, Debug)]
pub struct RocksdbBuilder {
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 90f0acc98..b5c780c50 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -58,7 +58,6 @@ const DEFAULT_WRITE_MIN_SIZE: usize = 8 * 1024 * 1024;
const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000;
/// Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support.
/// For more information about s3-compatible services, refer to [Compatible Services](#compatible-services).
-///
#[doc = include_str!("docs.md")]
#[doc = include_str!("compatible_services.md")]
#[derive(Default)]
diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs
index ed6e39ef6..4cb68f95a 100644
--- a/core/src/services/sftp/backend.rs
+++ b/core/src/services/sftp/backend.rs
@@ -44,7 +44,6 @@ use crate::*;
///
/// For example, the default value is 255 in macOS, and 1024 in linux. If you want to open
/// lots of files, you should pay attention to close the file after using it.
-///
#[doc = include_str!("docs.md")]
#[derive(Default)]
pub struct SftpBuilder {
diff --git a/core/src/services/sled/backend.rs b/core/src/services/sled/backend.rs
index 17a6527a7..43f855b5b 100644
--- a/core/src/services/sled/backend.rs
+++ b/core/src/services/sled/backend.rs
@@ -29,7 +29,6 @@ use crate::Scheme;
use crate::*;
/// Sled service support.
-///
#[doc = include_str!("docs.md")]
#[derive(Default)]
pub struct SledBuilder {
diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs
index 4f90b9d2c..ccfc145ab 100644
--- a/core/src/services/webhdfs/backend.rs
+++ b/core/src/services/webhdfs/backend.rs
@@ -41,7 +41,6 @@ use crate::*;
const WEBHDFS_DEFAULT_ENDPOINT: &str = "http://127.0.0.1:9870";
/// [WebHDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html)'s REST API support.
-///
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct WebhdfsBuilder {
diff --git a/core/src/services/webhdfs/error.rs b/core/src/services/webhdfs/error.rs
index 93c9bbd17..eafd36bee 100644
--- a/core/src/services/webhdfs/error.rs
+++ b/core/src/services/webhdfs/error.rs
@@ -80,9 +80,8 @@ mod tests {
use futures::stream;
use serde_json::from_reader;
- use crate::raw::oio::into_stream;
-
use super::*;
+ use crate::raw::oio::into_stream;
/// Error response example from https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Error%20Responses
#[tokio::test]