You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/05/22 09:32:16 UTC
[incubator-opendal] 02/02: Add via_map
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch refactor-builder
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 688c26f4975c9c14473de709c160e0686165cfb7
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon May 22 17:32:02 2023 +0800
Add via_map
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
bindings/nodejs/src/lib.rs | 61 +-------------------
bindings/python/src/lib.rs | 47 +--------------
bindings/ruby/src/lib.rs | 60 ++------------------
core/src/types/operator/builder.rs | 113 +++++++++++++++++++++++++++++++++++++
core/src/types/scheme.rs | 5 +-
5 files changed, 125 insertions(+), 161 deletions(-)
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
index 92c3f16b..f3aecfba 100644
--- a/bindings/nodejs/src/lib.rs
+++ b/bindings/nodejs/src/lib.rs
@@ -25,63 +25,6 @@ use std::time::Duration;
use futures::TryStreamExt;
use napi::bindgen_prelude::*;
-fn build_operator(
- scheme: opendal::Scheme,
- map: HashMap<String, String>,
-) -> Result<opendal::Operator> {
- use opendal::services::*;
-
- let op = match scheme {
- opendal::Scheme::Azblob => opendal::Operator::from_map::<Azblob>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Azdfs => opendal::Operator::from_map::<Azdfs>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Fs => opendal::Operator::from_map::<Fs>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Gcs => opendal::Operator::from_map::<Gcs>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Ghac => opendal::Operator::from_map::<Ghac>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Http => opendal::Operator::from_map::<Http>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Ipmfs => opendal::Operator::from_map::<Ipmfs>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Memory => opendal::Operator::from_map::<Memory>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Obs => opendal::Operator::from_map::<Obs>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Oss => opendal::Operator::from_map::<Oss>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::S3 => opendal::Operator::from_map::<S3>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Webdav => opendal::Operator::from_map::<Webdav>(map)
- .map_err(format_napi_error)?
- .finish(),
- opendal::Scheme::Webhdfs => opendal::Operator::from_map::<Webhdfs>(map)
- .map_err(format_napi_error)?
- .finish(),
- _ => {
- return Err(format_napi_error(opendal::Error::new(
- opendal::ErrorKind::Unexpected,
- "not supported scheme",
- )))
- }
- };
-
- Ok(op)
-}
-
#[napi]
pub struct Operator(opendal::Operator);
@@ -97,7 +40,9 @@ impl Operator {
.map_err(format_napi_error)?;
let options = options.unwrap_or_default();
- build_operator(scheme, options).map(Operator)
+ opendal::Operator::via_map(scheme, options)
+ .map_err(format_napi_error)
+ .map(Operator)
}
/// Get current path's metadata **without cache** directly.
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index 44a486ef..7d58d277 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -62,52 +62,7 @@ fn build_operator(
map: HashMap<String, String>,
layers: Vec<layers::Layer>,
) -> PyResult<od::Operator> {
- use od::services::*;
-
- let op = match scheme {
- od::Scheme::Azblob => od::Operator::from_map::<Azblob>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Azdfs => od::Operator::from_map::<Azdfs>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Fs => od::Operator::from_map::<Fs>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Gcs => od::Operator::from_map::<Gcs>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Ghac => od::Operator::from_map::<Ghac>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Http => od::Operator::from_map::<Http>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Ipmfs => od::Operator::from_map::<Ipmfs>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Memory => od::Operator::from_map::<Memory>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Obs => od::Operator::from_map::<Obs>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Oss => od::Operator::from_map::<Oss>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::S3 => od::Operator::from_map::<S3>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Webdav => od::Operator::from_map::<Webdav>(map)
- .map_err(format_pyerr)?
- .finish(),
- od::Scheme::Webhdfs => od::Operator::from_map::<Webhdfs>(map)
- .map_err(format_pyerr)?
- .finish(),
- _ => Err(PyNotImplementedError::new_err(format!(
- "unsupported scheme `{scheme}`"
- )))?,
- };
+ let op = od::Operator::via_map(scheme, map).map_err(format_pyerr)?;
add_layers(op, layers)
}
diff --git a/bindings/ruby/src/lib.rs b/bindings/ruby/src/lib.rs
index 3a3c404d..f69f52ba 100644
--- a/bindings/ruby/src/lib.rs
+++ b/bindings/ruby/src/lib.rs
@@ -29,60 +29,6 @@ use magnus::Error;
use magnus::RString;
use opendal as od;
-fn build_operator(scheme: od::Scheme, map: HashMap<String, String>) -> Result<od::Operator> {
- use od::services::*;
-
- let op = match scheme {
- od::Scheme::Azblob => od::Operator::from_map::<Azblob>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Azdfs => od::Operator::from_map::<Azdfs>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Fs => od::Operator::from_map::<Fs>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Gcs => od::Operator::from_map::<Gcs>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Ghac => od::Operator::from_map::<Ghac>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Http => od::Operator::from_map::<Http>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Ipmfs => od::Operator::from_map::<Ipmfs>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Memory => od::Operator::from_map::<Memory>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Obs => od::Operator::from_map::<Obs>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Oss => od::Operator::from_map::<Oss>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::S3 => od::Operator::from_map::<S3>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Webdav => od::Operator::from_map::<Webdav>(map)
- .map_err(format_magnus_error)?
- .finish(),
- od::Scheme::Webhdfs => od::Operator::from_map::<Webhdfs>(map)
- .map_err(format_magnus_error)?
- .finish(),
- _ => {
- return Err(format_magnus_error(od::Error::new(
- od::ErrorKind::Unexpected,
- "not supported scheme",
- )))
- }
- };
-
- Ok(op)
-}
-
#[magnus::wrap(class = "OpenDAL::Operator", free_immediately, size)]
#[derive(Clone, Debug)]
pub struct Operator(od::BlockingOperator);
@@ -95,7 +41,11 @@ impl Operator {
})
.map_err(format_magnus_error)?;
let options = options.unwrap_or_default();
- Ok(Operator(build_operator(scheme, options)?.blocking()))
+
+ let op = od::Operator::via_map(scheme, options)
+ .map_err(format_magnus_error)?
+ .blocking();
+ Ok(Operator(op))
}
/// Read the whole path into string.
diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs
index 3a300370..b1f8523a 100644
--- a/core/src/types/operator/builder.rs
+++ b/core/src/types/operator/builder.rs
@@ -82,6 +82,15 @@ impl Operator {
/// Create a new operator from given map.
///
+ /// # Notes
+ ///
+ /// from_map is using static dispatch layers which is zero cost. via_map is
+ /// using dynamic dispatch layers which has a bit runtime overhead with an
+ /// extra vtable lookup and unable to inline. But from_map requires generic
+ /// type parameter which is not always easy to be used.
+ ///
+ /// # Examples
+ ///
/// ```
/// # use anyhow::Result;
/// use std::collections::HashMap;
@@ -110,6 +119,110 @@ impl Operator {
Ok(OperatorBuilder::new(acc))
}
+ /// Create a new operator from given shceme and map.
+ ///
+ /// # Notes
+ ///
+ /// from_map is using static dispatch layers which is zero cost. via_map is
+ /// using dynamic dispatch layers which has a bit runtime overhead with an
+ /// extra vtable lookup and unable to inline. But from_map requires generic
+ /// type parameter which is not always easy to be used.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use anyhow::Result;
+ /// use std::collections::HashMap;
+ ///
+ /// use opendal::Scheme;
+ /// use opendal::Operator;
+ /// #[tokio::main]
+ /// async fn main() -> Result<()> {
+ /// let map = HashMap::from([
+ /// // Set the root for fs, all operations will happen under this root.
+ /// //
+ /// // NOTE: the root must be absolute path.
+ /// ("root".to_string(), "/tmp".to_string()),
+ /// ]);
+ ///
+ /// // Build an `Operator` to start operating the storage.
+ /// let op: Operator = Operator::via_map(Scheme::Fs, map)?.finish();
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub fn via_map(scheme: Scheme, map: HashMap<String, String>) -> Result<Operator> {
+ let op = match scheme {
+ #[cfg(feature = "services-azblob")]
+ Scheme::Azblob => Self::from_map::<services::Azblob>(map)?.finish(),
+ #[cfg(feature = "services-azdfs")]
+ Scheme::Azdfs => Self::from_map::<services::Azdfs>(map)?.finish(),
+ #[cfg(feature = "services-cos")]
+ Scheme::Cos => Self::from_map::<services::Cos>(map)?.finish(),
+ #[cfg(feature = "services-dashmap")]
+ Scheme::Dashmap => Self::from_map::<services::Dashmap>(map)?.finish(),
+ #[cfg(feature = "services-fs")]
+ Scheme::Fs => Self::from_map::<services::Fs>(map)?.finish(),
+ #[cfg(feature = "services-ftp")]
+ Scheme::Ftp => Self::from_map::<services::Ftp>(map)?.finish(),
+ #[cfg(feature = "services-gcs")]
+ Scheme::Gcs => Self::from_map::<services::Gcs>(map)?.finish(),
+ #[cfg(feature = "services-ghac")]
+ Scheme::Ghac => Self::from_map::<services::Ghac>(map)?.finish(),
+ #[cfg(feature = "services-hdfs")]
+ Scheme::Hdfs => Self::from_map::<services::Hdfs>(map)?.finish(),
+ #[cfg(feature = "services-http")]
+ Scheme::Http => Self::from_map::<services::Http>(map)?.finish(),
+ #[cfg(feature = "services-ipfs")]
+ Scheme::Ipfs => Self::from_map::<services::Ipfs>(map)?.finish(),
+ #[cfg(feature = "services-ipmfs")]
+ Scheme::Ipmfs => Self::from_map::<services::Ipmfs>(map)?.finish(),
+ #[cfg(feature = "services-memcached")]
+ Scheme::Memcached => Self::from_map::<services::Memcached>(map)?.finish(),
+ #[cfg(feature = "services-memory")]
+ Scheme::Memory => Self::from_map::<services::Memory>(map)?.finish(),
+ #[cfg(feature = "services-moka")]
+ Scheme::Moka => Self::from_map::<services::Moka>(map)?.finish(),
+ #[cfg(feature = "services-obs")]
+ Scheme::Obs => Self::from_map::<services::Obs>(map)?.finish(),
+ #[cfg(feature = "services-onedrive")]
+ Scheme::Onedrive => Self::from_map::<services::Onedrive>(map)?.finish(),
+ #[cfg(feature = "services-gdrive")]
+ Scheme::Gdrive => Self::from_map::<services::Gdrive>(map)?.finish(),
+ #[cfg(feature = "services-oss")]
+ Scheme::Oss => Self::from_map::<services::Oss>(map)?.finish(),
+ #[cfg(feature = "services-redis")]
+ Scheme::Redis => Self::from_map::<services::Redis>(map)?.finish(),
+ #[cfg(feature = "services-rocksdb")]
+ Scheme::Rocksdb => Self::from_map::<services::Rocksdb>(map)?.finish(),
+ #[cfg(feature = "services-s3")]
+ Scheme::S3 => Self::from_map::<services::S3>(map)?.finish(),
+ #[cfg(feature = "services-sftp")]
+ Scheme::Sftp => Self::from_map::<services::Sftp>(map)?.finish(),
+ #[cfg(feature = "services-sled")]
+ Scheme::Sled => Self::from_map::<services::Sled>(map)?.finish(),
+ #[cfg(feature = "services-supabase")]
+ Scheme::Supabase => Self::from_map::<services::Supabase>(map)?.finish(),
+ #[cfg(feature = "services-vercel-artifacts")]
+ Scheme::VercelArtifacts => Self::from_map::<services::VercelArtifacts>(map)?.finish(),
+ #[cfg(feature = "services-wasabi")]
+ Scheme::Wasabi => Self::from_map::<services::Wasabi>(map)?.finish(),
+ #[cfg(feature = "services-webdav")]
+ Scheme::Webdav => Self::from_map::<services::Webdav>(map)?.finish(),
+ #[cfg(feature = "services-webhdfs")]
+ Scheme::Webhdfs => Self::from_map::<services::Webhdfs>(map)?.finish(),
+ v => {
+ return Err(Error::new(
+ ErrorKind::Unsupported,
+ "scheme is not enabled or supported",
+ )
+ .with_context("scheme", v))
+ }
+ };
+
+ Ok(op)
+ }
+
/// Create a new layer with dynamic dispatch.
///
/// # Notes
diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs
index edc1dde1..07a80843 100644
--- a/core/src/types/scheme.rs
+++ b/core/src/types/scheme.rs
@@ -41,6 +41,8 @@ pub enum Scheme {
Dashmap,
/// [fs][crate::services::Fs]: POSIX alike file system.
Fs,
+ /// [ftp][crate::services::Ftp]: FTP backend.
+ Ftp,
/// [gcs][crate::services::Gcs]: Google Cloud Storage backend.
Gcs,
/// [ghac][crate::services::Ghac]: GitHub Action Cache services.
@@ -49,8 +51,7 @@ pub enum Scheme {
Hdfs,
/// [http][crate::services::Http]: HTTP backend.
Http,
- /// [ftp][crate::services::Ftp]: FTP backend.
- Ftp,
+
/// [ipmfs][crate::services::Ipfs]: IPFS HTTP Gateway
Ipfs,
/// [ipmfs][crate::services::Ipmfs]: IPFS mutable file system