You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/06/26 02:51:50 UTC
[incubator-opendal] branch main updated: feat(core): implement service for Mini Moka (#2537)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 39303a508 feat(core): implement service for Mini Moka (#2537)
39303a508 is described below
commit 39303a508a6f114444a56ab71c3c9d025e2a593f
Author: Morris Tai <mo...@gmail.com>
AuthorDate: Sun Jun 25 22:51:45 2023 -0400
feat(core): implement service for Mini Moka (#2537)
* feat(service/mini-moka): add service for mini-moka
* feat(website): add service mini-moka to docusaurus website
* chore(service/mini-moka): add moka comparison table in comment
* fix: drop miss-added file
---
.env.example | 2 +
.gitignore | 3 +
Cargo.lock | 16 ++
README.md | 3 +-
bin/oli/Cargo.toml | 2 +
bin/oli/src/config/mod.rs | 6 +
core/Cargo.toml | 2 +
core/README.md | 1 +
core/benches/ops/utils.rs | 2 +
core/src/docs/features.md | 1 +
core/src/services/mini_moka/backend.rs | 205 +++++++++++++++++++++
core/src/services/mini_moka/mod.rs | 19 ++
core/src/services/mod.rs | 5 +
core/src/types/operator/builder.rs | 2 +
core/src/types/scheme.rs | 4 +
core/tests/behavior/main.rs | 2 +
.../HomepageFeatures/_feature_services.mdx | 3 +-
17 files changed, 276 insertions(+), 2 deletions(-)
diff --git a/.env.example b/.env.example
index cb3161c2b..58577aaf2 100644
--- a/.env.example
+++ b/.env.example
@@ -78,6 +78,8 @@ OPENDAL_SFTP_KNOWN_HOSTS_STRATEGY=<accept|add|strict>
OPENDAL_SLED_TEST=false
OPENDAL_SLED_DATADIR=/path/to/database
OPENDAL_SLED_TREE=sled-tree
+# mini-moka
+OPENDAL_MINI_MOKA_TEST=false
# moka
OPENDAL_MOKA_TEST=false
# ghac
diff --git a/.gitignore b/.gitignore
index 03f8a1624..d5fd6da84 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,6 @@ flamegraph.svg
perf.*
**/.DS_Store
+
+# Yarn Integrity file
+.yarn-integrity
diff --git a/Cargo.lock b/Cargo.lock
index e687a7952..48ebdc80e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2334,6 +2334,21 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+[[package]]
+name = "mini-moka"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cafc5ec7807288595f9c20c86e6ce6d262b722f61e0547fe7e6e6e6451b58d5"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "dashmap",
+ "skeptic",
+ "smallvec",
+ "tagptr",
+ "triomphe",
+]
+
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@@ -2757,6 +2772,7 @@ dependencies = [
"madsim",
"md-5",
"metrics",
+ "mini-moka",
"minitrace",
"moka",
"once_cell",
diff --git a/README.md b/README.md
index 9ca10ebed..e1e0806a9 100644
--- a/README.md
+++ b/README.md
@@ -94,7 +94,8 @@ Major components of the project include:
- ghac: [Github Action Cache](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows) Service
- memcached: [Memcached](https://memcached.org/) service
-- moka: [moka](https://github.com/moka-rs/moka) backend
+- mini_moka: [Mini Moka](https://github.com/moka-rs/mini-moka) backend
+- moka: [Moka](https://github.com/moka-rs/moka) backend
- vercel_artifacts: [Vercel Remote Caching](https://vercel.com/docs/concepts/monorepos/remote-caching) Service *being worked on*
</details>
diff --git a/bin/oli/Cargo.toml b/bin/oli/Cargo.toml
index 9efc75e17..b2756f010 100644
--- a/bin/oli/Cargo.toml
+++ b/bin/oli/Cargo.toml
@@ -40,6 +40,8 @@ services-hdfs = ["opendal/services-hdfs"]
services-ipfs = ["opendal/services-ipfs"]
# Enable services memcached support
services-memcached = ["opendal/services-memcached"]
+# Enable services mini-moka support
+services-mini-moka = ["opendal/services-mini-moka"]
# Enable services moka support
services-moka = ["opendal/services-moka"]
# Enable services redis support
diff --git a/bin/oli/src/config/mod.rs b/bin/oli/src/config/mod.rs
index d9e30cd99..2a155e436 100644
--- a/bin/oli/src/config/mod.rs
+++ b/bin/oli/src/config/mod.rs
@@ -207,6 +207,12 @@ impl Config {
path,
)),
// ignore the memory backend
+ #[cfg(feature = "services-mini-moka")]
+ Scheme::MiniMoka => Ok((
+ Operator::from_map::<services::MiniMoka>(profile.clone())?.finish(),
+ path,
+ )),
+ // ignore the memory backend
#[cfg(feature = "services-moka")]
Scheme::Moka => Ok((
Operator::from_map::<services::Moka>(profile.clone())?.finish(),
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 154bf6933..a0b816226 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -127,6 +127,7 @@ services-ipfs = ["dep:prost"]
services-ipmfs = []
services-memcached = ["dep:bb8"]
services-memory = []
+services-mini-moka = ["dep:mini-moka"]
services-moka = ["dep:moka"]
services-obs = [
"dep:reqsign",
@@ -194,6 +195,7 @@ madsim = { version = "0.2.21", optional = true }
md-5 = "0.10"
metrics = { version = "0.20", optional = true }
minitrace = { version = "0.4.0", optional = true }
+mini-moka = { version = "0.10", optional = true }
moka = { version = "0.10", optional = true, features = ["future"] }
once_cell = "1"
openssh = { version = "0.9.9", optional = true }
diff --git a/core/README.md b/core/README.md
index ca692bab2..625c7c010 100644
--- a/core/README.md
+++ b/core/README.md
@@ -30,6 +30,7 @@
- [ipmfs](https://docs.rs/opendal/latest/opendal/services/struct.Ipmfs.html): [InterPlanetary File System](https://ipfs.tech/) MFS API support.
- [memcached](https://docs.rs/opendal/latest/opendal/services/struct.Memcached.html): [Memcached](https://memcached.org/) service support.
- [memory](https://docs.rs/opendal/latest/opendal/services/struct.Memory.html): In memory backend.
+- [mini_moka](https://docs.rs/opendal/latest/opendal/services/struct.MiniMoka.html): [mini-moka](https://github.com/moka-rs/mini-moka) backend support.
- [moka](https://docs.rs/opendal/latest/opendal/services/struct.Moka.html): [moka](https://github.com/moka-rs/moka) backend support.
- [obs](https://docs.rs/opendal/latest/opendal/services/struct.Obs.html): [Huawei Cloud Object Storage](https://www.huaweicloud.com/intl/en-us/product/obs.html) Service (OBS).
- [oss](https://docs.rs/opendal/latest/opendal/services/struct.Oss.html): [Aliyun Object Storage Service](https://www.aliyun.com/product/oss) (OSS).
diff --git a/core/benches/ops/utils.rs b/core/benches/ops/utils.rs
index 52a9a61ed..f8d9af2d7 100644
--- a/core/benches/ops/utils.rs
+++ b/core/benches/ops/utils.rs
@@ -54,6 +54,8 @@ pub fn services() -> Vec<(&'static str, Option<Operator>)> {
("fs", service::<services::Fs>()),
("s3", service::<services::S3>()),
("memory", service::<services::Memory>()),
+ #[cfg(feature = "services-mini-moka")]
+ ("mini-moka", service::<services::MiniMoka>()),
#[cfg(feature = "services-moka")]
("moka", service::<services::Moka>()),
#[cfg(feature = "services-redis")]
diff --git a/core/src/docs/features.md b/core/src/docs/features.md
index 0f3ed4e40..f3633a036 100644
--- a/core/src/docs/features.md
+++ b/core/src/docs/features.md
@@ -12,6 +12,7 @@
- `services-ftp`: Enable ftp service support.
- `services-hdfs`: Enable hdfs service support.
- `services-memcached`: Enable memcached service support.
+- `services-mini-moka`: Enable mini-moka service support.
- `services-moka`: Enable moka service support.
- `services-ipfs`: Enable ipfs service support.
- `services-redis`: Enable redis service support.
diff --git a/core/src/services/mini_moka/backend.rs b/core/src/services/mini_moka/backend.rs
new file mode 100644
index 000000000..942fe909f
--- /dev/null
+++ b/core/src/services/mini_moka/backend.rs
@@ -0,0 +1,205 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use log::debug;
+use mini_moka::sync::Cache;
+use mini_moka::sync::CacheBuilder;
+
+use crate::raw::adapters::typed_kv;
+use crate::*;
+
+/// [mini-moka](https://github.com/moka-rs/mini-moka) backend support.
+///
+/// # Capabilities
+///
+/// This service can be used to:
+///
+/// - [x] stat
+/// - [x] read
+/// - [x] write
+/// - [x] create_dir
+/// - [x] delete
+/// - [ ] copy
+/// - [ ] rename
+/// - [ ] list
+/// - [ ] ~~scan~~
+/// - [ ] presign
+/// - [ ] blocking
+///
+/// # Notes
+///
+/// To better assist you in choosing the right cache for your use case,
+/// Here's a comparison table with [moka](https://github.com/moka-rs/moka#choosing-the-right-cache-for-your-use-case)
+#[derive(Default, Debug)]
+pub struct MiniMokaBuilder {
+ /// Sets the max capacity of the cache.
+ ///
+ /// Refer to [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity)
+ max_capacity: Option<u64>,
+ /// Sets the time to live of the cache.
+ ///
+ /// Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live)
+ time_to_live: Option<Duration>,
+ /// Sets the time to idle of the cache.
+ ///
+ /// Refer to [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle)
+ time_to_idle: Option<Duration>,
+}
+
+impl MiniMokaBuilder {
+ /// Sets the max capacity of the cache.
+ ///
+ /// Refer to [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity)
+ pub fn max_capacity(&mut self, v: u64) -> &mut Self {
+ if v != 0 {
+ self.max_capacity = Some(v);
+ }
+ self
+ }
+
+ /// Sets the time to live of the cache.
+ ///
+ /// Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live)
+ pub fn time_to_live(&mut self, v: Duration) -> &mut Self {
+ if !v.is_zero() {
+ self.time_to_live = Some(v);
+ }
+ self
+ }
+
+ /// Sets the time to idle of the cache.
+ ///
+ /// Refer to [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle)
+ pub fn time_to_idle(&mut self, v: Duration) -> &mut Self {
+ if !v.is_zero() {
+ self.time_to_idle = Some(v);
+ }
+ self
+ }
+}
+
+impl Builder for MiniMokaBuilder {
+ const SCHEME: Scheme = Scheme::MiniMoka;
+ type Accessor = MiniMokaBackend;
+
+ fn from_map(map: HashMap<String, String>) -> Self {
+ let mut builder = MiniMokaBuilder::default();
+
+ map.get("max_capacity")
+ .map(|v| v.parse::<u64>().map(|v| builder.max_capacity(v)));
+ map.get("time_to_live").map(|v| {
+ v.parse::<u64>()
+ .map(|v| builder.time_to_live(Duration::from_secs(v)))
+ });
+ map.get("time_to_idle").map(|v| {
+ v.parse::<u64>()
+ .map(|v| builder.time_to_idle(Duration::from_secs(v)))
+ });
+ builder
+ }
+
+ fn build(&mut self) -> Result<Self::Accessor> {
+ debug!("backend build started: {:?}", &self);
+
+ let mut builder: CacheBuilder<String, typed_kv::Value, _> = Cache::builder();
+ // Use entries' bytes as capacity weigher.
+ builder = builder.weigher(|k, v| (k.len() + v.size()) as u32);
+ if let Some(v) = self.max_capacity {
+ builder = builder.max_capacity(v)
+ }
+ if let Some(v) = self.time_to_live {
+ builder = builder.time_to_live(v)
+ }
+ if let Some(v) = self.time_to_idle {
+ builder = builder.time_to_idle(v)
+ }
+
+ debug!("backend build finished: {:?}", &self);
+ Ok(MiniMokaBackend::new(Adapter {
+ inner: builder.build(),
+ }))
+ }
+}
+
+/// Backend is used to serve `Accessor` support in mini-moka.
+pub type MiniMokaBackend = typed_kv::Backend<Adapter>;
+
+#[derive(Clone)]
+pub struct Adapter {
+ inner: Cache<String, typed_kv::Value>,
+}
+
+impl Debug for Adapter {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Adapter")
+ .field("size", &self.inner.weighted_size())
+ .field("count", &self.inner.entry_count())
+ .finish()
+ }
+}
+
+#[async_trait]
+impl typed_kv::Adapter for Adapter {
+ fn info(&self) -> typed_kv::Info {
+ typed_kv::Info::new(
+ Scheme::MiniMoka,
+ "mini-moka",
+ typed_kv::Capability {
+ get: true,
+ set: true,
+ delete: true,
+ ..Default::default()
+ },
+ )
+ }
+
+ async fn get(&self, path: &str) -> Result<Option<typed_kv::Value>> {
+ self.blocking_get(path)
+ }
+
+ fn blocking_get(&self, path: &str) -> Result<Option<typed_kv::Value>> {
+ match self.inner.get(&path.to_string()) {
+ None => Ok(None),
+ Some(bs) => Ok(Some(bs)),
+ }
+ }
+
+ async fn set(&self, path: &str, value: typed_kv::Value) -> Result<()> {
+ self.blocking_set(path, value)
+ }
+
+ fn blocking_set(&self, path: &str, value: typed_kv::Value) -> Result<()> {
+ self.inner.insert(path.to_string(), value);
+
+ Ok(())
+ }
+
+ async fn delete(&self, path: &str) -> Result<()> {
+ self.blocking_delete(path)
+ }
+
+ fn blocking_delete(&self, path: &str) -> Result<()> {
+ self.inner.invalidate(&path.to_string());
+
+ Ok(())
+ }
+}
diff --git a/core/src/services/mini_moka/mod.rs b/core/src/services/mini_moka/mod.rs
new file mode 100644
index 000000000..d32c03504
--- /dev/null
+++ b/core/src/services/mini_moka/mod.rs
@@ -0,0 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod backend;
+pub use backend::MiniMokaBuilder as MiniMoka;
diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs
index 9aa631bed..0ec895854 100644
--- a/core/src/services/mod.rs
+++ b/core/src/services/mod.rs
@@ -89,6 +89,11 @@ mod memory;
#[cfg(feature = "services-memory")]
pub use memory::Memory;
+#[cfg(feature = "services-mini-moka")]
+mod mini_moka;
+#[cfg(feature = "services-mini-moka")]
+pub use self::mini_moka::MiniMoka;
+
#[cfg(feature = "services-moka")]
mod moka;
#[cfg(feature = "services-moka")]
diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs
index c4079c250..57a954521 100644
--- a/core/src/types/operator/builder.rs
+++ b/core/src/types/operator/builder.rs
@@ -181,6 +181,8 @@ impl Operator {
Scheme::Memcached => Self::from_map::<services::Memcached>(map)?.finish(),
#[cfg(feature = "services-memory")]
Scheme::Memory => Self::from_map::<services::Memory>(map)?.finish(),
+ #[cfg(feature = "services-mini-moka")]
+ Scheme::MiniMoka => Self::from_map::<services::MiniMoka>(map)?.finish(),
#[cfg(feature = "services-moka")]
Scheme::Moka => Self::from_map::<services::Moka>(map)?.finish(),
#[cfg(feature = "services-obs")]
diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs
index 4ae707fd9..553e31e1c 100644
--- a/core/src/types/scheme.rs
+++ b/core/src/types/scheme.rs
@@ -60,6 +60,8 @@ pub enum Scheme {
Memcached,
/// [memory][crate::services::Memory]: In memory backend support.
Memory,
+ /// [mini-moka][crate::services::MiniMoka]: Mini Moka backend support.
+ MiniMoka,
/// [moka][crate::services::Moka]: moka backend support.
Moka,
/// [obs][crate::services::Obs]: Huawei Cloud OBS services.
@@ -142,6 +144,7 @@ impl FromStr for Scheme {
"ipmfs" => Ok(Scheme::Ipmfs),
"memcached" => Ok(Scheme::Memcached),
"memory" => Ok(Scheme::Memory),
+ "mini_moka" => Ok(Scheme::MiniMoka),
"moka" => Ok(Scheme::Moka),
"obs" => Ok(Scheme::Obs),
"redis" => Ok(Scheme::Redis),
@@ -176,6 +179,7 @@ impl From<Scheme> for &'static str {
Scheme::Ipmfs => "ipmfs",
Scheme::Memcached => "memcached",
Scheme::Memory => "memory",
+ Scheme::MiniMoka => "mini_moka",
Scheme::Moka => "moka",
Scheme::Obs => "obs",
Scheme::Onedrive => "onedrive",
diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs
index e1357de00..da3e22078 100644
--- a/core/tests/behavior/main.rs
+++ b/core/tests/behavior/main.rs
@@ -122,6 +122,8 @@ fn main() -> anyhow::Result<()> {
tests.extend(behavior_test::<services::Memcached>());
#[cfg(feature = "services-memory")]
tests.extend(behavior_test::<services::Memory>());
+ #[cfg(feature = "services-mini-moka")]
+ tests.extend(behavior_test::<services::MiniMoka>());
#[cfg(feature = "services-moka")]
tests.extend(behavior_test::<services::Moka>());
#[cfg(feature = "services-obs")]
diff --git a/website/src/components/HomepageFeatures/_feature_services.mdx b/website/src/components/HomepageFeatures/_feature_services.mdx
index bef5e97f4..4335aab94 100644
--- a/website/src/components/HomepageFeatures/_feature_services.mdx
+++ b/website/src/components/HomepageFeatures/_feature_services.mdx
@@ -59,7 +59,8 @@ Apache OpenDAL provides native support for all kinds for storage systems.
- ghac: [Github Action Cache](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows) Service
- memcached: [Memcached](https://memcached.org/) service
-- moka: [moka](https://github.com/moka-rs/moka) backend
+- mini_moka: [Mini Moka](https://github.com/moka-rs/mini-moka) backend
+- moka: [Moka](https://github.com/moka-rs/moka) backend
- vercel_artifacts: [Vercel Remote Caching](https://vercel.com/docs/concepts/monorepos/remote-caching) Service *being worked on*
</details>