You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/10/07 08:40:21 UTC
[incubator-opendal] 01/02: Use moka to replace the path cache
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch polish-gdrive
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit c891652f2e37a6b717bc11e7e5d0d9679f9cf85a
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Sat Oct 7 16:14:15 2023 +0800
Use moka to replace the path cache
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/Cargo.toml | 2 +-
core/src/services/gdrive/backend.rs | 35 +++++++++++++----------------------
core/src/services/gdrive/builder.rs | 3 ++-
core/src/services/gdrive/core.rs | 10 +++++-----
core/src/services/gdrive/mod.rs | 3 ++-
5 files changed, 23 insertions(+), 30 deletions(-)
diff --git a/core/Cargo.toml b/core/Cargo.toml
index f1c568c51..796fa91aa 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -131,7 +131,7 @@ services-gcs = [
"reqsign?/services-google",
"reqsign?/reqwest_request",
]
-services-gdrive = []
+services-gdrive = ["dep:moka"]
services-ghac = []
services-hdfs = ["dep:hdrs"]
services-http = []
diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs
index e7bc2a2bd..1cccca607 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -55,23 +55,14 @@ impl Accessor for GdriveBackend {
.set_root(&self.core.root)
.set_native_capability(Capability {
stat: true,
-
read: true,
-
list: true,
-
list_with_delimiter_slash: true,
-
write: true,
-
create_dir: true,
-
rename: true,
-
delete: true,
-
copy: true,
-
..Default::default()
});
@@ -113,9 +104,7 @@ impl Accessor for GdriveBackend {
.map_err(new_json_deserialize_error)?;
if !meta.files.is_empty() {
- let mut cache = self.core.path_cache.lock().await;
-
- cache.insert(
+ self.core.path_cache.insert(
build_abs_path(&self.core.root, path),
meta.files[0].id.clone(),
);
@@ -136,9 +125,9 @@ impl Accessor for GdriveBackend {
let meta = serde_json::from_slice::<GdriveFile>(&body)
.map_err(new_json_deserialize_error)?;
- let mut cache = self.core.path_cache.lock().await;
-
- cache.insert(build_abs_path(&self.core.root, path), meta.id.clone());
+ self.core
+ .path_cache
+ .insert(build_abs_path(&self.core.root, path), meta.id.clone());
Ok(RpCreateDir::default())
}
@@ -216,10 +205,12 @@ impl Accessor for GdriveBackend {
let meta = serde_json::from_slice::<GdriveFile>(&body)
.map_err(new_json_deserialize_error)?;
- let mut cache = self.core.path_cache.lock().await;
-
- cache.remove(&build_abs_path(&self.core.root, from));
- cache.insert(build_abs_path(&self.core.root, to), meta.id.clone());
+ self.core
+ .path_cache
+ .invalidate(&build_abs_path(&self.core.root, from));
+ self.core
+ .path_cache
+ .insert(build_abs_path(&self.core.root, to), meta.id.clone());
Ok(RpRename::default())
}
@@ -234,9 +225,9 @@ impl Accessor for GdriveBackend {
match status {
StatusCode::NO_CONTENT => {
- let mut cache = self.core.path_cache.lock().await;
-
- cache.remove(&build_abs_path(&self.core.root, path));
+ self.core
+ .path_cache
+ .invalidate(&build_abs_path(&self.core.root, path));
return Ok(RpDelete::default());
}
diff --git a/core/src/services/gdrive/builder.rs b/core/src/services/gdrive/builder.rs
index 984c27909..28966ec17 100644
--- a/core/src/services/gdrive/builder.rs
+++ b/core/src/services/gdrive/builder.rs
@@ -23,6 +23,7 @@ use std::sync::Arc;
use chrono::DateTime;
use chrono::Utc;
use log::debug;
+use moka::sync::Cache;
use tokio::sync::Mutex;
use super::backend::GdriveBackend;
@@ -195,7 +196,7 @@ impl Builder for GdriveBuilder {
root,
signer: Arc::new(Mutex::new(signer)),
client,
- path_cache: Arc::default(),
+ path_cache: Cache::builder().max_capacity(10_000).build(),
}),
})
}
diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs
index 4e7216184..785895a41 100644
--- a/core/src/services/gdrive/core.rs
+++ b/core/src/services/gdrive/core.rs
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
@@ -29,6 +28,7 @@ use http::header;
use http::Request;
use http::Response;
use http::StatusCode;
+use moka::sync::Cache;
use serde::Deserialize;
use serde_json::json;
use tokio::sync::Mutex;
@@ -55,7 +55,7 @@ pub struct GdriveCore {
///
/// - The path is rooted at the root of the Google Drive.
/// - The path is absolute path, like `foo/bar`.
- pub path_cache: Arc<Mutex<HashMap<String, String>>>,
+ pub path_cache: Cache<String, String>,
}
impl Debug for GdriveCore {
@@ -80,7 +80,7 @@ impl GdriveCore {
pub(crate) async fn get_file_id_by_path(&self, file_path: &str) -> Result<String> {
let path = build_abs_path(&self.root, file_path);
- let mut cache = self.path_cache.lock().await;
+ let cache = &self.path_cache;
if let Some(id) = cache.get(&path) {
return Ok(id.to_owned());
@@ -161,7 +161,7 @@ impl GdriveCore {
let mut file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect();
file_path_items.pop();
- let mut cache = self.path_cache.lock().await;
+ let cache = &self.path_cache;
for (i, item) in file_path_items.iter().enumerate() {
let path_part = file_path_items[0..=i].join("/");
@@ -406,7 +406,7 @@ impl GdriveCore {
.collect();
source_parent.pop();
- let cache = self.path_cache.lock().await;
+ let cache = &self.path_cache;
let file_name = build_abs_path(&self.root, target)
.split('/')
diff --git a/core/src/services/gdrive/mod.rs b/core/src/services/gdrive/mod.rs
index 2ef0caaec..ec6274b28 100644
--- a/core/src/services/gdrive/mod.rs
+++ b/core/src/services/gdrive/mod.rs
@@ -17,9 +17,10 @@
mod backend;
mod builder;
+pub use builder::GdriveBuilder as Gdrive;
+
mod core;
mod error;
-pub use builder::GdriveBuilder as Gdrive;
mod pager;
mod writer;