You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/01/16 14:24:59 UTC
(incubator-opendal) branch path-cache updated (fc096ee5e3 -> 9f783e3d5e)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a change to branch path-cache
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
from fc096ee5e3 Fix rename
new 227f1d16c9 Fix typo
new 9f783e3d5e Polish
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
core/src/raw/path_cache.rs | 51 ++++++++++++++++++++-----------------
core/src/services/gdrive/backend.rs | 2 +-
2 files changed, 29 insertions(+), 24 deletions(-)
(incubator-opendal) 01/02: Fix typo
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch path-cache
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 227f1d16c95fabfe07c027bf6e74a960871ceadc
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Jan 16 22:19:13 2024 +0800
Fix typo
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/path_cache.rs | 6 +++---
core/src/services/gdrive/backend.rs | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/core/src/raw/path_cache.rs b/core/src/raw/path_cache.rs
index 94e7220c2a..2e424ba080 100644
--- a/core/src/raw/path_cache.rs
+++ b/core/src/raw/path_cache.rs
@@ -35,15 +35,15 @@ pub trait PathQuery {
/// PathCacher is a cache for path query.
///
-/// OpenDAL is designed for path based storage systemds, such as S3, HDFS, etc. But there are many
+/// OpenDAL is designed for path based storage systems, such as S3, HDFS, etc. But there are many
/// services that are not path based, such as OneDrive, Google Drive, etc. For these services, we
/// lookup files based on id. The lookup of id is very expensive, so we cache the path to id mapping
/// in PathCacher.
///
/// # Behavior
///
-/// The `path` in cache is always an absolute. For example, if service root is `/root/`, then the
-/// path of file `a/b` in cache will be `/root/a/b`.
+/// The `path` in the cache is always an absolute one. For example, if the service root is `/root/`,
+/// then the path of file `a/b` in cache will be `/root/a/b`.
pub struct PathCacher<Q: PathQuery> {
query: Q,
cache: Cache<String, String>,
diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs
index afc30249c2..ed9ed8354e 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -170,7 +170,7 @@ impl Accessor for GdriveBackend {
let from_file_id = self.core.path_cache.get(&from).await?.ok_or(Error::new(
ErrorKind::NotFound,
- "the file to copy is not exist",
+ "the file to copy does not exist",
))?;
let to_name = get_basename(to);
(incubator-opendal) 02/02: Polish
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch path-cache
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 9f783e3d5eefa13a8d39945fec9559b9fc24b517
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Jan 16 22:24:50 2024 +0800
Polish
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/path_cache.rs | 45 +++++++++++++++++++++++++--------------------
1 file changed, 25 insertions(+), 20 deletions(-)
diff --git a/core/src/raw/path_cache.rs b/core/src/raw/path_cache.rs
index 2e424ba080..45efbf8691 100644
--- a/core/src/raw/path_cache.rs
+++ b/core/src/raw/path_cache.rs
@@ -19,6 +19,7 @@ use crate::raw::*;
use crate::*;
use async_trait::async_trait;
use moka::sync::Cache;
+use std::collections::VecDeque;
use tokio::sync::{Mutex, MutexGuard};
/// The trait required for path cacher.
@@ -29,7 +30,7 @@ pub trait PathQuery {
async fn root(&self) -> Result<String>;
/// Query the id by parent_id and name.
async fn query(&self, parent_id: &str, name: &str) -> Result<Option<String>>;
- /// Create dir by parent_id and name.
+ /// Create a dir by parent_id and name.
async fn create_dir(&self, parent_id: &str, name: &str) -> Result<String>;
}
@@ -110,20 +111,39 @@ impl<Q: PathQuery> PathCacher<Q> {
return Ok(Some(id));
}
- let mut paths = vec![];
+ let mut paths = VecDeque::new();
let mut current_path = path;
while current_path != "/" && !current_path.is_empty() {
- paths.push(current_path.to_string());
+ paths.push_front(current_path.to_string());
current_path = get_parent(current_path);
if let Some(id) = self.cache.get(current_path) {
- return self.query_down(&id, &paths).await;
+ return self.query_down(&id, paths).await;
}
}
let root_id = self.query.root().await?;
self.cache.insert("/".to_string(), root_id.clone());
- self.query_down(&root_id, &paths).await
+ self.query_down(&root_id, paths).await
+ }
+
+ /// `start_id` is the `file_id` to the start dir to query down.
+ /// `paths` is in the order like `["/a/", "/a/b/", "/a/b/c/"]`.
+ ///
+ /// We should fetch the next `file_id` by sending `query`.
+ async fn query_down(&self, start_id: &str, paths: VecDeque<String>) -> Result<Option<String>> {
+ let mut current_id = start_id.to_string();
+ for path in paths.into_iter() {
+ let name = get_basename(&path);
+ current_id = match self.query.query(¤t_id, name).await? {
+ Some(id) => {
+ self.cache.insert(path, id.clone());
+ id
+ }
+ None => return Ok(None),
+ };
+ }
+ Ok(Some(current_id))
}
/// Ensure input dir exists.
@@ -167,21 +187,6 @@ impl<Q: PathQuery> PathCacher<Q> {
Ok(parent_id)
}
-
- async fn query_down(&self, start_id: &str, paths: &[String]) -> Result<Option<String>> {
- let mut current_id = start_id.to_string();
- for path in paths.iter().rev() {
- let name = get_basename(path);
- current_id = match self.query.query(¤t_id, name).await? {
- Some(id) => {
- self.cache.insert(path.clone(), id.clone());
- id
- }
- None => return Ok(None),
- };
- }
- Ok(Some(current_id))
- }
}
#[cfg(test)]