You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/01/12 08:47:46 UTC
(incubator-opendal) 02/02: feat: Implement path cache and refactor gdrive
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch path-cache
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 446a1aec37b38a79b35e412251271cbb9267a813
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Jan 12 16:47:25 2024 +0800
feat: Implement path cache and refactor gdrive
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/Cargo.toml | 2 +-
core/src/raw/path_cache.rs | 2 +-
core/src/services/gdrive/backend.rs | 53 +++----
core/src/services/gdrive/builder.rs | 33 ++--
core/src/services/gdrive/core.rs | 309 ++++++++++++++----------------------
core/src/services/gdrive/lister.rs | 2 +-
6 files changed, 158 insertions(+), 243 deletions(-)
diff --git a/core/Cargo.toml b/core/Cargo.toml
index b6090b2fe7..87c3b32d2f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -146,7 +146,7 @@ services-gcs = [
"reqsign?/services-google",
"reqsign?/reqwest_request",
]
-services-gdrive = []
+services-gdrive = ["internal-path-cache"]
services-ghac = []
services-gridfs = ["dep:mongodb"]
services-hdfs = ["dep:hdrs"]
diff --git a/core/src/raw/path_cache.rs b/core/src/raw/path_cache.rs
index 6e7c1dd4ce..ed99b9f5a3 100644
--- a/core/src/raw/path_cache.rs
+++ b/core/src/raw/path_cache.rs
@@ -74,7 +74,7 @@ impl<Q: PathQuery> PathCacher<Q> {
let mut paths = vec![];
let mut current_path = path;
- while current_path != "/" {
+ while current_path != "/" || !current_path.is_empty() {
paths.push(current_path.to_string());
current_path = get_parent(current_path);
if let Some(id) = self.cache.get(current_path) {
diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs
index 50a2531a29..9fe79da3bf 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -77,22 +77,7 @@ impl Accessor for GdriveBackend {
}
async fn create_dir(&self, path: &str, _args: OpCreateDir) -> Result<RpCreateDir> {
- let parent = self.core.ensure_parent_path(path).await?;
-
- // Make sure `/` has been trimmed.
- let path = get_basename(path).trim_end_matches('/');
-
- // As Google Drive allows files have the same name, we need to check if the folder exists.
- let folder_id = self.core.gdrive_search_folder(&parent, path).await?;
-
- let id = if let Some(id) = folder_id {
- id
- } else {
- self.core.gdrive_create_folder(&parent, path).await?
- };
-
- let mut cache = self.core.path_cache.lock().await;
- cache.insert(build_abs_path(&self.core.root, path), id);
+ let _ = self.core.ensure_parent_path(path).await?;
Ok(RpCreateDir::default())
}
@@ -167,7 +152,7 @@ impl Accessor for GdriveBackend {
match status {
StatusCode::NO_CONTENT | StatusCode::NOT_FOUND => {
- let mut cache = self.core.path_cache.lock().await;
+ let cache = &self.core.path_cache;
cache.remove(&build_abs_path(&self.core.root, path));
@@ -193,7 +178,8 @@ impl Accessor for GdriveBackend {
async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
let from_file_id = self
.core
- .get_file_id_by_path(from)
+ .path_cache
+ .get(from)
.await?
.ok_or(Error::new(ErrorKind::NotFound, "invalid 'from' path"))?;
@@ -208,23 +194,24 @@ impl Accessor for GdriveBackend {
let to_parent = to_path_items.join("/") + "/";
- let to_parent_id =
- if let Some(id) = self.core.get_file_id_by_path(to_parent.as_str()).await? {
- id
- } else {
- self.create_dir(&to_parent, OpCreateDir::new()).await?;
- self.core
- .get_file_id_by_path(to_parent.as_str())
- .await?
- .ok_or_else(|| {
- Error::new(ErrorKind::Unexpected, "create to's parent folder failed")
- })?
- };
+ let to_parent_id = if let Some(id) = self.core.path_cache.get(to_parent.as_str()).await? {
+ id
+ } else {
+ self.create_dir(&to_parent, OpCreateDir::new()).await?;
+ self.core
+ .path_cache
+ .get(to_parent.as_str())
+ .await?
+ .ok_or_else(|| {
+ Error::new(ErrorKind::Unexpected, "create to's parent folder failed")
+ })?
+ };
// copy will overwrite `to`, delete it if exist
if self
.core
- .get_file_id_by_path(to)
+ .path_cache
+ .get(to)
.await
.is_ok_and(|id| id.is_some())
{
@@ -266,10 +253,10 @@ impl Accessor for GdriveBackend {
let meta = serde_json::from_slice::<GdriveFile>(&body)
.map_err(new_json_deserialize_error)?;
- let mut cache = self.core.path_cache.lock().await;
+ let cache = &self.core.path_cache;
cache.remove(&build_abs_path(&self.core.root, from));
- cache.insert(build_abs_path(&self.core.root, to), meta.id.clone());
+ cache.insert(&build_abs_path(&self.core.root, to), &meta.id);
Ok(RpRename::default())
}
diff --git a/core/src/services/gdrive/builder.rs b/core/src/services/gdrive/builder.rs
index 984c279090..e3d070eb37 100644
--- a/core/src/services/gdrive/builder.rs
+++ b/core/src/services/gdrive/builder.rs
@@ -26,10 +26,10 @@ use log::debug;
use tokio::sync::Mutex;
use super::backend::GdriveBackend;
-use crate::raw::normalize_root;
use crate::raw::HttpClient;
-use crate::services::gdrive::core::GdriveCore;
+use crate::raw::{normalize_root, PathCacher};
use crate::services::gdrive::core::GdriveSigner;
+use crate::services::gdrive::core::{GdriveCore, GdrivePathQuery};
use crate::Scheme;
use crate::*;
@@ -144,13 +144,13 @@ impl Builder for GdriveBuilder {
})?
};
- let signer = match (self.access_token.take(), self.refresh_token.take()) {
- (Some(access_token), None) => GdriveSigner {
- access_token,
+ let mut signer = GdriveSigner::new(client.clone());
+ match (self.access_token.take(), self.refresh_token.take()) {
+ (Some(access_token), None) => {
+ signer.access_token = access_token;
// We will never expire user specified access token.
- expires_in: DateTime::<Utc>::MAX_UTC,
- ..Default::default()
- },
+ signer.expires_in = DateTime::<Utc>::MAX_UTC;
+ }
(None, Some(refresh_token)) => {
let client_id = self.client_id.take().ok_or_else(|| {
Error::new(
@@ -167,12 +167,10 @@ impl Builder for GdriveBuilder {
.with_context("service", Scheme::Gdrive)
})?;
- GdriveSigner {
- refresh_token,
- client_id,
- client_secret,
- ..Default::default()
- }
+ signer.refresh_token = refresh_token;
+ signer.client = client.clone();
+ signer.client_id = client_id;
+ signer.client_secret = client_secret;
}
(Some(_), Some(_)) => {
return Err(Error::new(
@@ -190,12 +188,13 @@ impl Builder for GdriveBuilder {
}
};
+ let signer = Arc::new(Mutex::new(signer));
Ok(GdriveBackend {
core: Arc::new(GdriveCore {
root,
- signer: Arc::new(Mutex::new(signer)),
- client,
- path_cache: Arc::default(),
+ signer: signer.clone(),
+ client: client.clone(),
+ path_cache: PathCacher::new(GdrivePathQuery::new(client, signer)),
}),
})
}
diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs
index f43e80acd3..bdbdb11394 100644
--- a/core/src/services/gdrive/core.rs
+++ b/core/src/services/gdrive/core.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use std::collections::HashMap;
+use async_trait::async_trait;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
@@ -46,15 +46,7 @@ pub struct GdriveCore {
pub signer: Arc<Mutex<GdriveSigner>>,
/// Cache the mapping from path to file id
- ///
- /// Google Drive uses file id to identify a file.
- /// As the path is immutable, we can cache the mapping from path to file id.
- ///
- /// # Notes
- ///
- /// - The path is rooted at the root of the Google Drive.
- /// - The path is absolute path, like `foo/bar`.
- pub path_cache: Arc<Mutex<HashMap<String, String>>>,
+ pub path_cache: PathCacher<GdrivePathQuery>,
}
impl Debug for GdriveCore {
@@ -66,54 +58,6 @@ impl Debug for GdriveCore {
}
impl GdriveCore {
- /// Get the file id by path.
- /// Including file and folder.
- ///
- /// The path is rooted at the root of the Google Drive.
- ///
- /// # Notes
- ///
- /// - A path is a sequence of file names separated by slashes.
- /// - A file only knows its parent id, but not its name.
- /// - To find the file id of a file, we need to traverse the path from the root to the file.
- pub(crate) async fn get_file_id_by_path(&self, file_path: &str) -> Result<Option<String>> {
- let path = build_abs_path(&self.root, file_path);
-
- let mut cache = self.path_cache.lock().await;
-
- if let Some(id) = cache.get(&path) {
- return Ok(Some(id.to_owned()));
- }
-
- let mut parent_id = "root".to_owned();
- let file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect();
-
- for (i, item) in file_path_items.iter().enumerate() {
- let path_part = file_path_items[0..=i].join("/");
- if let Some(id) = cache.get(&path_part) {
- parent_id = id.to_owned();
- continue;
- }
-
- let id = if i != file_path_items.len() - 1 || path.ends_with('/') {
- self.gdrive_search_folder(&parent_id, item).await?
- } else {
- self.gdrive_search_file(&parent_id, item)
- .await?
- .map(|v| v.id)
- };
-
- if let Some(id) = id {
- parent_id = id;
- cache.insert(path_part, parent_id.clone());
- } else {
- return Ok(None);
- };
- }
-
- Ok(Some(parent_id))
- }
-
/// Ensure the parent path exists.
/// If the parent path does not exist, create it.
///
@@ -122,116 +66,30 @@ impl GdriveCore {
/// - The path is rooted at the root of the Google Drive.
/// - Will create the parent path recursively.
pub(crate) async fn ensure_parent_path(&self, path: &str) -> Result<String> {
- let path = build_abs_path(&self.root, path);
-
- let mut parent: String = "root".to_owned();
- let mut file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect();
- file_path_items.pop();
-
- let mut cache = self.path_cache.lock().await;
-
- for (i, item) in file_path_items.iter().enumerate() {
- let path_part = file_path_items[0..=i].join("/");
- if let Some(id) = cache.get(&path_part) {
- parent = id.to_owned();
- continue;
+ let mut parents = vec![];
+ let mut current_path = path;
+ let mut parent_id = String::new();
+
+ while current_path != "/" || !current_path.is_empty() {
+ let id = self.path_cache.get(current_path).await?;
+ match id {
+ Some(id) => {
+ parent_id = id;
+ break;
+ }
+ None => parents.push(current_path),
}
-
- let folder_id = self.gdrive_search_folder(&parent, item).await?;
- let folder_id = if let Some(id) = folder_id {
- id
- } else {
- self.gdrive_create_folder(&parent, item).await?
- };
-
- parent = folder_id;
- cache.insert(path_part, parent.clone());
+ current_path = get_parent(current_path);
}
- Ok(parent.to_owned())
- }
-
- /// Search a folder by name
- ///
- /// returns it's file id if exists, otherwise returns `None`.
- pub async fn gdrive_search_file(
- &self,
- parent: &str,
- basename: &str,
- ) -> Result<Option<GdriveFile>> {
- let query =
- format!("name = \"{basename}\" and \"{parent}\" in parents and trashed = false");
- let url = format!(
- "https://www.googleapis.com/drive/v3/files?q={}",
- percent_encode_path(&query)
- );
-
- let mut req = Request::get(&url)
- .body(AsyncBody::Empty)
- .map_err(new_request_build_error)?;
-
- self.sign(&mut req).await?;
-
- let resp = self.client.send(req).await?;
- let status = resp.status();
- if !status.is_success() {
- return Err(parse_error(resp).await?);
- }
-
- let body = resp.into_body().bytes().await?;
- let mut file_list: GdriveFileList =
- serde_json::from_slice(&body).map_err(new_json_deserialize_error)?;
-
- if file_list.files.len() > 1 {
- return Err(Error::new(
- ErrorKind::Unexpected,
- "please ensure that the file corresponding to the path is unique.",
- ));
+ for parent in parents.iter().rev() {
+ parent_id = self
+ .gdrive_create_folder(&parent_id, get_basename(parent))
+ .await?;
+ self.path_cache.insert(parent, &parent_id);
}
- Ok(file_list.files.pop())
- }
-
- /// Search a folder by name
- ///
- /// returns it's file id if exists, otherwise returns `None`.
- pub async fn gdrive_search_folder(
- &self,
- parent: &str,
- basename: &str,
- ) -> Result<Option<String>> {
- let query = format!(
- "name = \"{}\" and \"{}\" in parents and trashed = false and mimeType = 'application/vnd.google-apps.folder'",
- basename, parent
- );
- let url = format!(
- "https://www.googleapis.com/drive/v3/files?q={}",
- percent_encode_path(query.as_str())
- );
-
- let mut req = Request::get(&url)
- .body(AsyncBody::Empty)
- .map_err(new_request_build_error)?;
-
- self.sign(&mut req).await?;
-
- let resp = self.client.send(req).await?;
- let status = resp.status();
-
- match status {
- StatusCode::OK => {
- let body = resp.into_body().bytes().await?;
- let meta: GdriveFileList =
- serde_json::from_slice(&body).map_err(new_json_deserialize_error)?;
-
- if let Some(f) = meta.files.first() {
- Ok(Some(f.id.clone()))
- } else {
- Ok(None)
- }
- }
- _ => Err(parse_error(resp).await?),
- }
+ Ok(parent_id)
}
/// Create a folder.
@@ -272,7 +130,7 @@ impl GdriveCore {
}
pub async fn gdrive_stat(&self, path: &str) -> Result<Response<IncomingAsyncBody>> {
- let path_id = self.get_file_id_by_path(path).await?.ok_or(Error::new(
+ let path_id = self.path_cache.get(path).await?.ok_or(Error::new(
ErrorKind::NotFound,
&format!("path not found: {}", path),
))?;
@@ -291,7 +149,7 @@ impl GdriveCore {
}
pub async fn gdrive_get(&self, path: &str) -> Result<Response<IncomingAsyncBody>> {
- let path_id = self.get_file_id_by_path(path).await?.ok_or(Error::new(
+ let path_id = self.path_cache.get(path).await?.ok_or(Error::new(
ErrorKind::NotFound,
&format!("path not found: {}", path),
))?;
@@ -339,7 +197,7 @@ impl GdriveCore {
source: &str,
target: &str,
) -> Result<Response<IncomingAsyncBody>> {
- let file_id = self.get_file_id_by_path(source).await?.ok_or(Error::new(
+ let file_id = self.path_cache.get(source).await?.ok_or(Error::new(
ErrorKind::NotFound,
&format!("source path not found: {}", source),
))?;
@@ -355,7 +213,7 @@ impl GdriveCore {
.collect();
source_parent.pop();
- let cache = self.path_cache.lock().await;
+ let cache = &self.path_cache;
let file_name = build_abs_path(&self.root, target)
.split('/')
@@ -364,9 +222,13 @@ impl GdriveCore {
.unwrap()
.to_string();
+ let old_parent = cache
+ .get(&source_parent.join("/"))
+ .await?
+ .expect("old parent must exist");
let metadata = &json!({
"name": file_name,
- "removeParents": [cache.get(&source_parent.join("/")).unwrap().to_string()],
+ "removeParents": [old_parent],
"addParents": [parent],
});
@@ -380,7 +242,7 @@ impl GdriveCore {
}
pub async fn gdrive_delete(&self, path: &str) -> Result<Response<IncomingAsyncBody>> {
- let file_id = self.get_file_id_by_path(path).await?.ok_or(Error::new(
+ let file_id = self.path_cache.get(path).await?.ok_or(Error::new(
ErrorKind::NotFound,
&format!("path not found: {}", path),
))?;
@@ -470,9 +332,40 @@ impl GdriveCore {
pub async fn sign<T>(&self, req: &mut Request<T>) -> Result<()> {
let mut signer = self.signer.lock().await;
+ signer.sign(req).await
+ }
+}
+
+#[derive(Clone)]
+pub struct GdriveSigner {
+ pub client: HttpClient,
+
+ pub client_id: String,
+ pub client_secret: String,
+ pub refresh_token: String,
- if !signer.access_token.is_empty() && signer.expires_in > Utc::now() {
- let value = format!("Bearer {}", signer.access_token)
+ pub access_token: String,
+ pub expires_in: DateTime<Utc>,
+}
+
+impl GdriveSigner {
+ /// Create a new signer.
+ pub fn new(client: HttpClient) -> Self {
+ GdriveSigner {
+ client,
+
+ client_id: "".to_string(),
+ client_secret: "".to_string(),
+ refresh_token: "".to_string(),
+ access_token: "".to_string(),
+ expires_in: DateTime::<Utc>::MIN_UTC,
+ }
+ }
+
+ /// Sign a request.
+ pub async fn sign<T>(&mut self, req: &mut Request<T>) -> Result<()> {
+ if !self.access_token.is_empty() && self.expires_in > Utc::now() {
+ let value = format!("Bearer {}", self.access_token)
.parse()
.expect("access token must be valid header value");
@@ -482,7 +375,7 @@ impl GdriveCore {
let url = format!(
"https://oauth2.googleapis.com/token?refresh_token={}&client_id={}&client_secret={}&grant_type=refresh_token",
- signer.refresh_token, signer.client_id, signer.client_secret
+ self.refresh_token, self.client_id, self.client_secret
);
{
@@ -498,8 +391,8 @@ impl GdriveCore {
let resp_body = &resp.into_body().bytes().await?;
let token = serde_json::from_slice::<GdriveTokenResponse>(resp_body)
.map_err(new_json_deserialize_error)?;
- signer.access_token = token.access_token.clone();
- signer.expires_in = Utc::now() + chrono::Duration::seconds(token.expires_in)
+ self.access_token = token.access_token.clone();
+ self.expires_in = Utc::now() + chrono::Duration::seconds(token.expires_in)
- chrono::Duration::seconds(120);
}
_ => {
@@ -508,7 +401,7 @@ impl GdriveCore {
}
}
- let auth_header_content = format!("Bearer {}", signer.access_token);
+ let auth_header_content = format!("Bearer {}", self.access_token);
req.headers_mut()
.insert(header::AUTHORIZATION, auth_header_content.parse().unwrap());
@@ -516,25 +409,61 @@ impl GdriveCore {
}
}
-#[derive(Clone)]
-pub struct GdriveSigner {
- pub client_id: String,
- pub client_secret: String,
- pub refresh_token: String,
+pub struct GdrivePathQuery {
+ pub client: HttpClient,
+ pub signer: Arc<Mutex<GdriveSigner>>,
+}
- pub access_token: String,
- pub expires_in: DateTime<Utc>,
+impl GdrivePathQuery {
+ pub fn new(client: HttpClient, signer: Arc<Mutex<GdriveSigner>>) -> Self {
+ GdrivePathQuery { client, signer }
+ }
}
-impl Default for GdriveSigner {
- fn default() -> Self {
- GdriveSigner {
- access_token: String::new(),
- expires_in: DateTime::<Utc>::MIN_UTC,
+#[async_trait]
+impl PathQuery for GdrivePathQuery {
+ async fn root(&self) -> Result<String> {
+ Ok("root".to_string())
+ }
+
+ async fn query(&self, parent_id: &str, name: &str) -> Result<Option<String>> {
+ let mut queries = vec![
+ format!("name = \"{}\"", name),
+ format!("\"{}\" in parents", parent_id),
+ "trashed = false".to_string(),
+ ];
+ if name.ends_with('/') {
+ queries.push("mimeType = 'application/vnd.google-apps.folder'".to_string());
+ }
+ let query = queries.join(" and ");
+
+ let url = format!(
+ "https://www.googleapis.com/drive/v3/files?q={}",
+ percent_encode_path(query.as_str())
+ );
+
+ let mut req = Request::get(&url)
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
- refresh_token: String::new(),
- client_id: String::new(),
- client_secret: String::new(),
+ self.signer.lock().await.sign(&mut req).await?;
+
+ let resp = self.client.send(req).await?;
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let body = resp.into_body().bytes().await?;
+ let meta: GdriveFileList =
+ serde_json::from_slice(&body).map_err(new_json_deserialize_error)?;
+
+ if let Some(f) = meta.files.first() {
+ Ok(Some(f.id.clone()))
+ } else {
+ Ok(None)
+ }
+ }
+ _ => Err(parse_error(resp).await?),
}
}
}
diff --git a/core/src/services/gdrive/lister.rs b/core/src/services/gdrive/lister.rs
index 9da239035d..7c665376af 100644
--- a/core/src/services/gdrive/lister.rs
+++ b/core/src/services/gdrive/lister.rs
@@ -41,7 +41,7 @@ impl GdriveLister {
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl oio::PageList for GdriveLister {
async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
- let file_id = self.core.get_file_id_by_path(&self.path).await?;
+ let file_id = self.core.path_cache.get(&self.path).await?;
let file_id = match file_id {
Some(file_id) => file_id,