You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/01/12 08:47:46 UTC

(incubator-opendal) 02/02: feat: Implement path cache and refactor gdrive

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch path-cache
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git

commit 446a1aec37b38a79b35e412251271cbb9267a813
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Jan 12 16:47:25 2024 +0800

    feat: Implement path cache and refactor gdrive
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 core/Cargo.toml                     |   2 +-
 core/src/raw/path_cache.rs          |   2 +-
 core/src/services/gdrive/backend.rs |  53 +++----
 core/src/services/gdrive/builder.rs |  33 ++--
 core/src/services/gdrive/core.rs    | 309 ++++++++++++++----------------------
 core/src/services/gdrive/lister.rs  |   2 +-
 6 files changed, 158 insertions(+), 243 deletions(-)

diff --git a/core/Cargo.toml b/core/Cargo.toml
index b6090b2fe7..87c3b32d2f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -146,7 +146,7 @@ services-gcs = [
   "reqsign?/services-google",
   "reqsign?/reqwest_request",
 ]
-services-gdrive = []
+services-gdrive = ["internal-path-cache"]
 services-ghac = []
 services-gridfs = ["dep:mongodb"]
 services-hdfs = ["dep:hdrs"]
diff --git a/core/src/raw/path_cache.rs b/core/src/raw/path_cache.rs
index 6e7c1dd4ce..ed99b9f5a3 100644
--- a/core/src/raw/path_cache.rs
+++ b/core/src/raw/path_cache.rs
@@ -74,7 +74,7 @@ impl<Q: PathQuery> PathCacher<Q> {
         let mut paths = vec![];
         let mut current_path = path;
 
-        while current_path != "/" {
+        while current_path != "/" || !current_path.is_empty() {
             paths.push(current_path.to_string());
             current_path = get_parent(current_path);
             if let Some(id) = self.cache.get(current_path) {
diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs
index 50a2531a29..9fe79da3bf 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -77,22 +77,7 @@ impl Accessor for GdriveBackend {
     }
 
     async fn create_dir(&self, path: &str, _args: OpCreateDir) -> Result<RpCreateDir> {
-        let parent = self.core.ensure_parent_path(path).await?;
-
-        // Make sure `/` has been trimmed.
-        let path = get_basename(path).trim_end_matches('/');
-
-        // As Google Drive allows files have the same name, we need to check if the folder exists.
-        let folder_id = self.core.gdrive_search_folder(&parent, path).await?;
-
-        let id = if let Some(id) = folder_id {
-            id
-        } else {
-            self.core.gdrive_create_folder(&parent, path).await?
-        };
-
-        let mut cache = self.core.path_cache.lock().await;
-        cache.insert(build_abs_path(&self.core.root, path), id);
+        let _ = self.core.ensure_parent_path(path).await?;
 
         Ok(RpCreateDir::default())
     }
@@ -167,7 +152,7 @@ impl Accessor for GdriveBackend {
 
             match status {
                 StatusCode::NO_CONTENT | StatusCode::NOT_FOUND => {
-                    let mut cache = self.core.path_cache.lock().await;
+                    let cache = &self.core.path_cache;
 
                     cache.remove(&build_abs_path(&self.core.root, path));
 
@@ -193,7 +178,8 @@ impl Accessor for GdriveBackend {
     async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
         let from_file_id = self
             .core
-            .get_file_id_by_path(from)
+            .path_cache
+            .get(from)
             .await?
             .ok_or(Error::new(ErrorKind::NotFound, "invalid 'from' path"))?;
 
@@ -208,23 +194,24 @@ impl Accessor for GdriveBackend {
 
         let to_parent = to_path_items.join("/") + "/";
 
-        let to_parent_id =
-            if let Some(id) = self.core.get_file_id_by_path(to_parent.as_str()).await? {
-                id
-            } else {
-                self.create_dir(&to_parent, OpCreateDir::new()).await?;
-                self.core
-                    .get_file_id_by_path(to_parent.as_str())
-                    .await?
-                    .ok_or_else(|| {
-                        Error::new(ErrorKind::Unexpected, "create to's parent folder failed")
-                    })?
-            };
+        let to_parent_id = if let Some(id) = self.core.path_cache.get(to_parent.as_str()).await? {
+            id
+        } else {
+            self.create_dir(&to_parent, OpCreateDir::new()).await?;
+            self.core
+                .path_cache
+                .get(to_parent.as_str())
+                .await?
+                .ok_or_else(|| {
+                    Error::new(ErrorKind::Unexpected, "create to's parent folder failed")
+                })?
+        };
 
         // copy will overwrite `to`, delete it if exist
         if self
             .core
-            .get_file_id_by_path(to)
+            .path_cache
+            .get(to)
             .await
             .is_ok_and(|id| id.is_some())
         {
@@ -266,10 +253,10 @@ impl Accessor for GdriveBackend {
                 let meta = serde_json::from_slice::<GdriveFile>(&body)
                     .map_err(new_json_deserialize_error)?;
 
-                let mut cache = self.core.path_cache.lock().await;
+                let cache = &self.core.path_cache;
 
                 cache.remove(&build_abs_path(&self.core.root, from));
-                cache.insert(build_abs_path(&self.core.root, to), meta.id.clone());
+                cache.insert(&build_abs_path(&self.core.root, to), &meta.id);
 
                 Ok(RpRename::default())
             }
diff --git a/core/src/services/gdrive/builder.rs b/core/src/services/gdrive/builder.rs
index 984c279090..e3d070eb37 100644
--- a/core/src/services/gdrive/builder.rs
+++ b/core/src/services/gdrive/builder.rs
@@ -26,10 +26,10 @@ use log::debug;
 use tokio::sync::Mutex;
 
 use super::backend::GdriveBackend;
-use crate::raw::normalize_root;
 use crate::raw::HttpClient;
-use crate::services::gdrive::core::GdriveCore;
+use crate::raw::{normalize_root, PathCacher};
 use crate::services::gdrive::core::GdriveSigner;
+use crate::services::gdrive::core::{GdriveCore, GdrivePathQuery};
 use crate::Scheme;
 use crate::*;
 
@@ -144,13 +144,13 @@ impl Builder for GdriveBuilder {
             })?
         };
 
-        let signer = match (self.access_token.take(), self.refresh_token.take()) {
-            (Some(access_token), None) => GdriveSigner {
-                access_token,
+        let mut signer = GdriveSigner::new(client.clone());
+        match (self.access_token.take(), self.refresh_token.take()) {
+            (Some(access_token), None) => {
+                signer.access_token = access_token;
                 // We will never expire user specified access token.
-                expires_in: DateTime::<Utc>::MAX_UTC,
-                ..Default::default()
-            },
+                signer.expires_in = DateTime::<Utc>::MAX_UTC;
+            }
             (None, Some(refresh_token)) => {
                 let client_id = self.client_id.take().ok_or_else(|| {
                     Error::new(
@@ -167,12 +167,10 @@ impl Builder for GdriveBuilder {
                     .with_context("service", Scheme::Gdrive)
                 })?;
 
-                GdriveSigner {
-                    refresh_token,
-                    client_id,
-                    client_secret,
-                    ..Default::default()
-                }
+                signer.refresh_token = refresh_token;
+                signer.client = client.clone();
+                signer.client_id = client_id;
+                signer.client_secret = client_secret;
             }
             (Some(_), Some(_)) => {
                 return Err(Error::new(
@@ -190,12 +188,13 @@ impl Builder for GdriveBuilder {
             }
         };
 
+        let signer = Arc::new(Mutex::new(signer));
         Ok(GdriveBackend {
             core: Arc::new(GdriveCore {
                 root,
-                signer: Arc::new(Mutex::new(signer)),
-                client,
-                path_cache: Arc::default(),
+                signer: signer.clone(),
+                client: client.clone(),
+                path_cache: PathCacher::new(GdrivePathQuery::new(client, signer)),
             }),
         })
     }
diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs
index f43e80acd3..bdbdb11394 100644
--- a/core/src/services/gdrive/core.rs
+++ b/core/src/services/gdrive/core.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::HashMap;
+use async_trait::async_trait;
 use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::sync::Arc;
@@ -46,15 +46,7 @@ pub struct GdriveCore {
     pub signer: Arc<Mutex<GdriveSigner>>,
 
     /// Cache the mapping from path to file id
-    ///
-    /// Google Drive uses file id to identify a file.
-    /// As the path is immutable, we can cache the mapping from path to file id.
-    ///
-    /// # Notes
-    ///
-    /// - The path is rooted at the root of the Google Drive.
-    /// - The path is absolute path, like `foo/bar`.
-    pub path_cache: Arc<Mutex<HashMap<String, String>>>,
+    pub path_cache: PathCacher<GdrivePathQuery>,
 }
 
 impl Debug for GdriveCore {
@@ -66,54 +58,6 @@ impl Debug for GdriveCore {
 }
 
 impl GdriveCore {
-    /// Get the file id by path.
-    /// Including file and folder.
-    ///
-    /// The path is rooted at the root of the Google Drive.
-    ///
-    /// # Notes
-    ///
-    /// - A path is a sequence of file names separated by slashes.
-    /// - A file only knows its parent id, but not its name.
-    /// - To find the file id of a file, we need to traverse the path from the root to the file.
-    pub(crate) async fn get_file_id_by_path(&self, file_path: &str) -> Result<Option<String>> {
-        let path = build_abs_path(&self.root, file_path);
-
-        let mut cache = self.path_cache.lock().await;
-
-        if let Some(id) = cache.get(&path) {
-            return Ok(Some(id.to_owned()));
-        }
-
-        let mut parent_id = "root".to_owned();
-        let file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect();
-
-        for (i, item) in file_path_items.iter().enumerate() {
-            let path_part = file_path_items[0..=i].join("/");
-            if let Some(id) = cache.get(&path_part) {
-                parent_id = id.to_owned();
-                continue;
-            }
-
-            let id = if i != file_path_items.len() - 1 || path.ends_with('/') {
-                self.gdrive_search_folder(&parent_id, item).await?
-            } else {
-                self.gdrive_search_file(&parent_id, item)
-                    .await?
-                    .map(|v| v.id)
-            };
-
-            if let Some(id) = id {
-                parent_id = id;
-                cache.insert(path_part, parent_id.clone());
-            } else {
-                return Ok(None);
-            };
-        }
-
-        Ok(Some(parent_id))
-    }
-
     /// Ensure the parent path exists.
     /// If the parent path does not exist, create it.
     ///
@@ -122,116 +66,30 @@ impl GdriveCore {
     /// - The path is rooted at the root of the Google Drive.
     /// - Will create the parent path recursively.
     pub(crate) async fn ensure_parent_path(&self, path: &str) -> Result<String> {
-        let path = build_abs_path(&self.root, path);
-
-        let mut parent: String = "root".to_owned();
-        let mut file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect();
-        file_path_items.pop();
-
-        let mut cache = self.path_cache.lock().await;
-
-        for (i, item) in file_path_items.iter().enumerate() {
-            let path_part = file_path_items[0..=i].join("/");
-            if let Some(id) = cache.get(&path_part) {
-                parent = id.to_owned();
-                continue;
+        let mut parents = vec![];
+        let mut current_path = path;
+        let mut parent_id = String::new();
+
+        while current_path != "/" || !current_path.is_empty() {
+            let id = self.path_cache.get(current_path).await?;
+            match id {
+                Some(id) => {
+                    parent_id = id;
+                    break;
+                }
+                None => parents.push(current_path),
             }
-
-            let folder_id = self.gdrive_search_folder(&parent, item).await?;
-            let folder_id = if let Some(id) = folder_id {
-                id
-            } else {
-                self.gdrive_create_folder(&parent, item).await?
-            };
-
-            parent = folder_id;
-            cache.insert(path_part, parent.clone());
+            current_path = get_parent(current_path);
         }
 
-        Ok(parent.to_owned())
-    }
-
-    /// Search a folder by name
-    ///
-    /// returns it's file id if exists, otherwise returns `None`.
-    pub async fn gdrive_search_file(
-        &self,
-        parent: &str,
-        basename: &str,
-    ) -> Result<Option<GdriveFile>> {
-        let query =
-            format!("name = \"{basename}\" and \"{parent}\" in parents and trashed = false");
-        let url = format!(
-            "https://www.googleapis.com/drive/v3/files?q={}",
-            percent_encode_path(&query)
-        );
-
-        let mut req = Request::get(&url)
-            .body(AsyncBody::Empty)
-            .map_err(new_request_build_error)?;
-
-        self.sign(&mut req).await?;
-
-        let resp = self.client.send(req).await?;
-        let status = resp.status();
-        if !status.is_success() {
-            return Err(parse_error(resp).await?);
-        }
-
-        let body = resp.into_body().bytes().await?;
-        let mut file_list: GdriveFileList =
-            serde_json::from_slice(&body).map_err(new_json_deserialize_error)?;
-
-        if file_list.files.len() > 1 {
-            return Err(Error::new(
-                ErrorKind::Unexpected,
-                "please ensure that the file corresponding to the path is unique.",
-            ));
+        for parent in parents.iter().rev() {
+            parent_id = self
+                .gdrive_create_folder(&parent_id, get_basename(parent))
+                .await?;
+            self.path_cache.insert(parent, &parent_id);
         }
 
-        Ok(file_list.files.pop())
-    }
-
-    /// Search a folder by name
-    ///
-    /// returns it's file id if exists, otherwise returns `None`.
-    pub async fn gdrive_search_folder(
-        &self,
-        parent: &str,
-        basename: &str,
-    ) -> Result<Option<String>> {
-        let query = format!(
-            "name = \"{}\" and \"{}\" in parents and trashed = false and mimeType = 'application/vnd.google-apps.folder'",
-            basename, parent
-        );
-        let url = format!(
-            "https://www.googleapis.com/drive/v3/files?q={}",
-            percent_encode_path(query.as_str())
-        );
-
-        let mut req = Request::get(&url)
-            .body(AsyncBody::Empty)
-            .map_err(new_request_build_error)?;
-
-        self.sign(&mut req).await?;
-
-        let resp = self.client.send(req).await?;
-        let status = resp.status();
-
-        match status {
-            StatusCode::OK => {
-                let body = resp.into_body().bytes().await?;
-                let meta: GdriveFileList =
-                    serde_json::from_slice(&body).map_err(new_json_deserialize_error)?;
-
-                if let Some(f) = meta.files.first() {
-                    Ok(Some(f.id.clone()))
-                } else {
-                    Ok(None)
-                }
-            }
-            _ => Err(parse_error(resp).await?),
-        }
+        Ok(parent_id)
     }
 
     /// Create a folder.
@@ -272,7 +130,7 @@ impl GdriveCore {
     }
 
     pub async fn gdrive_stat(&self, path: &str) -> Result<Response<IncomingAsyncBody>> {
-        let path_id = self.get_file_id_by_path(path).await?.ok_or(Error::new(
+        let path_id = self.path_cache.get(path).await?.ok_or(Error::new(
             ErrorKind::NotFound,
             &format!("path not found: {}", path),
         ))?;
@@ -291,7 +149,7 @@ impl GdriveCore {
     }
 
     pub async fn gdrive_get(&self, path: &str) -> Result<Response<IncomingAsyncBody>> {
-        let path_id = self.get_file_id_by_path(path).await?.ok_or(Error::new(
+        let path_id = self.path_cache.get(path).await?.ok_or(Error::new(
             ErrorKind::NotFound,
             &format!("path not found: {}", path),
         ))?;
@@ -339,7 +197,7 @@ impl GdriveCore {
         source: &str,
         target: &str,
     ) -> Result<Response<IncomingAsyncBody>> {
-        let file_id = self.get_file_id_by_path(source).await?.ok_or(Error::new(
+        let file_id = self.path_cache.get(source).await?.ok_or(Error::new(
             ErrorKind::NotFound,
             &format!("source path not found: {}", source),
         ))?;
@@ -355,7 +213,7 @@ impl GdriveCore {
             .collect();
         source_parent.pop();
 
-        let cache = self.path_cache.lock().await;
+        let cache = &self.path_cache;
 
         let file_name = build_abs_path(&self.root, target)
             .split('/')
@@ -364,9 +222,13 @@ impl GdriveCore {
             .unwrap()
             .to_string();
 
+        let old_parent = cache
+            .get(&source_parent.join("/"))
+            .await?
+            .expect("old parent must exist");
         let metadata = &json!({
             "name": file_name,
-            "removeParents": [cache.get(&source_parent.join("/")).unwrap().to_string()],
+            "removeParents": [old_parent],
             "addParents": [parent],
         });
 
@@ -380,7 +242,7 @@ impl GdriveCore {
     }
 
     pub async fn gdrive_delete(&self, path: &str) -> Result<Response<IncomingAsyncBody>> {
-        let file_id = self.get_file_id_by_path(path).await?.ok_or(Error::new(
+        let file_id = self.path_cache.get(path).await?.ok_or(Error::new(
             ErrorKind::NotFound,
             &format!("path not found: {}", path),
         ))?;
@@ -470,9 +332,40 @@ impl GdriveCore {
 
     pub async fn sign<T>(&self, req: &mut Request<T>) -> Result<()> {
         let mut signer = self.signer.lock().await;
+        signer.sign(req).await
+    }
+}
+
+#[derive(Clone)]
+pub struct GdriveSigner {
+    pub client: HttpClient,
+
+    pub client_id: String,
+    pub client_secret: String,
+    pub refresh_token: String,
 
-        if !signer.access_token.is_empty() && signer.expires_in > Utc::now() {
-            let value = format!("Bearer {}", signer.access_token)
+    pub access_token: String,
+    pub expires_in: DateTime<Utc>,
+}
+
+impl GdriveSigner {
+    /// Create a new signer.
+    pub fn new(client: HttpClient) -> Self {
+        GdriveSigner {
+            client,
+
+            client_id: "".to_string(),
+            client_secret: "".to_string(),
+            refresh_token: "".to_string(),
+            access_token: "".to_string(),
+            expires_in: DateTime::<Utc>::MIN_UTC,
+        }
+    }
+
+    /// Sign a request.
+    pub async fn sign<T>(&mut self, req: &mut Request<T>) -> Result<()> {
+        if !self.access_token.is_empty() && self.expires_in > Utc::now() {
+            let value = format!("Bearer {}", self.access_token)
                 .parse()
                 .expect("access token must be valid header value");
 
@@ -482,7 +375,7 @@ impl GdriveCore {
 
         let url = format!(
             "https://oauth2.googleapis.com/token?refresh_token={}&client_id={}&client_secret={}&grant_type=refresh_token",
-            signer.refresh_token, signer.client_id, signer.client_secret
+            self.refresh_token, self.client_id, self.client_secret
         );
 
         {
@@ -498,8 +391,8 @@ impl GdriveCore {
                     let resp_body = &resp.into_body().bytes().await?;
                     let token = serde_json::from_slice::<GdriveTokenResponse>(resp_body)
                         .map_err(new_json_deserialize_error)?;
-                    signer.access_token = token.access_token.clone();
-                    signer.expires_in = Utc::now() + chrono::Duration::seconds(token.expires_in)
+                    self.access_token = token.access_token.clone();
+                    self.expires_in = Utc::now() + chrono::Duration::seconds(token.expires_in)
                         - chrono::Duration::seconds(120);
                 }
                 _ => {
@@ -508,7 +401,7 @@ impl GdriveCore {
             }
         }
 
-        let auth_header_content = format!("Bearer {}", signer.access_token);
+        let auth_header_content = format!("Bearer {}", self.access_token);
         req.headers_mut()
             .insert(header::AUTHORIZATION, auth_header_content.parse().unwrap());
 
@@ -516,25 +409,61 @@ impl GdriveCore {
     }
 }
 
-#[derive(Clone)]
-pub struct GdriveSigner {
-    pub client_id: String,
-    pub client_secret: String,
-    pub refresh_token: String,
+pub struct GdrivePathQuery {
+    pub client: HttpClient,
+    pub signer: Arc<Mutex<GdriveSigner>>,
+}
 
-    pub access_token: String,
-    pub expires_in: DateTime<Utc>,
+impl GdrivePathQuery {
+    pub fn new(client: HttpClient, signer: Arc<Mutex<GdriveSigner>>) -> Self {
+        GdrivePathQuery { client, signer }
+    }
 }
 
-impl Default for GdriveSigner {
-    fn default() -> Self {
-        GdriveSigner {
-            access_token: String::new(),
-            expires_in: DateTime::<Utc>::MIN_UTC,
+#[async_trait]
+impl PathQuery for GdrivePathQuery {
+    async fn root(&self) -> Result<String> {
+        Ok("root".to_string())
+    }
+
+    async fn query(&self, parent_id: &str, name: &str) -> Result<Option<String>> {
+        let mut queries = vec![
+            format!("name = \"{}\"", name),
+            format!("\"{}\" in parents", parent_id),
+            "trashed = false".to_string(),
+        ];
+        if name.ends_with('/') {
+            queries.push("mimeType = 'application/vnd.google-apps.folder'".to_string());
+        }
+        let query = queries.join(" and ");
+
+        let url = format!(
+            "https://www.googleapis.com/drive/v3/files?q={}",
+            percent_encode_path(query.as_str())
+        );
+
+        let mut req = Request::get(&url)
+            .body(AsyncBody::Empty)
+            .map_err(new_request_build_error)?;
 
-            refresh_token: String::new(),
-            client_id: String::new(),
-            client_secret: String::new(),
+        self.signer.lock().await.sign(&mut req).await?;
+
+        let resp = self.client.send(req).await?;
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let body = resp.into_body().bytes().await?;
+                let meta: GdriveFileList =
+                    serde_json::from_slice(&body).map_err(new_json_deserialize_error)?;
+
+                if let Some(f) = meta.files.first() {
+                    Ok(Some(f.id.clone()))
+                } else {
+                    Ok(None)
+                }
+            }
+            _ => Err(parse_error(resp).await?),
         }
     }
 }
diff --git a/core/src/services/gdrive/lister.rs b/core/src/services/gdrive/lister.rs
index 9da239035d..7c665376af 100644
--- a/core/src/services/gdrive/lister.rs
+++ b/core/src/services/gdrive/lister.rs
@@ -41,7 +41,7 @@ impl GdriveLister {
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 impl oio::PageList for GdriveLister {
     async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
-        let file_id = self.core.get_file_id_by_path(&self.path).await?;
+        let file_id = self.core.path_cache.get(&self.path).await?;
 
         let file_id = match file_id {
             Some(file_id) => file_id,