You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/08/27 03:42:39 UTC
(opendal) branch main updated: fix(core): Fix failed list related tests (#5058)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 8bc883f971 fix(core): Fix failed list related tests (#5058)
8bc883f971 is described below
commit 8bc883f971902f0b9df335b95ba4cdf4aca24702
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Aug 27 11:42:32 2024 +0800
fix(core): Fix failed list related tests (#5058)
* fix(core): Fix failed list related tests
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix build
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix azdls
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix b2
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix b2
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix aliyun drive
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix aliyun drive
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/oio/entry.rs | 7 ++-
core/src/services/aliyun_drive/lister.rs | 91 +++++++++++---------------------
core/src/services/azdls/lister.rs | 8 ++-
core/src/services/azfile/lister.rs | 6 +++
core/src/services/b2/lister.rs | 3 --
core/src/services/cos/lister.rs | 7 ++-
core/src/services/gdrive/lister.rs | 7 +++
core/src/services/sqlite/backend.rs | 5 +-
8 files changed, 62 insertions(+), 72 deletions(-)
diff --git a/core/src/raw/oio/entry.rs b/core/src/raw/oio/entry.rs
index 43c4b96311..70320300b0 100644
--- a/core/src/raw/oio/entry.rs
+++ b/core/src/raw/oio/entry.rs
@@ -38,7 +38,12 @@ impl Entry {
}
/// Create a new entry with given value.
- pub fn with(path: String, meta: Metadata) -> Entry {
+ pub fn with(mut path: String, meta: Metadata) -> Entry {
+ // Normalize path as `/` if it's empty.
+ if path.is_empty() {
+ path = "/".to_string();
+ }
+
debug_assert!(
meta.mode().is_dir() == path.ends_with('/'),
"mode {:?} not match with path {}",
diff --git a/core/src/services/aliyun_drive/lister.rs b/core/src/services/aliyun_drive/lister.rs
index a94db21bcd..4a28e14600 100644
--- a/core/src/services/aliyun_drive/lister.rs
+++ b/core/src/services/aliyun_drive/lister.rs
@@ -65,26 +65,19 @@ impl oio::PageList for AliyunDriveLister {
};
let offset = if ctx.token.is_empty() {
- if !parent.path.ends_with('/') {
- // List "dir" should contains "dir/".
- let path = if !parent.path.starts_with('/') {
- format!("/{}", parent.path)
- } else {
- parent.path.clone()
- };
- ctx.entries.push_back(Entry::new(
- &format!("{}/", path),
- Metadata::new(EntryMode::DIR).with_last_modified(
- parent
- .updated_at
- .parse::<chrono::DateTime<Utc>>()
- .map_err(|e| {
- Error::new(ErrorKind::Unexpected, "parse last modified time")
- .set_source(e)
- })?,
- ),
- ));
- }
+ // Push self into the list result.
+ ctx.entries.push_back(Entry::new(
+ &parent.path,
+ Metadata::new(EntryMode::DIR).with_last_modified(
+ parent
+ .updated_at
+ .parse::<chrono::DateTime<Utc>>()
+ .map_err(|e| {
+ Error::new(ErrorKind::Unexpected, "parse last modified time")
+ .set_source(e)
+ })?,
+ ),
+ ));
None
} else {
Some(ctx.token.clone())
@@ -107,57 +100,35 @@ impl oio::PageList for AliyunDriveLister {
let result: AliyunDriveFileList =
serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
- let n = result.items.len();
-
for item in result.items {
- let path = if parent.path.starts_with('/') {
- build_abs_path(&parent.path, &item.name)
+ let (path, mut md) = if item.path_type == "folder" {
+ let path = format!("{}{}/", &parent.path.trim_start_matches('/'), &item.name);
+ (path, Metadata::new(EntryMode::DIR))
} else {
- build_abs_path(&format!("/{}", &parent.path), &item.name)
+ let path = format!("{}{}", &parent.path.trim_start_matches('/'), &item.name);
+ (path, Metadata::new(EntryMode::FILE))
};
- let (path, md) = if item.path_type == "folder" {
- let path = format!("{}/", path);
- let meta = Metadata::new(EntryMode::DIR).with_last_modified(
- item.updated_at
- .parse::<chrono::DateTime<Utc>>()
- .map_err(|e| {
- Error::new(ErrorKind::Unexpected, "parse last modified time")
- .set_source(e)
- })?,
- );
- (path, meta)
- } else {
- let mut meta = Metadata::new(EntryMode::FILE).with_last_modified(
- item.updated_at
- .parse::<chrono::DateTime<Utc>>()
- .map_err(|e| {
- Error::new(ErrorKind::Unexpected, "parse last modified time")
- .set_source(e)
- })?,
- );
- if let Some(v) = item.size {
- meta = meta.with_content_length(v);
- }
- if let Some(v) = item.content_type {
- meta = meta.with_content_type(v);
- }
- (path, meta)
- };
+ md = md.with_last_modified(item.updated_at.parse::<chrono::DateTime<Utc>>().map_err(
+ |e| Error::new(ErrorKind::Unexpected, "parse last modified time").set_source(e),
+ )?);
+ if let Some(v) = item.size {
+ md = md.with_content_length(v);
+ }
+ if let Some(v) = item.content_type {
+ md = md.with_content_type(v);
+ }
ctx.entries.push_back(Entry::new(&path, md));
}
- if self.limit.is_some_and(|x| x < n) || result.next_marker.is_none() {
+ let next_marker = result.next_marker.unwrap_or_default();
+ if next_marker.is_empty() {
ctx.done = true;
+ } else {
+ ctx.token = next_marker;
}
- if let Some(marker) = result.next_marker {
- if marker.is_empty() {
- ctx.done = true;
- }
- ctx.token = marker;
- }
Ok(())
}
}
diff --git a/core/src/services/azdls/lister.rs b/core/src/services/azdls/lister.rs
index 27d51e2667..7fd8c2cdc7 100644
--- a/core/src/services/azdls/lister.rs
+++ b/core/src/services/azdls/lister.rs
@@ -56,6 +56,12 @@ impl oio::PageList for AzdlsLister {
return Err(parse_error(resp).await?);
}
+ // Return self at the first page.
+ if ctx.token.is_empty() && !ctx.done {
+ let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR));
+ ctx.entries.push_back(e);
+ }
+
// Check whether this list is done.
if let Some(value) = resp.headers().get("x-ms-continuation") {
let value = value.to_str().map_err(|err| {
@@ -90,7 +96,7 @@ impl oio::PageList for AzdlsLister {
.with_last_modified(parse_datetime_from_rfc2822(&object.last_modified)?);
let mut path = build_rel_path(&self.core.root, &object.name);
- if mode == EntryMode::DIR {
+ if mode.is_dir() {
path += "/"
};
diff --git a/core/src/services/azfile/lister.rs b/core/src/services/azfile/lister.rs
index 913af07d83..127072a561 100644
--- a/core/src/services/azfile/lister.rs
+++ b/core/src/services/azfile/lister.rs
@@ -56,6 +56,12 @@ impl oio::PageList for AzfileLister {
return Err(parse_error(resp).await?);
}
+ // Return self at the first page.
+ if ctx.token.is_empty() && !ctx.done {
+ let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR));
+ ctx.entries.push_back(e);
+ }
+
let bs = resp.into_body();
let results: EnumerationResults =
diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs
index 9b27b121c1..7b78edb76e 100644
--- a/core/src/services/b2/lister.rs
+++ b/core/src/services/b2/lister.rs
@@ -96,9 +96,6 @@ impl oio::PageList for B2Lister {
continue;
}
}
- if file.file_name == build_abs_path(&self.core.root, &self.path) {
- continue;
- }
let file_name = file.file_name.clone();
let metadata = parse_file_info(&file);
diff --git a/core/src/services/cos/lister.rs b/core/src/services/cos/lister.rs
index 7022aa9def..b5b66596fa 100644
--- a/core/src/services/cos/lister.rs
+++ b/core/src/services/cos/lister.rs
@@ -81,10 +81,9 @@ impl oio::PageList for CosLister {
}
for object in output.contents {
- let path = build_rel_path(&self.core.root, &object.key);
-
- if path == self.path || path.is_empty() {
- continue;
+ let mut path = build_rel_path(&self.core.root, &object.key);
+ if path.is_empty() {
+ path = "/".to_string();
}
let meta = Metadata::new(EntryMode::from_path(&path)).with_content_length(object.size);
diff --git a/core/src/services/gdrive/lister.rs b/core/src/services/gdrive/lister.rs
index e77a5022da..108df8b42a 100644
--- a/core/src/services/gdrive/lister.rs
+++ b/core/src/services/gdrive/lister.rs
@@ -64,6 +64,13 @@ impl oio::PageList for GdriveLister {
return Ok(());
}
+ // Return self at the first page.
+ if ctx.token.is_empty() && !ctx.done {
+ let path = build_rel_path(&self.core.root, &self.path);
+ let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR));
+ ctx.entries.push_back(e);
+ }
+
let decoded_response =
serde_json::from_slice::<GdriveFileList>(&bytes).map_err(new_json_deserialize_error)?;
diff --git a/core/src/services/sqlite/backend.rs b/core/src/services/sqlite/backend.rs
index a8f182a901..f353292ad7 100644
--- a/core/src/services/sqlite/backend.rs
+++ b/core/src/services/sqlite/backend.rs
@@ -251,11 +251,10 @@ impl kv::Adapter for Adapter {
let pool = self.get_client().await?;
let value = sqlx::query_scalar(&format!(
- "SELECT `{}` FROM `{}` WHERE `{}` LIKE $1 and `{}` <> $2",
- self.key_field, self.table, self.key_field, self.key_field
+ "SELECT `{}` FROM `{}` WHERE `{}` LIKE $1",
+ self.key_field, self.table, self.key_field
))
.bind(format!("{path}%"))
- .bind(path)
.fetch_all(pool)
.await
.map_err(parse_sqlite_error)?;