You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/09/25 09:06:18 UTC
(opendal) branch main updated: test(core): add tests for versioning (#5132)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new d479ac39ef test(core): add tests for versioning (#5132)
d479ac39ef is described below
commit d479ac39efa9a88e10448ad81e47292b3b4ed688
Author: meteorgan <me...@gmail.com>
AuthorDate: Wed Sep 25 17:06:12 2024 +0800
test(core): add tests for versioning (#5132)
* test(core): add tests for versioning
* add tests for version doesn't match
* update comments
* add stat_with_versioning, read_with_versioning, delete_with_versioning, list_with_versioning_capabilities
* read/stat non-existent version returns NotFound, delete non-existent version returns Ok
* fix docs
---
core/src/services/s3/backend.rs | 5 +-
core/src/types/capability.rs | 14 ++++--
core/src/types/operator/operator_futures.rs | 2 +-
core/tests/behavior/async_delete.rs | 71 +++++++++++++++++++++++++-
core/tests/behavior/async_read.rs | 67 ++++++++++++++++++++++++-
core/tests/behavior/async_stat.rs | 78 +++++++++++++++++++++++++++--
6 files changed, 224 insertions(+), 13 deletions(-)
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 745b292f2b..8073cbfbb9 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -902,14 +902,15 @@ impl Access for S3Backend {
stat_with_override_cache_control: !self.core.disable_stat_with_override,
stat_with_override_content_disposition: !self.core.disable_stat_with_override,
stat_with_override_content_type: !self.core.disable_stat_with_override,
+ stat_with_version: self.core.enable_versioning,
read: true,
-
read_with_if_match: true,
read_with_if_none_match: true,
read_with_override_cache_control: true,
read_with_override_content_disposition: true,
read_with_override_content_type: true,
+ read_with_version: self.core.enable_versioning,
write: true,
write_can_empty: true,
@@ -932,6 +933,8 @@ impl Access for S3Backend {
},
delete: true,
+ delete_with_version: self.core.enable_versioning,
+
copy: true,
list: true,
diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs
index cae69f04ab..4a874b2387 100644
--- a/core/src/types/capability.rs
+++ b/core/src/types/capability.rs
@@ -61,12 +61,14 @@ pub struct Capability {
pub stat_with_if_match: bool,
/// If operator supports stat with if none match.
pub stat_with_if_none_match: bool,
- /// if operator supports read with override cache control.
+ /// if operator supports stat with override cache control.
pub stat_with_override_cache_control: bool,
- /// if operator supports read with override content disposition.
+ /// if operator supports stat with override content disposition.
pub stat_with_override_content_disposition: bool,
- /// if operator supports read with override content type.
+ /// if operator supports stat with override content type.
pub stat_with_override_content_type: bool,
+ /// if operator supports stat with version.
+ pub stat_with_version: bool,
/// If operator supports read.
pub read: bool,
@@ -80,6 +82,8 @@ pub struct Capability {
pub read_with_override_content_disposition: bool,
/// if operator supports read with override content type.
pub read_with_override_content_type: bool,
+ /// if operator supports read with version.
+ pub read_with_version: bool,
/// If operator supports write.
pub write: bool,
@@ -119,6 +123,8 @@ pub struct Capability {
/// If operator supports delete.
pub delete: bool,
+ /// if operator supports delete with version.
+ pub delete_with_version: bool,
/// If operator supports copy.
pub copy: bool,
@@ -134,7 +140,7 @@ pub struct Capability {
pub list_with_start_after: bool,
/// If backend supports list with recursive.
pub list_with_recursive: bool,
- /// If backend supports list with object version.
+ /// if operator supports list with version.
pub list_with_version: bool,
/// If operator supports presign.
diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs
index 777c817e0f..b5ddd1d4c3 100644
--- a/core/src/types/operator/operator_futures.rs
+++ b/core/src/types/operator/operator_futures.rs
@@ -406,7 +406,7 @@ impl<F: Future<Output = Result<Writer>>> FutureWriter<F> {
///
/// ## Notes
///
- /// we don't need to include the user defined metadata prefix in the key
+ /// we don't need to include the user defined metadata prefix in the key.
/// every service will handle it internally
pub fn user_metadata(self, data: impl IntoIterator<Item = (String, String)>) -> Self {
self.map(|(args, options)| (args.with_user_metadata(HashMap::from_iter(data)), options))
diff --git a/core/tests/behavior/async_delete.rs b/core/tests/behavior/async_delete.rs
index ab51712d69..0225f50ab6 100644
--- a/core/tests/behavior/async_delete.rs
+++ b/core/tests/behavior/async_delete.rs
@@ -33,7 +33,9 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
test_delete_with_special_chars,
test_delete_not_existing,
test_delete_stream,
- test_remove_one_file
+ test_remove_one_file,
+ test_delete_with_version,
+ test_delete_with_not_existing_version
));
if cap.list_with_recursive {
tests.extend(async_trials!(op, test_remove_all_basic));
@@ -212,3 +214,70 @@ pub async fn test_remove_all_with_prefix_exists(op: Operator) -> Result<()> {
.expect("write must succeed");
test_blocking_remove_all_with_objects(op, parent, ["a", "a/b", "a/c", "a/b/e"]).await
}
+
+pub async fn test_delete_with_version(op: Operator) -> Result<()> {
+ if !op.info().full_capability().delete_with_version {
+ return Ok(());
+ }
+
+ let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+
+ op.write(path.as_str(), content)
+ .await
+ .expect("write must success");
+ let meta = op.stat(path.as_str()).await.expect("stat must success");
+ let version = meta.version().expect("must have version");
+
+ op.delete(path.as_str()).await.expect("delete must success");
+ assert!(!op.is_exist(path.as_str()).await?);
+
+ // After a simple delete, the data can still be accessed using its version.
+ let meta = op
+ .stat_with(path.as_str())
+ .version(version)
+ .await
+ .expect("stat must success");
+ assert_eq!(version, meta.version().expect("must have version"));
+
+ // After deleting with the version, the data is removed permanently
+ op.delete_with(path.as_str())
+ .version(version)
+ .await
+ .expect("delete must success");
+ let ret = op.stat_with(path.as_str()).version(version).await;
+ assert!(ret.is_err());
+ assert_eq!(ret.unwrap_err().kind(), ErrorKind::NotFound);
+
+ Ok(())
+}
+
+pub async fn test_delete_with_not_existing_version(op: Operator) -> Result<()> {
+ if !op.info().full_capability().delete_with_version {
+ return Ok(());
+ }
+
+ // retrieve a valid version
+ let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+ op.write(path.as_str(), content)
+ .await
+ .expect("write must success");
+ let version = op
+ .stat(path.as_str())
+ .await
+ .expect("stat must success")
+ .version()
+ .expect("must have stat")
+ .to_string();
+
+ let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+ op.write(path.as_str(), content)
+ .await
+ .expect("write must success");
+ let ret = op
+ .delete_with(path.as_str())
+ .version(version.as_str())
+ .await;
+ assert!(ret.is_ok());
+
+ Ok(())
+}
diff --git a/core/tests/behavior/async_read.rs b/core/tests/behavior/async_read.rs
index 53e6f29a61..d691165e5f 100644
--- a/core/tests/behavior/async_read.rs
+++ b/core/tests/behavior/async_read.rs
@@ -44,7 +44,9 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
test_read_with_special_chars,
test_read_with_override_cache_control,
test_read_with_override_content_disposition,
- test_read_with_override_content_type
+ test_read_with_override_content_type,
+ test_read_with_version,
+ test_read_with_not_existing_version
))
}
@@ -553,3 +555,66 @@ pub async fn test_read_only_read_with_if_none_match(op: Operator) -> anyhow::Res
Ok(())
}
+
+pub async fn test_read_with_version(op: Operator) -> anyhow::Result<()> {
+ if !op.info().full_capability().read_with_version {
+ return Ok(());
+ }
+
+ let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+ op.write(path.as_str(), content.clone())
+ .await
+ .expect("write must success");
+ let meta = op.stat(path.as_str()).await.expect("stat must success");
+ let version = meta.version().expect("must have version");
+
+ let data = op
+ .read_with(path.as_str())
+ .version(version)
+ .await
+ .expect("read must success");
+ assert_eq!(content, data.to_vec());
+
+ op.write(path.as_str(), "1")
+ .await
+ .expect("write must success");
+
+ // After writing new data, we can still read the first version data
+ let second_data = op
+ .read_with(path.as_str())
+ .version(version)
+ .await
+ .expect("read must success");
+ assert_eq!(content, second_data.to_vec());
+
+ Ok(())
+}
+
+pub async fn test_read_with_not_existing_version(op: Operator) -> anyhow::Result<()> {
+ if !op.info().full_capability().read_with_version {
+ return Ok(());
+ }
+
+ // retrieve a valid version
+ let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+ op.write(path.as_str(), content.clone())
+ .await
+ .expect("write must success");
+ let version = op
+ .stat(path.as_str())
+ .await
+ .expect("stat must success")
+ .version()
+ .expect("must have version")
+ .to_string();
+
+ let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+ op.write(path.as_str(), content)
+ .await
+ .expect("write must success");
+ let ret = op.read_with(path.as_str()).version(&version).await;
+ assert!(ret.is_err());
+ assert_eq!(ret.unwrap_err().kind(), ErrorKind::NotFound);
+
+ Ok(())
+}
diff --git a/core/tests/behavior/async_stat.rs b/core/tests/behavior/async_stat.rs
index 2dcbde9b2d..38f50704ac 100644
--- a/core/tests/behavior/async_stat.rs
+++ b/core/tests/behavior/async_stat.rs
@@ -18,13 +18,12 @@
use std::str::FromStr;
use std::time::Duration;
+use crate::*;
use anyhow::Result;
use http::StatusCode;
use log::warn;
use reqwest::Url;
-use crate::*;
-
pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
let cap = op.info().full_capability();
@@ -42,7 +41,9 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
test_stat_with_override_cache_control,
test_stat_with_override_content_disposition,
test_stat_with_override_content_type,
- test_stat_root
+ test_stat_root,
+ test_stat_with_version,
+ stat_with_not_existing_version
))
}
@@ -166,12 +167,12 @@ pub async fn test_stat_not_cleaned_path(op: Operator) -> Result<()> {
pub async fn test_stat_not_exist(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- // Stat not exist file should returns NotFound.
+ // Stat not exist file should return NotFound.
let meta = op.stat(&path).await;
assert!(meta.is_err());
assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound);
- // Stat not exist dir should also returns NotFound.
+ // Stat not exist dir should also return NotFound.
if op.info().full_capability().create_dir {
let meta = op.stat(&format!("{path}/")).await;
assert!(meta.is_err());
@@ -499,3 +500,70 @@ pub async fn test_read_only_stat_root(op: Operator) -> Result<()> {
Ok(())
}
+
+pub async fn test_stat_with_version(op: Operator) -> Result<()> {
+ if !op.info().full_capability().stat_with_version {
+ return Ok(());
+ }
+
+ let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+
+ op.write(path.as_str(), content.clone())
+ .await
+ .expect("write must success");
+ let first_meta = op.stat(path.as_str()).await.expect("stat must success");
+ let first_version = first_meta.version().expect("must have version");
+
+ let first_versioning_meta = op
+ .stat_with(path.as_str())
+ .version(first_version)
+ .await
+ .expect("stat must success");
+ assert_eq!(first_meta, first_versioning_meta);
+
+ op.write(path.as_str(), content)
+ .await
+ .expect("write must success");
+ let second_meta = op.stat(path.as_str()).await.expect("stat must success");
+ let second_version = second_meta.version().expect("must have version");
+ assert_ne!(first_version, second_version);
+
+ // we can still `stat` with first_version after writing new data
+ let meta = op
+ .stat_with(path.as_str())
+ .version(first_version)
+ .await
+ .expect("stat must success");
+ assert_eq!(first_meta, meta);
+
+ Ok(())
+}
+
+pub async fn stat_with_not_existing_version(op: Operator) -> Result<()> {
+ if !op.info().full_capability().stat_with_version {
+ return Ok(());
+ }
+
+ // retrieve a valid version
+ let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+ op.write(path.as_str(), content.clone())
+ .await
+ .expect("write must success");
+ let version = op
+ .stat(path.as_str())
+ .await
+ .expect("stat must success")
+ .version()
+ .expect("must have version")
+ .to_string();
+
+ let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+ op.write(path.as_str(), content)
+ .await
+ .expect("write must success");
+ let ret = op.stat_with(path.as_str()).version(version.as_str()).await;
+ assert!(ret.is_err());
+ assert_eq!(ret.unwrap_err().kind(), ErrorKind::NotFound);
+
+ Ok(())
+}