You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by su...@apache.org on 2023/09/18 04:33:45 UTC
[incubator-opendal] branch empty-file-test updated (f11f868d1 -> 2d59e9ae4)
This is an automated email from the ASF dual-hosted git repository.
suyanhanx pushed a change to branch empty-file-test
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
omit f11f868d1 fix append close
omit 8a49a6910 fmt
omit 614ec11c1 remove empty file write test
omit 5ce8ce12d fmt
omit 235fd206c test(blocking): tests for blocking append
add 292cd8846 ci(services/tikv): add tikv integration test with tls (#3026)
add 3bced977b refactor: Relax bounds on Writer::{sink, copy} (#3027)
add 8407928cc docs: add service doc for gdrive & onedrive (#3028)
add 40d87937c chore(bindings/cpp): update CMakeLists.txt to prepare release (#3030)
add 587e2d977 refactor: Refactor oio::Write into poll-based to create more room for optimization (#3029)
add 4a52ba64c chore: fix typo of SftpWriter error message (#3032)
add bf85c1fd3 refactor: Polish multipart writer to allow oneshot optimization (#3031)
add 57581b90a feat(service/gdrive): add gdrive list support (#3025)
add 088881f42 refactor: Polish implementation details of WriteBuf and add vector chunks support (#3034)
add 2b83c710d refactor: Add ChunkedBytes to improve the exact buf write (#3035)
add 9535d0222 refactor: Polish RangeWrite implementation to remove the extra buffer logic (#3038)
add fb04515a3 ci: restrict workflow that need password (#3039)
add 082ebca8f feat(services/etcd): Enable etcd connection pool (#3041)
add f9c68be75 fix(services/oss): set content_md5 in lister (#3043)
add fda9670d6 refactor: Remove the requirement of passing `content_length` to writer (#3044)
add e8f695e36 feat: Add buffer support for all services (#3045)
add 0e0753af4 fix: Correct the name of azdfs to azdls (#3046)
add e895dced5 ci: Don't release while tag contains rc (#3048)
add 56d711907 fix: Don't apply blocking layer when service support blocking (#3050)
add eae0115cc feat(bindings/java): auto enable blocking layer (#3049)
add 08bf79ba4 test(bindings/java): fix mapped port (#3051)
add 7d5524f35 fix: call `flush` before `sync_all` (#3053)
add 9ac7bb9d8 ci(bindings/java): skip RedisServiceTest on macos and windows (#3054)
add 459431e30 ci: Disable PHP build temporarily (#3058)
add 91435233e ci(bindings/java): release workflow always uses bash (#3056)
add aad56c48a ci(binding/java): Enable release when OPENDAL_RELEASE is on (#3057)
add f7e33bf0a ci(binding/java): Use cargo profile instead of --release (#3059)
add a8518bd47 ci: Move platform build checks from java binding to rust core (#3060)
add 8175aa35c chore: Polish some details of layers implementation (#3061)
add 12bd76774 docs(services/sftp): update comments about password login (#3065)
new 2d59e9ae4 test for blocking append & fix fs
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f11f868d1)
\
N -- N -- N refs/heads/empty-file-test (2d59e9ae4)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/bindings_c.yml | 1 +
.github/workflows/bindings_cpp.yml | 9 +-
.github/workflows/bindings_java.yml | 23 +-
.github/workflows/bindings_nodejs.yml | 2 +-
.github/workflows/bindings_php.yml | 24 +-
.github/workflows/bindings_python.yml | 2 +-
.github/workflows/ci.yml | 89 ++-
.github/workflows/docs.yml | 2 +-
.github/workflows/publish.yml | 9 +-
.github/workflows/release_java.yml | 24 +-
.github/workflows/service_test_azblob.yml | 2 +-
...rvice_test_azdfs.yml => service_test_azdls.yml} | 20 +-
.github/workflows/service_test_cos.yml | 2 +-
.github/workflows/service_test_dropbox.yml | 2 +-
.github/workflows/service_test_gdrive.yml | 2 +-
.github/workflows/service_test_s3.yml | 2 +-
.github/workflows/service_test_supabase.yml | 2 +-
.github/workflows/service_test_tikv.yml | 68 ++-
.github/workflows/service_test_wasabi.yml | 2 +-
.github/workflows/service_test_webhdfs.yml | 4 +-
.typos.toml | 1 +
README.md | 4 +-
bin/oli/README.md | 2 +-
bin/oli/src/config/mod.rs | 4 +-
bindings/cpp/CMakeLists.txt | 148 +++--
bindings/cpp/CONTRIBUTING.md | 6 +-
bindings/cpp/README.md | 51 +-
bindings/cpp/src/lib.rs | 3 +-
bindings/cpp/src/lister.rs | 3 +-
bindings/cpp/src/reader.rs | 3 +-
bindings/cpp/src/types.rs | 3 +-
bindings/java/Cargo.toml | 94 +++-
bindings/java/pom.xml | 35 +-
bindings/java/src/blocking_operator.rs | 10 +-
bindings/java/src/error.rs | 7 +-
bindings/java/src/metadata.rs | 3 +-
bindings/java/src/operator.rs | 9 +-
.../java/org/apache/opendal/RedisServiceTest.java | 80 +++
bindings/java/tools/build.py | 13 +-
core/Cargo.toml | 6 +-
core/README.md | 4 +-
core/benches/oio/utils.rs | 19 +-
core/benches/oio/write.rs | 2 +-
core/fuzz/fuzz_writer.rs | 2 +-
core/src/layers/blocking.rs | 15 +-
core/src/layers/complete.rs | 602 +++++++++++----------
core/src/layers/concurrent_limit.rs | 12 +-
core/src/layers/error_context.rs | 49 +-
core/src/layers/logging.rs | 49 +-
core/src/layers/madsim.rs | 22 +-
core/src/layers/metrics.rs | 15 +-
core/src/layers/minitrace.rs | 36 +-
core/src/layers/mod.rs | 24 +-
core/src/layers/oteltrace.rs | 45 +-
core/src/layers/prometheus.rs | 15 +-
core/src/layers/retry.rs | 132 +++--
core/src/layers/throttle.rs | 22 +-
core/src/layers/timeout.rs | 103 +++-
core/src/layers/tracing.rs | 12 +-
core/src/layers/type_eraser.rs | 26 +-
core/src/raw/adapters/kv/backend.rs | 135 +++--
core/src/raw/adapters/typed_kv/api.rs | 8 +-
core/src/raw/adapters/typed_kv/backend.rs | 86 ++-
core/src/raw/http_util/body.rs | 4 +
core/src/raw/http_util/client.rs | 1 +
core/src/raw/http_util/multipart.rs | 1 +
core/src/raw/oio/buf/chunked_bytes.rs | 521 ++++++++++++++++++
core/src/{services/cos => raw/oio/buf}/mod.rs | 10 +-
core/src/raw/oio/{buf.rs => buf/write_buf.rs} | 197 ++++---
core/src/raw/oio/cursor.rs | 560 -------------------
core/src/raw/oio/mod.rs | 4 +-
core/src/raw/oio/page/api.rs | 9 +-
core/src/raw/oio/read/cloneable_read.rs | 11 +-
core/src/raw/oio/read/into_read_from_stream.rs | 12 +-
core/src/raw/oio/write/api.rs | 123 ++++-
core/src/raw/oio/write/append_object_write.rs | 97 ++--
core/src/raw/oio/write/compose_write.rs | 45 +-
core/src/raw/oio/write/exact_buf_write.rs | 105 +---
core/src/raw/oio/write/mod.rs | 5 +
core/src/raw/oio/write/multipart_upload_write.rs | 274 ++++++++--
core/src/raw/oio/write/one_shot_write.rs | 92 +++-
core/src/raw/oio/write/range_write.rs | 286 ++++++++++
core/src/raw/ops.rs | 35 +-
core/src/services/azblob/backend.rs | 15 +-
core/src/services/azblob/core.rs | 15 +-
core/src/services/azblob/writer.rs | 100 +---
core/src/services/{azdfs => azdls}/backend.rs | 107 ++--
core/src/services/{azdfs => azdls}/core.rs | 28 +-
core/src/services/{azdfs => azdls}/docs.md | 12 +-
core/src/services/{azdfs => azdls}/error.rs | 12 +-
core/src/services/{azdfs => azdls}/mod.rs | 2 +-
core/src/services/{azdfs => azdls}/pager.rs | 20 +-
core/src/services/{azdfs => azdls}/writer.rs | 42 +-
core/src/services/cos/backend.rs | 34 +-
core/src/services/cos/writer.rs | 19 +-
core/src/services/dropbox/backend.rs | 14 +-
core/src/services/dropbox/writer.rs | 20 +-
core/src/services/etcd/backend.rs | 52 +-
core/src/services/fs/backend.rs | 50 +-
core/src/services/fs/writer.rs | 124 +++--
core/src/services/ftp/backend.rs | 20 +-
core/src/services/ftp/writer.rs | 25 +-
core/src/services/gcs/backend.rs | 47 +-
core/src/services/gcs/core.rs | 32 +-
core/src/services/gcs/writer.rs | 134 ++---
core/src/services/gdrive/backend.rs | 31 +-
core/src/services/gdrive/builder.rs | 69 +--
core/src/services/gdrive/core.rs | 65 ++-
core/src/services/gdrive/docs.md | 65 +++
core/src/services/gdrive/error.rs | 32 +-
core/src/services/gdrive/mod.rs | 1 +
core/src/services/gdrive/pager.rs | 102 ++++
core/src/services/gdrive/writer.rs | 33 +-
core/src/services/ghac/backend.rs | 9 +-
core/src/services/ghac/writer.rs | 143 +++--
core/src/services/hdfs/writer.rs | 25 +-
core/src/services/ipmfs/backend.rs | 19 +-
core/src/services/ipmfs/writer.rs | 22 +-
core/src/services/mod.rs | 8 +-
core/src/services/obs/backend.rs | 34 +-
core/src/services/obs/writer.rs | 19 +-
core/src/services/onedrive/backend.rs | 11 +-
core/src/services/onedrive/builder.rs | 46 +-
core/src/services/{ftp => onedrive}/docs.md | 24 +-
core/src/services/onedrive/writer.rs | 19 +-
core/src/services/oss/backend.rs | 34 +-
core/src/services/oss/pager.rs | 1 +
core/src/services/oss/writer.rs | 19 +-
core/src/services/s3/backend.rs | 34 +-
core/src/services/s3/writer.rs | 16 +-
core/src/services/sftp/backend.rs | 3 +-
core/src/services/sftp/docs.md | 5 +-
core/src/services/sftp/writer.rs | 37 +-
core/src/services/supabase/backend.rs | 11 +-
core/src/services/supabase/writer.rs | 34 +-
core/src/services/tikv/fixtures/gen_cert.sh | 85 +++
.../tikv/fixtures/{pd.toml => pd-tls.toml} | 9 +
core/src/services/tikv/fixtures/ssl/ca-key.pem | 27 +
core/src/services/tikv/fixtures/ssl/ca.pem | 22 +
core/src/services/tikv/fixtures/ssl/client-key.pem | 27 +
core/src/services/tikv/fixtures/ssl/client.pem | 21 +
.../services/tikv/fixtures/ssl/pd-server-key.pem | 27 +
core/src/services/tikv/fixtures/ssl/pd-server.pem | 21 +
.../services/tikv/fixtures/ssl/tikv-server-key.pem | 27 +
.../src/services/tikv/fixtures/ssl/tikv-server.pem | 22 +
.../tikv/fixtures/{tikv.toml => tikv-tls.toml} | 9 +
core/src/services/vercel_artifacts/backend.rs | 15 +-
core/src/services/vercel_artifacts/writer.rs | 28 +-
core/src/services/wasabi/backend.rs | 11 +-
core/src/services/wasabi/writer.rs | 21 +-
core/src/services/webdav/backend.rs | 14 +-
core/src/services/webdav/writer.rs | 32 +-
core/src/services/webhdfs/backend.rs | 11 +-
core/src/services/webhdfs/writer.rs | 22 +-
core/src/types/capability.rs | 83 +--
core/src/types/operator/blocking_operator.rs | 2 +-
core/src/types/operator/builder.rs | 4 +-
core/src/types/operator/operator.rs | 6 +-
core/src/types/operator/operator_functions.rs | 28 +-
core/src/types/operator/operator_futures.rs | 50 +-
core/src/types/scheme.rs | 12 +-
core/src/types/writer.rs | 249 ++-------
core/tests/behavior/main.rs | 4 +-
core/tests/behavior/utils.rs | 9 +-
core/tests/behavior/write.rs | 21 +-
.../index.md | 2 +-
website/docs/services/{azdfs.mdx => azdls.mdx} | 10 +-
website/docs/services/{dropbox.mdx => gdrive.mdx} | 13 +-
.../docs/services/{dropbox.mdx => onedrive.mdx} | 13 +-
.../HomepageFeatures/_feature_services.mdx | 2 +-
170 files changed, 4424 insertions(+), 3033 deletions(-)
rename .github/workflows/{service_test_azdfs.yml => service_test_azdls.yml} (75%)
create mode 100644 bindings/java/src/test/java/org/apache/opendal/RedisServiceTest.java
create mode 100644 core/src/raw/oio/buf/chunked_bytes.rs
copy core/src/{services/cos => raw/oio/buf}/mod.rs (88%)
rename core/src/raw/oio/{buf.rs => buf/write_buf.rs} (54%)
create mode 100644 core/src/raw/oio/write/range_write.rs
rename core/src/services/{azdfs => azdls}/backend.rs (82%)
rename core/src/services/{azdfs => azdls}/core.rs (94%)
rename core/src/services/{azdfs => azdls}/docs.md (75%)
rename core/src/services/{azdfs => azdls}/error.rs (92%)
rename core/src/services/{azdfs => azdls}/mod.rs (95%)
rename core/src/services/{azdfs => azdls}/pager.rs (92%)
rename core/src/services/{azdfs => azdls}/writer.rs (70%)
create mode 100644 core/src/services/gdrive/docs.md
create mode 100644 core/src/services/gdrive/pager.rs
copy core/src/services/{ftp => onedrive}/docs.md (55%)
create mode 100755 core/src/services/tikv/fixtures/gen_cert.sh
copy core/src/services/tikv/fixtures/{pd.toml => pd-tls.toml} (64%)
create mode 100644 core/src/services/tikv/fixtures/ssl/ca-key.pem
create mode 100644 core/src/services/tikv/fixtures/ssl/ca.pem
create mode 100644 core/src/services/tikv/fixtures/ssl/client-key.pem
create mode 100644 core/src/services/tikv/fixtures/ssl/client.pem
create mode 100644 core/src/services/tikv/fixtures/ssl/pd-server-key.pem
create mode 100644 core/src/services/tikv/fixtures/ssl/pd-server.pem
create mode 100644 core/src/services/tikv/fixtures/ssl/tikv-server-key.pem
create mode 100644 core/src/services/tikv/fixtures/ssl/tikv-server.pem
copy core/src/services/tikv/fixtures/{tikv.toml => tikv-tls.toml} (70%)
rename website/docs/services/{azdfs.mdx => azdls.mdx} (87%)
copy website/docs/services/{dropbox.mdx => gdrive.mdx} (77%)
copy website/docs/services/{dropbox.mdx => onedrive.mdx} (77%)
[incubator-opendal] 01/01: test for blocking append & fix fs
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyanhanx pushed a commit to branch empty-file-test
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 2d59e9ae44ae0947344126e0c766dbb96c22b894
Author: suyanhanx <su...@gmail.com>
AuthorDate: Mon Sep 18 12:32:52 2023 +0800
test for blocking append & fix fs
Signed-off-by: suyanhanx <su...@gmail.com>
---
core/src/services/fs/backend.rs | 42 +++++--
core/src/services/fs/writer.rs | 1 +
core/tests/behavior/blocking_append.rs | 220 +++++++++++++++++++++++++++++++++
core/tests/behavior/main.rs | 3 +
4 files changed, 258 insertions(+), 8 deletions(-)
diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs
index b1f1fdf49..c28342eba 100644
--- a/core/src/services/fs/backend.rs
+++ b/core/src/services/fs/backend.rs
@@ -55,6 +55,11 @@ impl FsBuilder {
}
/// Set temp dir for atomic write.
+ ///
+ /// # Notes
+ ///
+ /// - When append is enabled, we will not use atomic write
+ /// to avoid data loss and performance issue.
pub fn atomic_write_dir(&mut self, dir: &str) -> &mut Self {
self.atomic_write_dir = if dir.is_empty() {
None
@@ -363,7 +368,7 @@ impl Accessor for FsBackend {
}
async fn write(&self, path: &str, op: OpWrite) -> Result<(RpWrite, Self::Writer)> {
- let (target_path, tmp_path) = if let Some(atomic_write_dir) = &self.atomic_write_dir {
+ let (target_path, mut tmp_path) = if let Some(atomic_write_dir) = &self.atomic_write_dir {
let target_path = Self::ensure_write_abs_path(&self.root, path).await?;
let tmp_path =
Self::ensure_write_abs_path(atomic_write_dir, &tmp_file_of(path)).await?;
@@ -375,10 +380,17 @@ impl Accessor for FsBackend {
};
let mut open_options = tokio::fs::OpenOptions::new();
-
open_options.create(true).write(true);
if op.append() {
open_options.append(true);
+
+ // If the target file exists, we should append to the end of it directly.
+ if tokio::fs::try_exists(&target_path)
+ .await
+ .map_err(parse_io_error)?
+ {
+ tmp_path = None;
+ }
} else {
open_options.truncate(true);
}
@@ -554,8 +566,8 @@ impl Accessor for FsBackend {
Ok((RpRead::new(end - start), r))
}
- fn blocking_write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let (target_path, tmp_path) = if let Some(atomic_write_dir) = &self.atomic_write_dir {
+ fn blocking_write(&self, path: &str, op: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
+ let (target_path, mut tmp_path) = if let Some(atomic_write_dir) = &self.atomic_write_dir {
let target_path = Self::blocking_ensure_write_abs_path(&self.root, path)?;
let tmp_path =
Self::blocking_ensure_write_abs_path(atomic_write_dir, &tmp_file_of(path))?;
@@ -566,10 +578,24 @@ impl Accessor for FsBackend {
(p, None)
};
- let f = std::fs::OpenOptions::new()
- .create(true)
- .truncate(true)
- .write(true)
+ let mut f = std::fs::OpenOptions::new();
+ f.create(true).write(true);
+
+ if op.append() {
+ f.append(true);
+
+ // If the target file exists, we should append to the end of it directly.
+ if Path::new(&target_path)
+ .try_exists()
+ .map_err(parse_io_error)?
+ {
+ tmp_path = None;
+ }
+ } else {
+ f.truncate(true);
+ }
+
+ let f = f
.open(tmp_path.as_ref().unwrap_or(&target_path))
.map_err(parse_io_error)?;
diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs
index 5b1ae953e..d1283d4ca 100644
--- a/core/src/services/fs/writer.rs
+++ b/core/src/services/fs/writer.rs
@@ -45,6 +45,7 @@ impl<F> FsWriter<F> {
Self {
target_path,
tmp_path,
+
f: Some(f),
fut: None,
}
diff --git a/core/tests/behavior/blocking_append.rs b/core/tests/behavior/blocking_append.rs
new file mode 100644
index 000000000..ccb89f786
--- /dev/null
+++ b/core/tests/behavior/blocking_append.rs
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::vec;
+
+use anyhow::Result;
+use sha2::Digest;
+use sha2::Sha256;
+use std::io::BufReader;
+use std::io::Cursor;
+
+use crate::*;
+
+pub fn behavior_blocking_append_tests(op: &Operator) -> Vec<Trial> {
+ let cap = op.info().full_capability();
+
+ if !(cap.read && cap.write && cap.blocking && cap.write_can_append) {
+ return vec![];
+ }
+
+ blocking_trials!(
+ op,
+ test_blocking_append_create_append,
+ test_blocking_append_with_dir_path,
+ test_blocking_append_with_cache_control,
+ test_blocking_append_with_content_type,
+ test_blocking_append_with_content_disposition,
+ test_blocking_appender_std_copy,
+ test_blocking_fuzz_appender
+ )
+}
+
+/// Test append to a file must success.
+pub fn test_blocking_append_create_append(op: BlockingOperator) -> Result<()> {
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content_one, size_one) = gen_bytes();
+ let (content_two, size_two) = gen_bytes();
+
+ op.write_with(&path, content_one.clone())
+ .append(true)
+ .call()
+ .expect("append file first time must success");
+
+ op.write_with(&path, content_two.clone())
+ .append(true)
+ .call()
+ .expect("append to an existing file must success");
+
+ let bs = op.read(&path).expect("read file must success");
+
+ assert_eq!(bs.len(), size_one + size_two);
+ assert_eq!(bs[..size_one], content_one);
+ assert_eq!(bs[size_one..], content_two);
+
+ op.delete(&path).expect("delete file must success");
+
+ Ok(())
+}
+
+/// Test append to a directory path must fail.
+pub fn test_blocking_append_with_dir_path(op: BlockingOperator) -> Result<()> {
+ let path = format!("{}/", uuid::Uuid::new_v4());
+ let (content, _) = gen_bytes();
+
+ let res = op.write_with(&path, content).append(true).call();
+ assert!(res.is_err());
+ assert_eq!(res.unwrap_err().kind(), ErrorKind::IsADirectory);
+
+ Ok(())
+}
+
+/// Test append with cache control must success.
+pub fn test_blocking_append_with_cache_control(op: BlockingOperator) -> Result<()> {
+ if !op.info().full_capability().write_with_cache_control {
+ return Ok(());
+ }
+
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content, _) = gen_bytes();
+
+ let target_cache_control = "no-cache, no-store, max-age=300";
+ op.write_with(&path, content)
+ .append(true)
+ .cache_control(target_cache_control)
+ .call()?;
+
+ let meta = op.stat(&path).expect("stat must succeed");
+ assert_eq!(meta.mode(), EntryMode::FILE);
+ assert_eq!(
+ meta.cache_control().expect("cache control must exist"),
+ target_cache_control
+ );
+
+ op.delete(&path).expect("delete must succeed");
+
+ Ok(())
+}
+
+/// Test append with content type must success.
+pub fn test_blocking_append_with_content_type(op: BlockingOperator) -> Result<()> {
+ if !op.info().full_capability().write_with_content_type {
+ return Ok(());
+ }
+
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content, size) = gen_bytes();
+
+ let target_content_type = "application/json";
+ op.write_with(&path, content)
+ .append(true)
+ .content_type(target_content_type)
+ .call()?;
+
+ let meta = op.stat(&path).expect("stat must succeed");
+ assert_eq!(meta.mode(), EntryMode::FILE);
+ assert_eq!(
+ meta.content_type().expect("content type must exist"),
+ target_content_type
+ );
+ assert_eq!(meta.content_length(), size as u64);
+
+ op.delete(&path).expect("delete must succeed");
+
+ Ok(())
+}
+
+/// Write a single file with content disposition should succeed.
+pub fn test_blocking_append_with_content_disposition(op: BlockingOperator) -> Result<()> {
+ if !op.info().full_capability().write_with_content_disposition {
+ return Ok(());
+ }
+
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content, size) = gen_bytes();
+
+ let target_content_disposition = "attachment; filename=\"filename.jpg\"";
+ op.write_with(&path, content)
+ .append(true)
+ .content_disposition(target_content_disposition)
+ .call()?;
+
+ let meta = op.stat(&path).expect("stat must succeed");
+ assert_eq!(meta.mode(), EntryMode::FILE);
+ assert_eq!(
+ meta.content_disposition().expect("content type must exist"),
+ target_content_disposition
+ );
+ assert_eq!(meta.content_length(), size as u64);
+
+ op.delete(&path).expect("delete must succeed");
+
+ Ok(())
+}
+
+/// Copy data from reader to writer
+pub fn test_blocking_appender_std_copy(op: BlockingOperator) -> Result<()> {
+ let path = uuid::Uuid::new_v4().to_string();
+ let (content, size): (Vec<u8>, usize) =
+ gen_bytes_with_range(10 * 1024 * 1024..20 * 1024 * 1024);
+
+ let mut a = op.writer_with(&path).append(true).call()?;
+
+ // Wrap a buf reader here to make sure content is read in 1MiB chunks.
+ let mut cursor = BufReader::with_capacity(1024 * 1024, Cursor::new(content.clone()));
+ std::io::copy(&mut cursor, &mut a)?;
+ a.close()?;
+
+ let meta = op.stat(&path).expect("stat must succeed");
+ assert_eq!(meta.content_length(), size as u64);
+
+ let bs = op.read(&path)?;
+ assert_eq!(bs.len(), size, "read size");
+ assert_eq!(
+ format!("{:x}", Sha256::digest(&bs[..size])),
+ format!("{:x}", Sha256::digest(content)),
+ "read content"
+ );
+
+ op.delete(&path).expect("delete must succeed");
+ Ok(())
+}
+
+/// Test for fuzzing appender.
+pub fn test_blocking_fuzz_appender(op: BlockingOperator) -> Result<()> {
+ let path = uuid::Uuid::new_v4().to_string();
+
+ let mut fuzzer = ObjectWriterFuzzer::new(&path, None);
+
+ let mut a = op.writer_with(&path).append(true).call()?;
+
+ for _ in 0..100 {
+ match fuzzer.fuzz() {
+ ObjectWriterAction::Write(bs) => {
+ a.write(bs)?;
+ }
+ }
+ }
+ a.close()?;
+
+ let content = op.read(&path)?;
+ fuzzer.check(&content);
+
+ op.delete(&path).expect("delete file must success");
+
+ Ok(())
+}
diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs
index bc7678407..c5c3084a7 100644
--- a/core/tests/behavior/main.rs
+++ b/core/tests/behavior/main.rs
@@ -42,12 +42,14 @@ use rename::behavior_rename_tests;
use write::behavior_write_tests;
// Blocking test cases
+mod blocking_append;
mod blocking_copy;
mod blocking_list;
mod blocking_read_only;
mod blocking_rename;
mod blocking_write;
+use blocking_append::behavior_blocking_append_tests;
use blocking_copy::behavior_blocking_copy_tests;
use blocking_list::behavior_blocking_list_tests;
use blocking_read_only::behavior_blocking_read_only_tests;
@@ -73,6 +75,7 @@ fn behavior_test<B: Builder>() -> Vec<Trial> {
let mut trials = vec![];
// Blocking tests
+ trials.extend(behavior_blocking_append_tests(&operator));
trials.extend(behavior_blocking_copy_tests(&operator));
trials.extend(behavior_blocking_list_tests(&operator));
trials.extend(behavior_blocking_read_only_tests(&operator));