You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2020/11/17 04:58:37 UTC
[kudu] branch master updated: [tablet_copy_client-test] fix
scenario on DownloadBlocks() error reporting
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 0f133b9 [tablet_copy_client-test] fix scenario on DownloadBlocks() error reporting
0f133b9 is described below
commit 0f133b9aed0dcf5622361fdfdd67f1bdff26809d
Author: ningw <19...@gmail.com>
AuthorDate: Wed Nov 11 13:56:07 2020 +0800
[tablet_copy_client-test] fix scenario on DownloadBlocks() error reporting
In patch [1] the status reporting of the multi-threaded DownloadBlocks()
was not tested properly. The original test scenario called only
StartCopy(), but DownloadBlocks() was not called.
This patch moves the corresponding test code into a separate scenario,
targeting testing failure paths in any of the download threads spawned
from TabletCopyClient::DownloadBlocks(). With this patch,
both StartCopy() and DownloadBlocks() are called in the scenario.
[1] https://gerrit.cloudera.org/#/c/16274/8/src/kudu/tserver/tablet_copy_client-test.cc
Change-Id: I3bfbe12bf13d0cc274ad266f62c5539f7da791b8
Reviewed-on: http://gerrit.cloudera.org:8080/16709
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
src/kudu/tserver/tablet_copy_client-test.cc | 26 +++++++++++++-------------
src/kudu/tserver/tablet_copy_client.cc | 9 +++++++++
src/kudu/tserver/tablet_copy_client.h | 1 +
3 files changed, 23 insertions(+), 13 deletions(-)
diff --git a/src/kudu/tserver/tablet_copy_client-test.cc b/src/kudu/tserver/tablet_copy_client-test.cc
index 10b3470..efd2940 100644
--- a/src/kudu/tserver/tablet_copy_client-test.cc
+++ b/src/kudu/tserver/tablet_copy_client-test.cc
@@ -75,6 +75,7 @@ using std::thread;
using std::vector;
DECLARE_double(env_inject_eio);
+DECLARE_double(tablet_copy_fault_crash_during_download_block);
DECLARE_int32(tablet_copy_download_threads_nums_per_session);
DECLARE_string(block_manager);
DECLARE_string(env_inject_eio_globs);
@@ -213,19 +214,6 @@ TEST_F(TabletCopyClientTest, TestLifeCycle) {
ASSERT_FALSE(meta_);
}
- // Rowset are download by multithreads, the error status can be corretly
- // collected among all threads.
- {
- google::FlagSaver fs;
- FLAGS_env_inject_eio = 0.5;
- FLAGS_tablet_copy_download_threads_nums_per_session = 16;
- s = StartCopy();
- ASSERT_TRUE(s.IsIOError()) << s.ToString();
- ASSERT_STR_CONTAINS(s.ToString(), "Failed to write tablet metadata");
- ASSERT_EQ(TabletCopyClient::State::kInitialized, client_->state_);
- ASSERT_FALSE(meta_);
- }
-
// Now let's try replacing a tablet. Set a metadata that we can replace.
ASSERT_OK(ResetTabletCopyClient());
ASSERT_OK(StartCopy());
@@ -303,6 +291,18 @@ TEST_F(TabletCopyClientTest, TestDownloadBlock) {
ASSERT_OK(ReadLocalBlockFile(fs_manager_.get(), new_block_id, &scratch, &slice));
}
+// Test that error status is properly reported if there was a failure in any
+// of multiple threads downloading tablet's data blocks.
+TEST_F(TabletCopyClientTest, TestDownloadBlockMayFail) {
+ FLAGS_tablet_copy_fault_crash_during_download_block = 0.5;
+ FLAGS_tablet_copy_download_threads_nums_per_session = 16;
+
+ ASSERT_OK(StartCopy());
+ Status s = client_->DownloadBlocks();
+ ASSERT_TRUE(s.IsIOError());
+ ASSERT_STR_CONTAINS(s.ToString(), "Injected failure on downloading block");
+}
+
// Basic WAL segment download unit test.
TEST_F(TabletCopyClientTest, TestDownloadWalSegment) {
ASSERT_OK(StartCopy());
diff --git a/src/kudu/tserver/tablet_copy_client.cc b/src/kudu/tserver/tablet_copy_client.cc
index 6a16794..64c3a0f 100644
--- a/src/kudu/tserver/tablet_copy_client.cc
+++ b/src/kudu/tserver/tablet_copy_client.cc
@@ -25,6 +25,7 @@
#include <utility>
#include <boost/optional/optional.hpp>
+#include <boost/type_traits/decay.hpp>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <google/protobuf/stubs/port.h>
@@ -103,6 +104,12 @@ DEFINE_double(tablet_copy_fault_crash_before_write_cmeta, 0.0,
TAG_FLAG(tablet_copy_fault_crash_before_write_cmeta, unsafe);
TAG_FLAG(tablet_copy_fault_crash_before_write_cmeta, runtime);
+DEFINE_double(tablet_copy_fault_crash_during_download_block, 0.0,
+ "Fraction of the time that DownloadBlock() will fail. "
+ "For use in test only.");
+TAG_FLAG(tablet_copy_fault_crash_during_download_block, unsafe);
+TAG_FLAG(tablet_copy_fault_crash_during_download_block, runtime);
+
DEFINE_int32(tablet_copy_download_threads_nums_per_session, 4,
"Number of threads per tablet copy session for downloading tablet data blocks.");
DEFINE_validator(tablet_copy_download_threads_nums_per_session,
@@ -777,6 +784,8 @@ Status TabletCopyClient::DownloadAndRewriteBlockIfEndStatusOK(const BlockIdPB& s
Status TabletCopyClient::DownloadBlock(const BlockId& old_block_id,
BlockId* new_block_id) {
VLOG_WITH_PREFIX(1) << "Downloading block with block_id " << old_block_id.ToString();
+ MAYBE_RETURN_FAILURE(FLAGS_tablet_copy_fault_crash_during_download_block,
+ Status::IOError("Injected failure on downloading block"));
RETURN_NOT_OK_PREPEND(CheckHealthyDirGroup(), "Not downloading block for replica");
unique_ptr<WritableBlock> block;
diff --git a/src/kudu/tserver/tablet_copy_client.h b/src/kudu/tserver/tablet_copy_client.h
index 52d91ba..3d5395e 100644
--- a/src/kudu/tserver/tablet_copy_client.h
+++ b/src/kudu/tserver/tablet_copy_client.h
@@ -139,6 +139,7 @@ class TabletCopyClient {
FRIEND_TEST(TabletCopyClientTest, TestDownloadBlock);
FRIEND_TEST(TabletCopyClientTest, TestLifeCycle);
FRIEND_TEST(TabletCopyClientTest, TestVerifyData);
+ FRIEND_TEST(TabletCopyClientTest, TestDownloadBlockMayFail);
FRIEND_TEST(TabletCopyClientTest, TestDownloadWalSegment);
FRIEND_TEST(TabletCopyClientTest, TestDownloadAllBlocks);
FRIEND_TEST(TabletCopyClientAbortTest, TestAbort);