You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2017/11/22 06:44:29 UTC
[3/3] kudu git commit: handle disk failures during tablet copies
handle disk failures during tablet copies
There are two components in a tablet copy: the copy client (that
receives data) and the copy session source (that sends data).
Coarse-grain handling of disk failures during tablet copies is done for
the tablet copy client as:
- Before starting a copy client, if no disks are available to place the
tablet, simply return (instead of failing a CHECK).
- Before downloading each WAL segments or block, check that the tablet
is in a healthy group.
And for the tablet copy session as:
- Before sending a block or log segment, check if the tablet has an
error.
Upon returning an error, the tablet copy client will shutdown the
replica, leaving it in a failed state.
A test is added to ensure that both copy clients and that source
sessions with failed disks will return errors to the copying client.
Change-Id: Ic18d93c218ea13f3086f420a4847cb5e29a47bc7
Reviewed-on: http://gerrit.cloudera.org:8080/7654
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy <mp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8a81f4ff
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8a81f4ff
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8a81f4ff
Branch: refs/heads/master
Commit: 8a81f4ff18ee30951a25bab247fa5a3c20058e49
Parents: ae0143c
Author: Andrew Wong <aw...@cloudera.com>
Authored: Wed Oct 25 14:49:57 2017 -0700
Committer: Andrew Wong <aw...@cloudera.com>
Committed: Wed Nov 22 06:39:38 2017 +0000
----------------------------------------------------------------------
src/kudu/tserver/tablet_copy-test-base.h | 5 ++-
src/kudu/tserver/tablet_copy_client-test.cc | 47 ++++++++++++++++++---
src/kudu/tserver/tablet_copy_client.cc | 14 +++++-
src/kudu/tserver/tablet_copy_client.h | 4 ++
src/kudu/tserver/tablet_copy_service-test.cc | 40 ++++++++++++++++++
src/kudu/tserver/tablet_copy_source_session.cc | 17 ++++++++
src/kudu/tserver/tablet_copy_source_session.h | 4 ++
7 files changed, 124 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy-test-base.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy-test-base.h b/src/kudu/tserver/tablet_copy-test-base.h
index 8da5f37..638435d 100644
--- a/src/kudu/tserver/tablet_copy-test-base.h
+++ b/src/kudu/tserver/tablet_copy-test-base.h
@@ -43,7 +43,10 @@ class TabletCopyTest : public TabletServerTestBase {
public:
virtual void SetUp() OVERRIDE {
NO_FATALS(TabletServerTestBase::SetUp());
- NO_FATALS(StartTabletServer(/* num_data_dirs */ 1));
+ // Create a tablet server with multiple data dirs. In most cases, this is
+ // unimportant, but in some cases can be helpful to test multi-disk
+ // behavior and disk failures.
+ NO_FATALS(StartTabletServer(/* num_data_dirs */ 3));
// Prevent logs from being deleted out from under us until / unless we want
// to test that we are anchoring correctly. Since GenerateTestData() does a
// Flush(), Log GC is allowed to eat the logs before we get around to
http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_client-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_client-test.cc b/src/kudu/tserver/tablet_copy_client-test.cc
index ab99a21..277d8ff 100644
--- a/src/kudu/tserver/tablet_copy_client-test.cc
+++ b/src/kudu/tserver/tablet_copy_client-test.cc
@@ -20,7 +20,9 @@
#include <limits>
#include <memory>
#include <ostream>
+#include <stdlib.h>
#include <string>
+#include <thread>
#include <vector>
#include <gflags/gflags_declare.h>
@@ -38,6 +40,7 @@
#include "kudu/consensus/raft_consensus.h"
#include "kudu/fs/block_id.h"
#include "kudu/fs/block_manager.h"
+#include "kudu/fs/data_dirs.h"
#include "kudu/fs/fs_manager.h"
#include "kudu/gutil/casts.h"
#include "kudu/gutil/gscoped_ptr.h"
@@ -64,6 +67,7 @@
using std::shared_ptr;
using std::string;
+using std::thread;
using std::vector;
DECLARE_string(block_manager);
@@ -77,6 +81,7 @@ using consensus::ConsensusMetadataManager;
using consensus::ConsensusStatePB;
using consensus::GetRaftConfigLeader;
using consensus::RaftPeerPB;
+using fs::DataDirManager;
using std::tuple;
using std::unique_ptr;
using tablet::TabletMetadata;
@@ -86,10 +91,14 @@ class TabletCopyClientTest : public TabletCopyTest {
virtual void SetUp() OVERRIDE {
NO_FATALS(TabletCopyTest::SetUp());
- const string kTestDir = GetTestPath("client_tablet");
+ // To be a bit more flexible in testing, create a FS layout with multiple disks.
+ const string kTestWalDir = GetTestPath("client_tablet_wal");
+ const string kTestDataDirPrefix = GetTestPath("client_tablet_data");
FsManagerOpts opts;
- opts.wal_root = kTestDir;
- opts.data_roots = { kTestDir };
+ opts.wal_root = kTestWalDir;
+ opts.data_roots.emplace_back(kTestDataDirPrefix + "A");
+ opts.data_roots.emplace_back(kTestDataDirPrefix + "B");
+
metric_entity_ = METRIC_ENTITY_server.Instantiate(&metric_registry_, "test");
opts.metric_entity = metric_entity_;
fs_manager_.reset(new FsManager(Env::Default(), opts));
@@ -251,10 +260,10 @@ TEST_F(TabletCopyClientTest, TestDownloadAllBlocks) {
// Verify the disk synchronization count.
if (FLAGS_block_manager == "log") {
- ASSERT_EQ(3, down_cast<Counter*>(
+ ASSERT_EQ(6, down_cast<Counter*>(
metric_entity_->FindOrNull(METRIC_block_manager_total_disk_sync).get())->value());
} else {
- ASSERT_EQ(14, down_cast<Counter*>(
+ ASSERT_EQ(18, down_cast<Counter*>(
metric_entity_->FindOrNull(METRIC_block_manager_total_disk_sync).get())->value());
}
@@ -271,6 +280,34 @@ TEST_F(TabletCopyClientTest, TestDownloadAllBlocks) {
}
}
+// Test that failing a disk outside fo the tablet copy client will eventually
+// stop the copy client and cause it to fail.
+TEST_F(TabletCopyClientTest, TestFailedDiskStopsClient) {
+ DataDirManager* dd_manager = fs_manager_->dd_manager();
+
+ // Repeatedly fetch files for the client.
+ Status s;
+ auto copy_thread = thread([&] {
+ while (s.ok()) {
+ s = client_->FetchAll(nullptr);
+ }
+ });
+
+ // In a separate thread, mark one of the directories as failed.
+ while (true) {
+ if (rand() % 10 == 0) {
+ dd_manager->MarkDataDirFailed(0, "injected failure in non-client thread");
+ LOG(INFO) << "INJECTING FAILURE";
+ break;
+ }
+ SleepFor(MonoDelta::FromMilliseconds(50));
+ }
+
+ // The copy thread should stop and the copy client should return an error.
+ copy_thread.join();
+ ASSERT_TRUE(s.IsIOError());
+}
+
enum DownloadBlocks {
kDownloadBlocks, // Fetch blocks from remote.
kNoDownloadBlocks, // Do not fetch blocks from remote.
http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_client.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_client.cc b/src/kudu/tserver/tablet_copy_client.cc
index 13fe64e..4e16e7d 100644
--- a/src/kudu/tserver/tablet_copy_client.cc
+++ b/src/kudu/tserver/tablet_copy_client.cc
@@ -325,7 +325,8 @@ Status TabletCopyClient::Start(const HostPort& copy_source_addr,
tablet::TABLET_DATA_COPYING,
/*last_logged_opid=*/ boost::none),
"Could not replace superblock with COPYING data state");
- CHECK_OK(fs_manager_->dd_manager()->CreateDataDirGroup(tablet_id_));
+ RETURN_NOT_OK_PREPEND(fs_manager_->dd_manager()->CreateDataDirGroup(tablet_id_),
+ "Could not create a new directory group for tablet copy");
} else {
// HACK: Set the initial tombstoned last-logged OpId to 1.0 when copying a
// replica for the first time, so that if the tablet copy fails, the
@@ -586,6 +587,8 @@ Status TabletCopyClient::DownloadBlocks() {
Status TabletCopyClient::DownloadWAL(uint64_t wal_segment_seqno) {
VLOG_WITH_PREFIX(1) << "Downloading WAL segment with seqno " << wal_segment_seqno;
+ RETURN_NOT_OK_PREPEND(CheckHealthyDirGroup(), "Not downloading WAL for replica");
+
DataIdPB data_id;
data_id.set_type(DataIdPB::LOG_SEGMENT);
data_id.set_wal_segment_seqno(wal_segment_seqno);
@@ -648,6 +651,7 @@ Status TabletCopyClient::DownloadAndRewriteBlock(const BlockIdPB& src_block_id,
Status TabletCopyClient::DownloadBlock(const BlockId& old_block_id,
BlockId* new_block_id) {
VLOG_WITH_PREFIX(1) << "Downloading block with block_id " << old_block_id.ToString();
+ RETURN_NOT_OK_PREPEND(CheckHealthyDirGroup(), "Not downloading block for replica");
unique_ptr<WritableBlock> block;
RETURN_NOT_OK_PREPEND(fs_manager_->CreateNewBlock(CreateBlockOptions({ tablet_id_ }), &block),
@@ -740,6 +744,14 @@ string TabletCopyClient::LogPrefix() {
tablet_id_, fs_manager_->uuid());
}
+Status TabletCopyClient::CheckHealthyDirGroup() const {
+ if (fs_manager_->dd_manager()->IsTabletInFailedDir(tablet_id_)) {
+ return Status::IOError(
+ Substitute("Tablet $0 is in a failed directory", tablet_id_));
+ }
+ return Status::OK();
+}
+
template<typename F>
Status TabletCopyClient::SendRpcWithRetry(rpc::RpcController* controller, F f) {
MonoTime deadline = MonoTime::Now() + MonoDelta::FromMilliseconds(session_idle_timeout_millis_);
http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_client.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_client.h b/src/kudu/tserver/tablet_copy_client.h
index f44e5c2..018c6db 100644
--- a/src/kudu/tserver/tablet_copy_client.h
+++ b/src/kudu/tserver/tablet_copy_client.h
@@ -145,6 +145,10 @@ class TabletCopyClient {
static Status UnwindRemoteError(const Status& status, const rpc::RpcController& controller);
+ // Returns an error if any directories in the tablet's directory group are
+ // unhealthy.
+ Status CheckHealthyDirGroup() const;
+
// Set a new status message on the TabletReplica.
// The string "TabletCopy: " will be prepended to each message.
void SetStatusMessage(const std::string& message);
http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_service-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_service-test.cc b/src/kudu/tserver/tablet_copy_service-test.cc
index 7dd409f..8707f9e 100644
--- a/src/kudu/tserver/tablet_copy_service-test.cc
+++ b/src/kudu/tserver/tablet_copy_service-test.cc
@@ -20,6 +20,7 @@
#include <cstdint>
#include <limits>
#include <memory>
+#include <ostream>
#include <string>
#include <thread>
#include <vector>
@@ -34,7 +35,9 @@
#include "kudu/consensus/log_reader.h"
#include "kudu/consensus/log_util.h"
#include "kudu/fs/block_id.h"
+#include "kudu/fs/data_dirs.h"
#include "kudu/fs/fs.pb.h"
+#include "kudu/fs/fs_manager.h"
#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
@@ -55,6 +58,8 @@
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
+DECLARE_bool(crash_on_eio);
+DECLARE_double(env_inject_eio);
DECLARE_uint64(tablet_copy_idle_timeout_sec);
DECLARE_uint64(tablet_copy_timeout_poll_period_ms);
@@ -509,5 +514,40 @@ TEST_F(TabletCopyServiceTest, TestSessionTimeout) {
ASSERT_FALSE(resp.session_is_active()) << "Tablet Copy session did not time out!";
}
+// Test that the tablet copy session will terminate on disk failures.
+TEST_F(TabletCopyServiceTest, TestDiskFailureDuringSession) {
+ string session_id;
+ tablet::TabletSuperBlockPB superblock;
+ ASSERT_OK(DoBeginValidTabletCopySession(&session_id, &superblock));
+
+ // Get a block id locally that we'll copy.
+ BlockId block_id = FirstColumnBlockId(superblock);
+ Slice local_data;
+ faststring scratch;
+ ASSERT_OK(ReadLocalBlockFile(mini_server_->server()->fs_manager(), block_id,
+ &scratch, &local_data));
+
+ // Copy over the block while one of the directories is failed.
+ FetchDataResponsePB resp;
+ RpcController controller;
+ ASSERT_OK(mini_server_->server()->fs_manager()->dd_manager()->MarkDataDirFailed(1));
+ Status s = DoFetchData(session_id, AsDataTypeId(block_id), nullptr, nullptr, &resp, &controller);
+ LOG(INFO) << "Fetch data request responded with: " << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "Unable to get piece of data block");
+ ASSERT_TRUE(s.IsRemoteError());
+
+ // Now close the copy session.
+ RpcController end_copy_controller;
+ EndTabletCopySessionResponsePB end_session_resp;
+ ASSERT_OK(DoEndTabletCopySession(
+ session_id, true, nullptr, &end_session_resp, &end_copy_controller));
+
+ // Starting a new session should fail.
+ s = DoBeginValidTabletCopySession(&session_id, &superblock);
+ LOG(INFO) << "Begin copy session request responded with: " << s.ToString();
+ ASSERT_STR_CONTAINS(s.ToString(), "Error beginning tablet copy session");
+ ASSERT_TRUE(s.IsRemoteError());
+}
+
} // namespace tserver
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_source_session.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_source_session.cc b/src/kudu/tserver/tablet_copy_source_session.cc
index 229fe3b..e2a5227 100644
--- a/src/kudu/tserver/tablet_copy_source_session.cc
+++ b/src/kudu/tserver/tablet_copy_source_session.cc
@@ -34,6 +34,7 @@
#include "kudu/consensus/opid_util.h"
#include "kudu/consensus/raft_consensus.h"
#include "kudu/fs/block_manager.h"
+#include "kudu/fs/data_dirs.h"
#include "kudu/fs/fs.pb.h"
#include "kudu/fs/fs_manager.h"
#include "kudu/gutil/basictypes.h"
@@ -129,6 +130,7 @@ Status TabletCopySourceSession::InitOnce() {
}
RETURN_NOT_OK(tablet_replica_->CheckRunning());
+ RETURN_NOT_OK(CheckHealthyDirGroup());
const string& tablet_id = tablet_replica_->tablet_id();
@@ -300,6 +302,8 @@ Status TabletCopySourceSession::GetBlockPiece(const BlockId& block_id,
string* data, int64_t* block_file_size,
TabletCopyErrorPB::Code* error_code) {
DCHECK(init_once_.init_succeeded());
+ RETURN_NOT_OK_PREPEND(CheckHealthyDirGroup(error_code),
+ "Tablet copy source could not get block");
ImmutableReadableBlockInfo* block_info;
RETURN_NOT_OK(FindBlock(block_id, &block_info, error_code));
@@ -320,6 +324,8 @@ Status TabletCopySourceSession::GetLogSegmentPiece(uint64_t segment_seqno,
std::string* data, int64_t* log_file_size,
TabletCopyErrorPB::Code* error_code) {
DCHECK(init_once_.init_succeeded());
+ RETURN_NOT_OK_PREPEND(CheckHealthyDirGroup(error_code),
+ "Tablet copy source could not get log segment");
ImmutableRandomAccessFileInfo* file_info;
RETURN_NOT_OK(FindLogSegment(segment_seqno, &file_info, error_code));
RETURN_NOT_OK(ReadFileChunkToBuf(file_info, offset, client_maxlen,
@@ -356,6 +362,17 @@ static Status AddImmutableFileToMap(Collection* const cache,
return Status::OK();
}
+Status TabletCopySourceSession::CheckHealthyDirGroup(TabletCopyErrorPB::Code* error_code) const {
+ if (fs_manager_->dd_manager()->IsTabletInFailedDir(tablet_id())) {
+ if (error_code) {
+ *error_code = TabletCopyErrorPB::IO_ERROR;
+ }
+ return Status::IOError(
+ Substitute("Tablet $0 is in a failed directory", tablet_id()));
+ }
+ return Status::OK();
+}
+
Status TabletCopySourceSession::OpenBlock(const BlockId& block_id) {
unique_ptr<ReadableBlock> block;
Status s = fs_manager_->OpenBlock(block_id, &block);
http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_source_session.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_source_session.h b/src/kudu/tserver/tablet_copy_source_session.h
index 4f6c350..3c433ed 100644
--- a/src/kudu/tserver/tablet_copy_source_session.h
+++ b/src/kudu/tserver/tablet_copy_source_session.h
@@ -174,6 +174,10 @@ class TabletCopySourceSession : public RefCountedThreadSafe<TabletCopySourceSess
// Internal helper method for Init().
Status InitOnce();
+ // Returns an error if any directories in the tablet's directory group are
+ // unhealthy and sets 'error_code' appropriately.
+ Status CheckHealthyDirGroup(TabletCopyErrorPB::Code* error_code = nullptr) const;
+
// Open the block and add it to the block map.
Status OpenBlock(const BlockId& block_id);