You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2017/11/22 06:44:29 UTC

[3/3] kudu git commit: handle disk failures during tablet copies

handle disk failures during tablet copies

There are two components in a tablet copy: the copy client (that
receives data) and the copy session source (that sends data).

Coarse-grain handling of disk failures during tablet copies is done for
the tablet copy client as:
- Before starting a copy client, if no disks are available to place the
  tablet, simply return (instead of failing a CHECK).
- Before downloading each WAL segments or block, check that the tablet
  is in a healthy group.
And for the tablet copy session as:
- Before sending a block or log segment, check if the tablet has an
  error.

Upon returning an error, the tablet copy client will shutdown the
replica, leaving it in a failed state.

A test is added to ensure that both copy clients and that source
sessions with failed disks will return errors to the copying client.

Change-Id: Ic18d93c218ea13f3086f420a4847cb5e29a47bc7
Reviewed-on: http://gerrit.cloudera.org:8080/7654
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy <mp...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8a81f4ff
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8a81f4ff
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8a81f4ff

Branch: refs/heads/master
Commit: 8a81f4ff18ee30951a25bab247fa5a3c20058e49
Parents: ae0143c
Author: Andrew Wong <aw...@cloudera.com>
Authored: Wed Oct 25 14:49:57 2017 -0700
Committer: Andrew Wong <aw...@cloudera.com>
Committed: Wed Nov 22 06:39:38 2017 +0000

----------------------------------------------------------------------
 src/kudu/tserver/tablet_copy-test-base.h       |  5 ++-
 src/kudu/tserver/tablet_copy_client-test.cc    | 47 ++++++++++++++++++---
 src/kudu/tserver/tablet_copy_client.cc         | 14 +++++-
 src/kudu/tserver/tablet_copy_client.h          |  4 ++
 src/kudu/tserver/tablet_copy_service-test.cc   | 40 ++++++++++++++++++
 src/kudu/tserver/tablet_copy_source_session.cc | 17 ++++++++
 src/kudu/tserver/tablet_copy_source_session.h  |  4 ++
 7 files changed, 124 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy-test-base.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy-test-base.h b/src/kudu/tserver/tablet_copy-test-base.h
index 8da5f37..638435d 100644
--- a/src/kudu/tserver/tablet_copy-test-base.h
+++ b/src/kudu/tserver/tablet_copy-test-base.h
@@ -43,7 +43,10 @@ class TabletCopyTest : public TabletServerTestBase {
  public:
   virtual void SetUp() OVERRIDE {
     NO_FATALS(TabletServerTestBase::SetUp());
-    NO_FATALS(StartTabletServer(/* num_data_dirs */ 1));
+    // Create a tablet server with multiple data dirs. In most cases, this is
+    // unimportant, but in some cases can be helpful to test multi-disk
+    // behavior and disk failures.
+    NO_FATALS(StartTabletServer(/* num_data_dirs */ 3));
     // Prevent logs from being deleted out from under us until / unless we want
     // to test that we are anchoring correctly. Since GenerateTestData() does a
     // Flush(), Log GC is allowed to eat the logs before we get around to

http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_client-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_client-test.cc b/src/kudu/tserver/tablet_copy_client-test.cc
index ab99a21..277d8ff 100644
--- a/src/kudu/tserver/tablet_copy_client-test.cc
+++ b/src/kudu/tserver/tablet_copy_client-test.cc
@@ -20,7 +20,9 @@
 #include <limits>
 #include <memory>
 #include <ostream>
+#include <stdlib.h>
 #include <string>
+#include <thread>
 #include <vector>
 
 #include <gflags/gflags_declare.h>
@@ -38,6 +40,7 @@
 #include "kudu/consensus/raft_consensus.h"
 #include "kudu/fs/block_id.h"
 #include "kudu/fs/block_manager.h"
+#include "kudu/fs/data_dirs.h"
 #include "kudu/fs/fs_manager.h"
 #include "kudu/gutil/casts.h"
 #include "kudu/gutil/gscoped_ptr.h"
@@ -64,6 +67,7 @@
 
 using std::shared_ptr;
 using std::string;
+using std::thread;
 using std::vector;
 
 DECLARE_string(block_manager);
@@ -77,6 +81,7 @@ using consensus::ConsensusMetadataManager;
 using consensus::ConsensusStatePB;
 using consensus::GetRaftConfigLeader;
 using consensus::RaftPeerPB;
+using fs::DataDirManager;
 using std::tuple;
 using std::unique_ptr;
 using tablet::TabletMetadata;
@@ -86,10 +91,14 @@ class TabletCopyClientTest : public TabletCopyTest {
   virtual void SetUp() OVERRIDE {
     NO_FATALS(TabletCopyTest::SetUp());
 
-    const string kTestDir = GetTestPath("client_tablet");
+    // To be a bit more flexible in testing, create a FS layout with multiple disks.
+    const string kTestWalDir = GetTestPath("client_tablet_wal");
+    const string kTestDataDirPrefix = GetTestPath("client_tablet_data");
     FsManagerOpts opts;
-    opts.wal_root = kTestDir;
-    opts.data_roots = { kTestDir };
+    opts.wal_root = kTestWalDir;
+    opts.data_roots.emplace_back(kTestDataDirPrefix + "A");
+    opts.data_roots.emplace_back(kTestDataDirPrefix + "B");
+
     metric_entity_ = METRIC_ENTITY_server.Instantiate(&metric_registry_, "test");
     opts.metric_entity = metric_entity_;
     fs_manager_.reset(new FsManager(Env::Default(), opts));
@@ -251,10 +260,10 @@ TEST_F(TabletCopyClientTest, TestDownloadAllBlocks) {
 
   // Verify the disk synchronization count.
   if (FLAGS_block_manager == "log") {
-    ASSERT_EQ(3, down_cast<Counter*>(
+    ASSERT_EQ(6, down_cast<Counter*>(
         metric_entity_->FindOrNull(METRIC_block_manager_total_disk_sync).get())->value());
   } else {
-    ASSERT_EQ(14, down_cast<Counter*>(
+    ASSERT_EQ(18, down_cast<Counter*>(
         metric_entity_->FindOrNull(METRIC_block_manager_total_disk_sync).get())->value());
   }
 
@@ -271,6 +280,34 @@ TEST_F(TabletCopyClientTest, TestDownloadAllBlocks) {
   }
 }
 
+// Test that failing a disk outside fo the tablet copy client will eventually
+// stop the copy client and cause it to fail.
+TEST_F(TabletCopyClientTest, TestFailedDiskStopsClient) {
+  DataDirManager* dd_manager = fs_manager_->dd_manager();
+
+  // Repeatedly fetch files for the client.
+  Status s;
+  auto copy_thread = thread([&] {
+    while (s.ok()) {
+      s = client_->FetchAll(nullptr);
+    }
+  });
+
+  // In a separate thread, mark one of the directories as failed.
+  while (true) {
+    if (rand() % 10 == 0) {
+      dd_manager->MarkDataDirFailed(0, "injected failure in non-client thread");
+      LOG(INFO) << "INJECTING FAILURE";
+      break;
+    }
+    SleepFor(MonoDelta::FromMilliseconds(50));
+  }
+
+  // The copy thread should stop and the copy client should return an error.
+  copy_thread.join();
+  ASSERT_TRUE(s.IsIOError());
+}
+
 enum DownloadBlocks {
   kDownloadBlocks,    // Fetch blocks from remote.
   kNoDownloadBlocks,  // Do not fetch blocks from remote.

http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_client.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_client.cc b/src/kudu/tserver/tablet_copy_client.cc
index 13fe64e..4e16e7d 100644
--- a/src/kudu/tserver/tablet_copy_client.cc
+++ b/src/kudu/tserver/tablet_copy_client.cc
@@ -325,7 +325,8 @@ Status TabletCopyClient::Start(const HostPort& copy_source_addr,
                                           tablet::TABLET_DATA_COPYING,
                                           /*last_logged_opid=*/ boost::none),
         "Could not replace superblock with COPYING data state");
-    CHECK_OK(fs_manager_->dd_manager()->CreateDataDirGroup(tablet_id_));
+    RETURN_NOT_OK_PREPEND(fs_manager_->dd_manager()->CreateDataDirGroup(tablet_id_),
+        "Could not create a new directory group for tablet copy");
   } else {
     // HACK: Set the initial tombstoned last-logged OpId to 1.0 when copying a
     // replica for the first time, so that if the tablet copy fails, the
@@ -586,6 +587,8 @@ Status TabletCopyClient::DownloadBlocks() {
 
 Status TabletCopyClient::DownloadWAL(uint64_t wal_segment_seqno) {
   VLOG_WITH_PREFIX(1) << "Downloading WAL segment with seqno " << wal_segment_seqno;
+  RETURN_NOT_OK_PREPEND(CheckHealthyDirGroup(), "Not downloading WAL for replica");
+
   DataIdPB data_id;
   data_id.set_type(DataIdPB::LOG_SEGMENT);
   data_id.set_wal_segment_seqno(wal_segment_seqno);
@@ -648,6 +651,7 @@ Status TabletCopyClient::DownloadAndRewriteBlock(const BlockIdPB& src_block_id,
 Status TabletCopyClient::DownloadBlock(const BlockId& old_block_id,
                                        BlockId* new_block_id) {
   VLOG_WITH_PREFIX(1) << "Downloading block with block_id " << old_block_id.ToString();
+  RETURN_NOT_OK_PREPEND(CheckHealthyDirGroup(), "Not downloading block for replica");
 
   unique_ptr<WritableBlock> block;
   RETURN_NOT_OK_PREPEND(fs_manager_->CreateNewBlock(CreateBlockOptions({ tablet_id_ }), &block),
@@ -740,6 +744,14 @@ string TabletCopyClient::LogPrefix() {
                     tablet_id_, fs_manager_->uuid());
 }
 
+Status TabletCopyClient::CheckHealthyDirGroup() const {
+  if (fs_manager_->dd_manager()->IsTabletInFailedDir(tablet_id_)) {
+    return Status::IOError(
+        Substitute("Tablet $0 is in a failed directory", tablet_id_));
+  }
+  return Status::OK();
+}
+
 template<typename F>
 Status TabletCopyClient::SendRpcWithRetry(rpc::RpcController* controller, F f) {
   MonoTime deadline = MonoTime::Now() + MonoDelta::FromMilliseconds(session_idle_timeout_millis_);

http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_client.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_client.h b/src/kudu/tserver/tablet_copy_client.h
index f44e5c2..018c6db 100644
--- a/src/kudu/tserver/tablet_copy_client.h
+++ b/src/kudu/tserver/tablet_copy_client.h
@@ -145,6 +145,10 @@ class TabletCopyClient {
 
   static Status UnwindRemoteError(const Status& status, const rpc::RpcController& controller);
 
+  // Returns an error if any directories in the tablet's directory group are
+  // unhealthy.
+  Status CheckHealthyDirGroup() const;
+
   // Set a new status message on the TabletReplica.
   // The string "TabletCopy: " will be prepended to each message.
   void SetStatusMessage(const std::string& message);

http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_service-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_service-test.cc b/src/kudu/tserver/tablet_copy_service-test.cc
index 7dd409f..8707f9e 100644
--- a/src/kudu/tserver/tablet_copy_service-test.cc
+++ b/src/kudu/tserver/tablet_copy_service-test.cc
@@ -20,6 +20,7 @@
 #include <cstdint>
 #include <limits>
 #include <memory>
+#include <ostream>
 #include <string>
 #include <thread>
 #include <vector>
@@ -34,7 +35,9 @@
 #include "kudu/consensus/log_reader.h"
 #include "kudu/consensus/log_util.h"
 #include "kudu/fs/block_id.h"
+#include "kudu/fs/data_dirs.h"
 #include "kudu/fs/fs.pb.h"
+#include "kudu/fs/fs_manager.h"
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/port.h"
 #include "kudu/gutil/ref_counted.h"
@@ -55,6 +58,8 @@
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 
+DECLARE_bool(crash_on_eio);
+DECLARE_double(env_inject_eio);
 DECLARE_uint64(tablet_copy_idle_timeout_sec);
 DECLARE_uint64(tablet_copy_timeout_poll_period_ms);
 
@@ -509,5 +514,40 @@ TEST_F(TabletCopyServiceTest, TestSessionTimeout) {
   ASSERT_FALSE(resp.session_is_active()) << "Tablet Copy session did not time out!";
 }
 
+// Test that the tablet copy session will terminate on disk failures.
+TEST_F(TabletCopyServiceTest, TestDiskFailureDuringSession) {
+  string session_id;
+  tablet::TabletSuperBlockPB superblock;
+  ASSERT_OK(DoBeginValidTabletCopySession(&session_id, &superblock));
+
+  // Get a block id locally that we'll copy.
+  BlockId block_id = FirstColumnBlockId(superblock);
+  Slice local_data;
+  faststring scratch;
+  ASSERT_OK(ReadLocalBlockFile(mini_server_->server()->fs_manager(), block_id,
+                               &scratch, &local_data));
+
+  // Copy over the block while one of the directories is failed.
+  FetchDataResponsePB resp;
+  RpcController controller;
+  ASSERT_OK(mini_server_->server()->fs_manager()->dd_manager()->MarkDataDirFailed(1));
+  Status s = DoFetchData(session_id, AsDataTypeId(block_id), nullptr, nullptr, &resp, &controller);
+  LOG(INFO) << "Fetch data request responded with: " << s.ToString();
+  ASSERT_STR_CONTAINS(s.ToString(), "Unable to get piece of data block");
+  ASSERT_TRUE(s.IsRemoteError());
+
+  // Now close the copy session.
+  RpcController end_copy_controller;
+  EndTabletCopySessionResponsePB end_session_resp;
+  ASSERT_OK(DoEndTabletCopySession(
+      session_id, true, nullptr, &end_session_resp, &end_copy_controller));
+
+  // Starting a new session should fail.
+  s = DoBeginValidTabletCopySession(&session_id, &superblock);
+  LOG(INFO) << "Begin copy session request responded with: " << s.ToString();
+  ASSERT_STR_CONTAINS(s.ToString(), "Error beginning tablet copy session");
+  ASSERT_TRUE(s.IsRemoteError());
+}
+
 } // namespace tserver
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_source_session.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_source_session.cc b/src/kudu/tserver/tablet_copy_source_session.cc
index 229fe3b..e2a5227 100644
--- a/src/kudu/tserver/tablet_copy_source_session.cc
+++ b/src/kudu/tserver/tablet_copy_source_session.cc
@@ -34,6 +34,7 @@
 #include "kudu/consensus/opid_util.h"
 #include "kudu/consensus/raft_consensus.h"
 #include "kudu/fs/block_manager.h"
+#include "kudu/fs/data_dirs.h"
 #include "kudu/fs/fs.pb.h"
 #include "kudu/fs/fs_manager.h"
 #include "kudu/gutil/basictypes.h"
@@ -129,6 +130,7 @@ Status TabletCopySourceSession::InitOnce() {
   }
 
   RETURN_NOT_OK(tablet_replica_->CheckRunning());
+  RETURN_NOT_OK(CheckHealthyDirGroup());
 
   const string& tablet_id = tablet_replica_->tablet_id();
 
@@ -300,6 +302,8 @@ Status TabletCopySourceSession::GetBlockPiece(const BlockId& block_id,
                                              string* data, int64_t* block_file_size,
                                              TabletCopyErrorPB::Code* error_code) {
   DCHECK(init_once_.init_succeeded());
+  RETURN_NOT_OK_PREPEND(CheckHealthyDirGroup(error_code),
+                        "Tablet copy source could not get block");
   ImmutableReadableBlockInfo* block_info;
   RETURN_NOT_OK(FindBlock(block_id, &block_info, error_code));
 
@@ -320,6 +324,8 @@ Status TabletCopySourceSession::GetLogSegmentPiece(uint64_t segment_seqno,
                                                    std::string* data, int64_t* log_file_size,
                                                    TabletCopyErrorPB::Code* error_code) {
   DCHECK(init_once_.init_succeeded());
+  RETURN_NOT_OK_PREPEND(CheckHealthyDirGroup(error_code),
+                        "Tablet copy source could not get log segment");
   ImmutableRandomAccessFileInfo* file_info;
   RETURN_NOT_OK(FindLogSegment(segment_seqno, &file_info, error_code));
   RETURN_NOT_OK(ReadFileChunkToBuf(file_info, offset, client_maxlen,
@@ -356,6 +362,17 @@ static Status AddImmutableFileToMap(Collection* const cache,
   return Status::OK();
 }
 
+Status TabletCopySourceSession::CheckHealthyDirGroup(TabletCopyErrorPB::Code* error_code) const {
+  if (fs_manager_->dd_manager()->IsTabletInFailedDir(tablet_id())) {
+    if (error_code) {
+      *error_code = TabletCopyErrorPB::IO_ERROR;
+    }
+    return Status::IOError(
+        Substitute("Tablet $0 is in a failed directory", tablet_id()));
+  }
+  return Status::OK();
+}
+
 Status TabletCopySourceSession::OpenBlock(const BlockId& block_id) {
   unique_ptr<ReadableBlock> block;
   Status s = fs_manager_->OpenBlock(block_id, &block);

http://git-wip-us.apache.org/repos/asf/kudu/blob/8a81f4ff/src/kudu/tserver/tablet_copy_source_session.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_source_session.h b/src/kudu/tserver/tablet_copy_source_session.h
index 4f6c350..3c433ed 100644
--- a/src/kudu/tserver/tablet_copy_source_session.h
+++ b/src/kudu/tserver/tablet_copy_source_session.h
@@ -174,6 +174,10 @@ class TabletCopySourceSession : public RefCountedThreadSafe<TabletCopySourceSess
   // Internal helper method for Init().
   Status InitOnce();
 
+  // Returns an error if any directories in the tablet's directory group are
+  // unhealthy and sets 'error_code' appropriately.
+  Status CheckHealthyDirGroup(TabletCopyErrorPB::Code* error_code = nullptr) const;
+
   // Open the block and add it to the block map.
   Status OpenBlock(const BlockId& block_id);