You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2018/01/06 03:23:03 UTC
kudu git commit: [tests] update for 3-4-3 replica management scheme
(3/3)
Repository: kudu
Updated Branches:
refs/heads/master 36995e260 -> 391e3255d
[tests] update for 3-4-3 replica management scheme (3/3)
Updated scenarios of the following tests to run with 3-4-3 replica
management scheme:
* DiskFailureITest
* TabletCopyFailureITest
* TabletCopyITest
* TsTabletManagerITest
This patch is a part of the work done in the context of KUDU-1097.
Change-Id: Id68e29fcee4e6dbd166ada9f18dac2d8fe351a3b
Reviewed-on: http://gerrit.cloudera.org:8080/8858
Reviewed-by: Mike Percy <mp...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/391e3255
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/391e3255
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/391e3255
Branch: refs/heads/master
Commit: 391e3255d6f0ae59f375e5532cea985384f636f8
Parents: 36995e2
Author: Alexey Serbin <as...@cloudera.com>
Authored: Fri Dec 15 15:09:24 2017 -0800
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Sat Jan 6 03:22:24 2018 +0000
----------------------------------------------------------------------
.../integration-tests/disk_failure-itest.cc | 61 +++++++++----
src/kudu/integration-tests/tablet_copy-itest.cc | 18 +++-
.../ts_tablet_manager-itest.cc | 90 +++++++++++++-------
3 files changed, 120 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/391e3255/src/kudu/integration-tests/disk_failure-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/disk_failure-itest.cc b/src/kudu/integration-tests/disk_failure-itest.cc
index 3657846..8d79884 100644
--- a/src/kudu/integration-tests/disk_failure-itest.cc
+++ b/src/kudu/integration-tests/disk_failure-itest.cc
@@ -18,6 +18,7 @@
#include <cstdint>
#include <memory>
#include <string>
+#include <utility>
#include <vector>
#include <gtest/gtest.h>
@@ -29,6 +30,7 @@
#include "kudu/integration-tests/external_mini_cluster-itest-base.h"
#include "kudu/integration-tests/test_workload.h"
#include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/tserver/tserver.pb.h"
#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
#include "kudu/util/path_util.h"
@@ -38,19 +40,19 @@
METRIC_DECLARE_gauge_uint64(data_dirs_failed);
-namespace kudu {
-
-using cluster::ExternalMiniClusterOptions;
-using cluster::ExternalTabletServer;
-using fs::BlockManager;
+using kudu::cluster::ExternalMiniClusterOptions;
+using kudu::cluster::ExternalTabletServer;
+using kudu::fs::BlockManager;
using std::string;
using std::vector;
using strings::Substitute;
+namespace kudu {
+
const MonoDelta kAgreementTimeout = MonoDelta::FromSeconds(30);
class DiskFailureITest : public ExternalMiniClusterITestBase,
- public ::testing::WithParamInterface<string> {
+ public ::testing::WithParamInterface<std::tuple<string, bool>> {
public:
// Waits for 'ext_tserver' to experience 'target_failed_disks' disk failures.
@@ -70,14 +72,24 @@ class DiskFailureITest : public ExternalMiniClusterITestBase,
// shut down and restarted. Errors are injected to one of the directories while
// it is shut down.
TEST_P(DiskFailureITest, TestFailDuringServerStartup) {
- // Set up a cluster with three servers with five disks each.
+ const string block_manager_type = std::get<0>(GetParam());
+ const bool is_3_4_3_mode = std::get<1>(GetParam());
+ // Set up a cluster with five disks at each tablet server. In case of 3-4-3
+ // replication scheme one more tablet server is needed to put the replacement
+ // non-voter replica there.
+ const auto kNumTabletServers = is_3_4_3_mode ? 4 : 3;
+ const auto kNumTablets = 5;
+ const auto kNumRows = 100;
+
ExternalMiniClusterOptions opts;
- opts.num_tablet_servers = 3;
+ opts.num_tablet_servers = kNumTabletServers;
opts.num_data_dirs = 5;
- opts.block_manager_type = GetParam();
+ opts.block_manager_type = block_manager_type;
+ opts.extra_master_flags.push_back(
+ Substitute("--raft_prepare_replacement_before_eviction=$0", is_3_4_3_mode));
+ opts.extra_tserver_flags.push_back(
+ Substitute("--raft_prepare_replacement_before_eviction=$0", is_3_4_3_mode));
NO_FATALS(StartClusterWithOpts(opts));
- const int kNumTablets = 5;
- const int kNumRows = 100;
// Write some data to a tablet. This will spread blocks across all
// directories.
@@ -90,14 +102,25 @@ TEST_P(DiskFailureITest, TestFailDuringServerStartup) {
});
write_workload.StopAndJoin();
- // Ensure the tablets get to a running state.
- ExternalTabletServer* ts = cluster_->tablet_server(0);
- ASSERT_OK(cluster_->WaitForTabletsRunning(ts, kNumTablets, kAgreementTimeout));
+ // Arbitrarily select one tablet server which hosts a replica of the tablet.
+ ExternalTabletServer* ts = nullptr;
+ for (const auto& e : ts_map_) {
+ vector<tserver::ListTabletsResponsePB::StatusAndSchemaPB> tablets;
+ ASSERT_OK(itest::ListTablets(e.second, kAgreementTimeout, &tablets));
+ if (!tablets.empty()) {
+ ts = cluster_->tablet_server_by_uuid(e.first);
+ break;
+ }
+ }
+ ASSERT_NE(nullptr, ts);
+
+ // Ensure at least one tablet get to a running state at one of the tablet servers.
+ ASSERT_OK(cluster_->WaitForTabletsRunning(ts, 1, kAgreementTimeout));
// Introduce flags to fail one of the directories, avoiding the metadata
// directory, the next time the tablet server starts.
- string failed_dir = ts->data_dirs()[1];
- vector<string> extra_flags = {
+ const string& failed_dir = ts->data_dirs()[1];
+ const vector<string> extra_flags = {
Substitute("--env_inject_eio_globs=$0", JoinPathSegments(failed_dir, "**")),
"--env_inject_eio=1.0",
"--crash_on_eio=false",
@@ -107,7 +130,7 @@ TEST_P(DiskFailureITest, TestFailDuringServerStartup) {
// Restart the tablet server with disk failures and ensure it can startup.
ASSERT_OK(ts->Restart());
- NO_FATALS(WaitForDiskFailures(cluster_->tablet_server(0)));
+ NO_FATALS(WaitForDiskFailures(ts));
// Ensure that the tablets are successfully evicted and copied.
ClusterVerifier v(cluster_.get());
@@ -117,6 +140,8 @@ TEST_P(DiskFailureITest, TestFailDuringServerStartup) {
}
INSTANTIATE_TEST_CASE_P(DiskFailure, DiskFailureITest,
- ::testing::ValuesIn(BlockManager::block_manager_types()));
+ ::testing::Combine(
+ ::testing::ValuesIn(BlockManager::block_manager_types()),
+ ::testing::Bool()));
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/391e3255/src/kudu/integration-tests/tablet_copy-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/tablet_copy-itest.cc b/src/kudu/integration-tests/tablet_copy-itest.cc
index f977f18..3be6d0f 100644
--- a/src/kudu/integration-tests/tablet_copy-itest.cc
+++ b/src/kudu/integration-tests/tablet_copy-itest.cc
@@ -1138,11 +1138,25 @@ INSTANTIATE_TEST_CASE_P(FailureCause, TabletCopyFailureITest,
TEST_P(TabletCopyFailureITest, TestTabletCopyNewReplicaFailureCanVote) {
const MonoDelta kTimeout = MonoDelta::FromSeconds(30);
const string& failure_flag = GetParam();
- NO_FATALS(StartCluster({failure_flag}, {}, /*num_tablet_servers=*/ 4));
+ const vector<string> kMasterFlags = {
+ // If running with the 3-4-3 replication scheme, it's necessary to disable
+ // the default catalog manager's behavior of adding and evicting tablet
+ // replicas: this test scenario manages replicasa on its own.
+ "--catalog_manager_evict_excess_replicas=false",
+ "--master_add_server_when_underreplicated=false",
+ };
+ const vector<string> kTserverFlags = {
+ failure_flag
+ };
+ constexpr auto kNumReplicas = 3;
+ constexpr auto kNumTabletServers = kNumReplicas + 1;
+
+ NO_FATALS(StartCluster(kTserverFlags, kMasterFlags, kNumTabletServers));
+
TestWorkload workload(cluster_.get());
workload.Setup();
- ASSERT_OK(inspect_->WaitForReplicaCount(3));
+ ASSERT_OK(inspect_->WaitForReplicaCount(kNumReplicas));
master::GetTableLocationsResponsePB table_locations;
ASSERT_OK(itest::GetTableLocations(cluster_->master_proxy(), TestWorkload::kDefaultTableName,
kTimeout, master::VOTER_REPLICA, &table_locations));
http://git-wip-us.apache.org/repos/asf/kudu/blob/391e3255/src/kudu/integration-tests/ts_tablet_manager-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/ts_tablet_manager-itest.cc b/src/kudu/integration-tests/ts_tablet_manager-itest.cc
index ee5a957..c4ff736 100644
--- a/src/kudu/integration-tests/ts_tablet_manager-itest.cc
+++ b/src/kudu/integration-tests/ts_tablet_manager-itest.cc
@@ -110,10 +110,8 @@ class TsTabletManagerITest : public KuduTest {
ASSERT_OK(bld.Build(&client_messenger_));
}
- void StartCluster(int num_replicas) {
- InternalMiniClusterOptions opts;
- opts.num_tablet_servers = num_replicas;
- cluster_.reset(new InternalMiniCluster(env_, opts));
+ void StartCluster(InternalMiniClusterOptions opts) {
+ cluster_.reset(new InternalMiniCluster(env_, std::move(opts)));
ASSERT_OK(cluster_->Start());
ASSERT_OK(cluster_->CreateClient(nullptr, &client_));
}
@@ -185,61 +183,90 @@ void TsTabletManagerITest::GetIncrementalTabletReports(
reports->swap(r);
}
-// Test that when a tablet is marked as failed, it will eventually be evicted
-// and replaced.
-TEST_F(TsTabletManagerITest, TestFailedTabletsAreReplaced) {
- const int kNumReplicas = 3;
- NO_FATALS(StartCluster(kNumReplicas));
-
- InternalMiniClusterOptions opts;
- opts.num_tablet_servers = kNumReplicas;
- unique_ptr<InternalMiniCluster> cluster(new InternalMiniCluster(env_, opts));
- ASSERT_OK(cluster->Start());
- TestWorkload work(cluster.get());
- work.set_num_replicas(3);
+class FailedTabletsAreReplacedITest :
+ public TsTabletManagerITest,
+ public ::testing::WithParamInterface<bool> {
+};
+// Test that when a tablet replica is marked as failed, it will eventually be
+// evicted and replaced.
+TEST_P(FailedTabletsAreReplacedITest, OneReplica) {
+ const bool is_3_4_3_mode = GetParam();
+ FLAGS_raft_prepare_replacement_before_eviction = is_3_4_3_mode;
+ const auto kNumReplicas = 3;
+ const auto kNumTabletServers = kNumReplicas + (is_3_4_3_mode ? 1 : 0);
+
+ {
+ InternalMiniClusterOptions opts;
+ opts.num_tablet_servers = kNumTabletServers;
+ NO_FATALS(StartCluster(std::move(opts)));
+ }
+ TestWorkload work(cluster_.get());
+ work.set_num_replicas(kNumReplicas);
work.Setup();
work.Start();
// Insert data until the tablet becomes visible to the server.
- // We'll operate on the first tablet server, chosen arbitrarily.
- MiniTabletServer* ts = cluster->mini_tablet_server(0);
string tablet_id;
ASSERT_EVENTUALLY([&] {
- vector<string> tablet_ids = ts->ListTablets();
+ auto idx = rand() % kNumTabletServers;
+ vector<string> tablet_ids = cluster_->mini_tablet_server(idx)->ListTablets();
ASSERT_EQ(1, tablet_ids.size());
tablet_id = tablet_ids[0];
});
- // Wait until the replica is running before failing it.
+ // Wait until all the replicas are running before failing one arbitrarily.
const auto wait_until_running = [&]() {
AssertEventually([&]{
- scoped_refptr<TabletReplica> replica;
- ASSERT_OK(ts->server()->tablet_manager()->GetTabletReplica(tablet_id, &replica));
- ASSERT_EQ(replica->state(), tablet::RUNNING);
+ auto num_replicas_running = 0;
+ for (auto idx = 0; idx < cluster_->num_tablet_servers(); ++idx) {
+ MiniTabletServer* ts = cluster_->mini_tablet_server(idx);
+ scoped_refptr<TabletReplica> replica;
+ Status s = ts->server()->tablet_manager()->GetTabletReplica(tablet_id, &replica);
+ if (s.IsNotFound()) {
+ continue;
+ }
+ ASSERT_OK(s);
+ if (tablet::RUNNING == replica->state()) {
+ ++num_replicas_running;
+ }
+ }
+ ASSERT_EQ(kNumReplicas, num_replicas_running);
}, MonoDelta::FromSeconds(60));
NO_PENDING_FATALS();
};
- wait_until_running();
+ NO_FATALS(wait_until_running());
{
- // Inject an error to the replica. Shutting it down will leave it FAILED.
+ // Inject an error into one of replicas. Shutting it down will leave it in
+ // the FAILED state.
scoped_refptr<TabletReplica> replica;
- ASSERT_OK(ts->server()->tablet_manager()->GetTabletReplica(tablet_id, &replica));
+ ASSERT_EVENTUALLY([&] {
+ auto idx = rand() % kNumTabletServers;
+ MiniTabletServer* ts = cluster_->mini_tablet_server(idx);
+ ASSERT_OK(ts->server()->tablet_manager()->GetTabletReplica(tablet_id, &replica));
+ });
replica->SetError(Status::IOError("INJECTED ERROR: tablet failed"));
replica->Shutdown();
ASSERT_EQ(tablet::FAILED, replica->state());
}
// Ensure the tablet eventually is replicated.
- wait_until_running();
+ NO_FATALS(wait_until_running());
work.StopAndJoin();
}
+INSTANTIATE_TEST_CASE_P(,
+ FailedTabletsAreReplacedITest,
+ ::testing::Bool());
// Test that when the leader changes, the tablet manager gets notified and
// includes that information in the next tablet report.
TEST_F(TsTabletManagerITest, TestReportNewLeaderOnLeaderChange) {
const int kNumReplicas = 2;
- NO_FATALS(StartCluster(kNumReplicas));
+ {
+ InternalMiniClusterOptions opts;
+ opts.num_tablet_servers = kNumReplicas;
+ NO_FATALS(StartCluster(std::move(opts)));
+ }
// We need to control elections precisely for this test since we're using
// EmulateElection() with a distributed consensus configuration.
@@ -331,8 +358,13 @@ TEST_F(TsTabletManagerITest, ReportOnReplicaHealthStatus) {
constexpr int kNumReplicas = 3;
const auto kTimeout = MonoDelta::FromSeconds(60);
+ // This test is specific to the 3-4-3 replica management scheme.
FLAGS_raft_prepare_replacement_before_eviction = true;
- NO_FATALS(StartCluster(kNumReplicas));
+ {
+ InternalMiniClusterOptions opts;
+ opts.num_tablet_servers = kNumReplicas;
+ NO_FATALS(StartCluster(std::move(opts)));
+ }
// Create the table.
client::sp::shared_ptr<KuduTable> table;