You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/01/25 04:46:34 UTC
incubator-kudu git commit: Delete a tablet explicitly when it is
replaced
Repository: incubator-kudu
Updated Branches:
refs/heads/master 687fabf40 -> c5d748198
Delete a tablet explicitly when it is replaced
When master send CreateReplica to all replica peers of a tablet, if any
peer timeout, master will replace the old tablet with a new one, but it
will not delete the old replicas on TS. These relicas will be removed if
TS report them to master, but only when:
1. Quorum config change.
2. TS start and make a full report.
If none of those happen, these replicas become orphaned replicas.
Also we may need to add periodic full replica report in TS, to prevent
potential inconsistencies like this, Haven't figure out how to identify
and log the inconsistency accurately, so defer this.
Change-Id: Ib58b1efc334a27e825e68f34dbf3f05c7da60606
Reviewed-on: http://gerrit.cloudera.org:8080/1770
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/c5d74819
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/c5d74819
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/c5d74819
Branch: refs/heads/master
Commit: c5d748198df48007b2a6fe07c2a214a885fc41ee
Parents: 687fabf
Author: Binglin Chang <de...@gmail.com>
Authored: Mon Jan 11 20:24:03 2016 +0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Mon Jan 25 03:46:19 2016 +0000
----------------------------------------------------------------------
.../integration-tests/create-table-itest.cc | 15 ++++++++++
src/kudu/master/catalog_manager.cc | 29 ++++++++++++++------
src/kudu/master/catalog_manager.h | 3 ++
3 files changed, 39 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c5d74819/src/kudu/integration-tests/create-table-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/create-table-itest.cc b/src/kudu/integration-tests/create-table-itest.cc
index e1a360e..055fa44 100644
--- a/src/kudu/integration-tests/create-table-itest.cc
+++ b/src/kudu/integration-tests/create-table-itest.cc
@@ -30,6 +30,7 @@ using std::vector;
METRIC_DECLARE_entity(server);
METRIC_DECLARE_histogram(handler_latency_kudu_tserver_TabletServerAdminService_CreateTablet);
+METRIC_DECLARE_histogram(handler_latency_kudu_tserver_TabletServerAdminService_DeleteTablet);
namespace kudu {
@@ -77,6 +78,20 @@ TEST_F(CreateTableITest, TestCreateWhenMajorityOfReplicasFailCreation) {
LOG(INFO) << "Waiting for the master to retry creating the tablet 3 times... "
<< num_create_attempts << " RPCs seen so far";
+ int64_t num_delete_tablet_rpc;
+ ASSERT_OK(cluster_->tablet_server(0)->GetInt64Metric(
+ &METRIC_ENTITY_server,
+ "kudu.tabletserver",
+ &METRIC_handler_latency_kudu_tserver_TabletServerAdminService_DeleteTablet,
+ "total_count",
+ &num_delete_tablet_rpc));
+ // When attempting to replace old tablet and create new tablet,
+ // master should also send delete tablet rpc to tablet servers.
+ // There may be race for async create/delete rpcs, but there
+ // absolute difference should be less than or equal to 1.
+ ASSERT_GE(num_delete_tablet_rpc - num_create_attempts, -1);
+ ASSERT_LE(num_delete_tablet_rpc - num_create_attempts, 1);
+
// The CreateTable operation should still be considered in progress, even though
// we'll be successful at creating a single replica.
bool in_progress = false;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c5d74819/src/kudu/master/catalog_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index 74bfd44..e328a8f 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -2409,6 +2409,19 @@ void CatalogManager::SendAlterTabletRequest(const scoped_refptr<TabletInfo>& tab
WARN_NOT_OK(call->Run(), "Failed to send alter table request");
}
+void CatalogManager::DeleteTabletReplicas(
+ const TabletInfo* tablet,
+ const std::string& msg) {
+ TabletInfo::ReplicaMap locations;
+ tablet->GetReplicaLocations(&locations);
+ LOG(INFO) << "Sending DeleteTablet for " << locations.size()
+ << " replicas of tablet " << tablet->tablet_id();
+ for (const TabletInfo::ReplicaMap::value_type& r : locations) {
+ SendDeleteTabletRequest(tablet->tablet_id(), TABLET_DATA_DELETED,
+ boost::none, tablet->table(), r.second.ts_desc, msg);
+ }
+}
+
void CatalogManager::DeleteTabletsAndSendRequests(const scoped_refptr<TableInfo>& table) {
vector<scoped_refptr<TabletInfo> > tablets;
table->GetAllTablets(&tablets);
@@ -2416,14 +2429,7 @@ void CatalogManager::DeleteTabletsAndSendRequests(const scoped_refptr<TableInfo>
string deletion_msg = "Table deleted at " + LocalTimeAsString();
for (const scoped_refptr<TabletInfo>& tablet : tablets) {
- TabletInfo::ReplicaMap locations;
- tablet->GetReplicaLocations(&locations);
- LOG(INFO) << "Sending DeleteTablet for " << locations.size()
- << " replicas of tablet " << tablet->tablet_id();
- for (const TabletInfo::ReplicaMap::value_type& r : locations) {
- SendDeleteTabletRequest(tablet->tablet_id(), TABLET_DATA_DELETED,
- boost::none, table, r.second.ts_desc, deletion_msg);
- }
+ DeleteTabletReplicas(tablet.get(), deletion_msg);
TabletMetadataLock tablet_lock(tablet.get(), TabletMetadataLock::WRITE);
tablet_lock.mutable_data()->set_state(SysTabletsEntryPB::DELETED, deletion_msg);
@@ -2746,6 +2752,13 @@ Status CatalogManager::ProcessPendingAssignments(
return s;
}
+ // Send DeleteTablet requests to tablet servers serving deleted tablets.
+ // This is asynchronous / non-blocking.
+ for (const TabletInfo* tablet : deferred.tablets_to_update) {
+ if (tablet->metadata().dirty().is_deleted()) {
+ DeleteTabletReplicas(tablet, tablet->metadata().dirty().pb.state_msg());
+ }
+ }
// Send the CreateTablet() requests to the servers. This is asynchronous / non-blocking.
SendCreateTabletRequests(deferred.needs_create_rpc);
return Status::OK();
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/c5d74819/src/kudu/master/catalog_manager.h
----------------------------------------------------------------------
diff --git a/src/kudu/master/catalog_manager.h b/src/kudu/master/catalog_manager.h
index 55d918e..b74cf39 100644
--- a/src/kudu/master/catalog_manager.h
+++ b/src/kudu/master/catalog_manager.h
@@ -560,6 +560,9 @@ class CatalogManager : public tserver::TabletPeerLookupIf {
// tablet.
void SendAlterTabletRequest(const scoped_refptr<TabletInfo>& tablet);
+ // Request tablet servers to delete all replicas of the tablet.
+ void DeleteTabletReplicas(const TabletInfo* tablet, const std::string& msg);
+
// Marks each of the tablets in the given table as deleted and triggers requests
// to the tablet servers to delete them.
void DeleteTabletsAndSendRequests(const scoped_refptr<TableInfo>& table);