You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2017/11/22 06:44:28 UTC
[2/3] kudu git commit: tablet: add early returns to maintenance
functions
tablet: add early returns to maintenance functions
When a Tablet is stopped, further maintenance ops are not scheduled,
further IO is prevented, etc. This patch optimizes this further to stop
various functions that are called by maintenance threads to prevent
their execution, returning an error instead.
Previously, certain ops (e.g. flush DMS) would guarantee durability by
checking for the success these functions. These checks are now replaced
with on-error checks that the tablet has been stopped, since these
failures are inconsequential if the tablet is stopped.
Currently this is an optimization for tablets that are shutting down to
return early from these calls, but in the future, this could be useful
in stopping all IO done by a tablet that is failing, e.g. due to disk
failure.
Change-Id: I84ad557851863f6fd9acff28ddcd1244e62cf516
Reviewed-on: http://gerrit.cloudera.org:8080/8606
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy <mp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ae0143ca
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ae0143ca
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ae0143ca
Branch: refs/heads/master
Commit: ae0143ca5dca54a527c905043f96ebf2b293629b
Parents: fb0eced
Author: Andrew Wong <aw...@cloudera.com>
Authored: Thu Oct 26 12:10:21 2017 -0700
Committer: Andrew Wong <aw...@cloudera.com>
Committed: Wed Nov 22 06:39:16 2017 +0000
----------------------------------------------------------------------
src/kudu/integration-tests/stop_tablet-itest.cc | 6 ++++++
src/kudu/tablet/tablet.cc | 9 +++++++--
src/kudu/tablet/tablet_replica.cc | 7 +++++++
src/kudu/tablet/tablet_replica.h | 4 ++++
src/kudu/tablet/tablet_replica_mm_ops.cc | 16 ++++++++++------
5 files changed, 34 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/ae0143ca/src/kudu/integration-tests/stop_tablet-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/stop_tablet-itest.cc b/src/kudu/integration-tests/stop_tablet-itest.cc
index c182b6c..04b57b6 100644
--- a/src/kudu/integration-tests/stop_tablet-itest.cc
+++ b/src/kudu/integration-tests/stop_tablet-itest.cc
@@ -97,6 +97,12 @@ class StopTabletITest : public MiniClusterITestBase,
LOG(INFO) << Substitute("Stopping T $0 P $1", tablet_id, ts->uuid());;
ASSERT_OK(ts->server()->tablet_manager()->GetTabletReplica(tablet_id, &replica));
replica->tablet()->Stop();
+
+ // We need to also stop replica ops since, upon running with a stopped
+ // tablet, they'll abort immediately and hog the maintenance threads. In
+ // production, this isn't an issue because replica ops are expected to be
+ // unregistered shortly after stopping the tablet.
+ replica->CancelMaintenanceOpsForTests();
}
// Starts a new cluster and creates a tablet with the specified number of
http://git-wip-us.apache.org/repos/asf/kudu/blob/ae0143ca/src/kudu/tablet/tablet.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet.cc b/src/kudu/tablet/tablet.cc
index 15d4d4f..887bb7e 100644
--- a/src/kudu/tablet/tablet.cc
+++ b/src/kudu/tablet/tablet.cc
@@ -1065,6 +1065,10 @@ Status Tablet::Flush() {
Status Tablet::FlushUnlocked() {
TRACE_EVENT0("tablet", "Tablet::FlushUnlocked");
+ {
+ std::lock_guard<simple_spinlock> l(state_lock_);
+ RETURN_NOT_OK(CheckHasNotBeenStoppedUnlocked());
+ }
RowSetsInCompaction input;
shared_ptr<MemRowSet> old_mrs;
{
@@ -1665,8 +1669,7 @@ Status Tablet::Compact(CompactFlags flags) {
input.DumpToLog();
- return DoMergeCompactionOrFlush(input,
- TabletMetadata::kNoMrsFlushed);
+ return DoMergeCompactionOrFlush(input, TabletMetadata::kNoMrsFlushed);
}
void Tablet::UpdateCompactionStats(MaintenanceOpStats* stats) {
@@ -1874,6 +1877,7 @@ void Tablet::GetInfoForBestDMSToFlush(const ReplaySizeMap& replay_size_map,
}
Status Tablet::FlushDMSWithHighestRetention(const ReplaySizeMap& replay_size_map) const {
+ RETURN_IF_STOPPED_OR_CHECK_STATE(kOpen);
shared_ptr<RowSet> rowset = FindBestDMSToFlush(replay_size_map);
if (rowset) {
return rowset->FlushDeltas();
@@ -2112,6 +2116,7 @@ Status Tablet::InitAncientUndoDeltas(MonoDelta time_budget, int64_t* bytes_in_an
}
Status Tablet::DeleteAncientUndoDeltas(int64_t* blocks_deleted, int64_t* bytes_deleted) {
+ RETURN_IF_STOPPED_OR_CHECK_STATE(kOpen);
MonoTime tablet_delete_start = MonoTime::Now();
Timestamp ancient_history_mark;
http://git-wip-us.apache.org/repos/asf/kudu/blob/ae0143ca/src/kudu/tablet/tablet_replica.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet_replica.cc b/src/kudu/tablet/tablet_replica.cc
index 3dcc68c..05c8e79 100644
--- a/src/kudu/tablet/tablet_replica.cc
+++ b/src/kudu/tablet/tablet_replica.cc
@@ -686,6 +686,13 @@ void TabletReplica::RegisterMaintenanceOps(MaintenanceManager* maint_mgr) {
tablet_->RegisterMaintenanceOps(maint_mgr);
}
+void TabletReplica::CancelMaintenanceOpsForTests() {
+ std::lock_guard<simple_spinlock> l(state_change_lock_);
+ for (MaintenanceOp* op : maintenance_ops_) {
+ op->CancelAndDisable();
+ }
+}
+
void TabletReplica::UnregisterMaintenanceOps() {
DCHECK(state_change_lock_.is_locked());
for (MaintenanceOp* op : maintenance_ops_) {
http://git-wip-us.apache.org/repos/asf/kudu/blob/ae0143ca/src/kudu/tablet/tablet_replica.h
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet_replica.h b/src/kudu/tablet/tablet_replica.h
index ca2a2eb..5cc7c9f 100644
--- a/src/kudu/tablet/tablet_replica.h
+++ b/src/kudu/tablet/tablet_replica.h
@@ -271,6 +271,10 @@ class TabletReplica : public RefCountedThreadSafe<TabletReplica>,
// This method is not thread safe.
void UnregisterMaintenanceOps();
+ // Cancels the maintenance ops associated with this replica's tablet.
+ // Only to be used in tests.
+ void CancelMaintenanceOpsForTests();
+
// Return pointer to the transaction tracker for this peer.
const TransactionTracker* transaction_tracker() const { return &txn_tracker_; }
http://git-wip-us.apache.org/repos/asf/kudu/blob/ae0143ca/src/kudu/tablet/tablet_replica_mm_ops.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/tablet_replica_mm_ops.cc b/src/kudu/tablet/tablet_replica_mm_ops.cc
index e643abd..41f04a3 100644
--- a/src/kudu/tablet/tablet_replica_mm_ops.cc
+++ b/src/kudu/tablet/tablet_replica_mm_ops.cc
@@ -141,8 +141,7 @@ void FlushMRSOp::Perform() {
Status s = tablet->FlushUnlocked();
if (PREDICT_FALSE(!s.ok())) {
- LOG(WARNING) << Substitute("FlushMRS failed on $0: $1",
- tablet->tablet_id(), s.ToString());
+ LOG(WARNING) << tablet->LogPrefix() << "failed to flush MRS: " << s.ToString();
CHECK(tablet->HasBeenStopped()) << "FlushMRS failure is only allowed if the "
"tablet is stopped first";
return;
@@ -194,10 +193,15 @@ void FlushDeltaMemStoresOp::Perform() {
<< tablet_replica_->tablet_id();
return;
}
- KUDU_CHECK_OK_PREPEND(tablet_replica_->tablet()->FlushDMSWithHighestRetention(
- max_idx_to_replay_size),
- Substitute("Failed to flush DMS on $0",
- tablet_replica_->tablet()->tablet_id()));
+ Tablet* tablet = tablet_replica_->tablet();
+ Status s = tablet->FlushDMSWithHighestRetention(max_idx_to_replay_size);
+ if (PREDICT_FALSE(!s.ok())) {
+ LOG(WARNING) << tablet->LogPrefix() << "failed to flush DMS: " << s.ToString();
+ CHECK(tablet->HasBeenStopped()) << "FlushDMS failure is only allowed if the "
+ "tablet is stopped first";
+ return;
+ }
+
{
std::lock_guard<simple_spinlock> l(lock_);
time_since_flush_.start();