[3/3] kudu git commit: maintenance_manager: schedule work immediately when threads are free

[to...@apache.org]

maintenance_manager: schedule work immediately when threads are free

This changes the MM so that, when a worker thread becomes available, it
immediately wakes up the scheduler to schedule the next available work item.
Additionally, the scheduler will loop scheduling new items as long as there
are free worker threads, instead of only scheduling once per polling
interval.

Change-Id: I63c4b48f5f02f3a1d3a8964993e78037ce72b1da
Reviewed-on: http://gerrit.cloudera.org:8080/6815
Reviewed-by: David Ribeiro Alves <da...@gmail.com>
Tested-by: Todd Lipcon <to...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/40aa4c3c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/40aa4c3c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/40aa4c3c

Branch: refs/heads/master
Commit: 40aa4c3c271c9df20a17a1d353ce582ee3fda742
Parents: b1aacd9
Author: Todd Lipcon <to...@apache.org>
Authored: Fri May 5 17:03:53 2017 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue May 9 20:47:46 2017 +0000

----------------------------------------------------------------------
 src/kudu/util/maintenance_manager.cc | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/40aa4c3c/src/kudu/util/maintenance_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/maintenance_manager.cc b/src/kudu/util/maintenance_manager.cc
index 387e79f..18ed8b8 100644
--- a/src/kudu/util/maintenance_manager.cc
+++ b/src/kudu/util/maintenance_manager.cc
@@ -210,24 +210,37 @@ void MaintenanceManager::UnregisterOp(MaintenanceOp* op) {
 }
 
 void MaintenanceManager::RunSchedulerThread() {
+  if (!FLAGS_enable_maintenance_manager) {
+    LOG(INFO) << "Maintenance manager is disabled. Stopping thread.";
+    return;
+  }
+
   MonoDelta polling_interval = MonoDelta::FromMilliseconds(polling_interval_ms_);
 
   std::unique_lock<Mutex> guard(lock_);
+
+  // Set to true if the scheduler runs and finds that there is no work to do.
+  bool prev_iter_found_no_work = false;
+
   while (true) {
-    // Loop until we are shutting down or it is time to run another op.
-    cond_.TimedWait(polling_interval);
+    // We'll keep sleeping if:
+    //    1) there are no free threads available to perform a maintenance op.
+    // or 2) we just tried to schedule an op but found nothing to run.
+    // However, if it's time to shut down, we want to do so immediately.
+    while ((running_ops_ >= num_threads_ || prev_iter_found_no_work) && !shutdown_) {
+      cond_.TimedWait(polling_interval);
+      prev_iter_found_no_work = false;
+    }
     if (shutdown_) {
       VLOG_AND_TRACE("maintenance", 1) << LogPrefix() << "Shutting down maintenance manager.";
       return;
     }
 
-    if (!FLAGS_enable_maintenance_manager) {
-      KLOG_EVERY_N_SECS(INFO, 30) << "Maintenance manager is disabled. Doing nothing";
-      return;
-    }
-
     // Find the best op.
     MaintenanceOp* op = FindBestOp();
+    // If we found no work to do, then we should sleep before trying again to schedule.
+    // Otherwise, we can go right into trying to find the next op.
+    prev_iter_found_no_work = (op == nullptr);
     if (!op) {
       VLOG_AND_TRACE("maintenance", 2) << LogPrefix()
                                        << "No maintenance operations look worth doing.";
@@ -437,6 +450,7 @@ void MaintenanceManager::LaunchOp(MaintenanceOp* op) {
   running_ops_--;
   op->running_--;
   op->cond_->Signal();
+  cond_.Signal(); // wake up scheduler
 }
 
 void MaintenanceManager::GetMaintenanceManagerStatusDump(MaintenanceManagerStatusPB* out_pb) {