You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/03/30 19:18:38 UTC

svn commit: r1462831 - in /hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase: ipc/ regionserver/

Author: liyin
Date: Sat Mar 30 18:18:38 2013
New Revision: 1462831

URL: http://svn.apache.org/r1462831
Log:
[HBASE-8221] [0.89-fb] Ensure that we flush/close the regions before RS stops answering requests

Author: aaiyer

Summary:
Ensure that we will first close the regions, and then stop taking
    client requests.

    This will reduce the number of errors seen by the client.

Test Plan:
    MR unit tests
    TestHRegionCloseRetry is the only test that fails.

Reviewers: kannan, liyintang, kranganathan, rshroff

Reviewed By: rshroff

CC: hbase-eng@

Differential Revision: https://phabricator.fb.com/D729486

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java?rev=1462831&r1=1462830&r2=1462831&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java Sat Mar 30 18:18:38 2013
@@ -618,6 +618,13 @@ public class HBaseRPC {
             HRegion.incrNumericPersistentMetric("slowResponse."
                 + call.getMethodName() + ".cumulative", 1);
           }
+          if (tooLarge) {
+            // increment global slow RPC response counter
+            rpcMetrics.inc("largeResponse.kb.", (int)(responseSize/1024));
+            HRegion.incrNumericPersistentMetric("largeResponse.all.cumulative", 1);
+            HRegion.incrNumericPersistentMetric("largeResponse."
+                + call.getMethodName() + ".cumulative", 1);
+          }
         }
         if (processingTime > 1000) {
           // we use a hard-coded one second period so that we can clearly

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java?rev=1462831&r1=1462830&r2=1462831&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java Sat Mar 30 18:18:38 2013
@@ -1158,10 +1158,13 @@ public abstract class HBaseServer {
     }
 
     private void queueRawCall() throws InterruptedException {
+      // To ensure that the received timestamp is set
+      // accurately create the call before the throttler.
+      RawCall call = new RawCall(this, data);
       callQueueThrottler.increase(data.limit());
       try {
         // queue the call
-        callQueue.put(new RawCall(this, data));
+        callQueue.put(call);
       } catch (InterruptedException e)  {
         callQueueThrottler.decrease(data.limit());
       }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=1462831&r1=1462830&r2=1462831&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Sat Mar 30 18:18:38 2013
@@ -143,7 +143,7 @@ public class CompactSplitThread {
   public synchronized void requestCompaction(final HRegion r, final Store s,
       final String why, int priority) {
 
-    if (this.server.stopRequested.get()) {
+    if (this.server.stopRequestedAtStageOne.get()) {
       return;
     }
 

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1462831&r1=1462830&r2=1462831&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Sat Mar 30 18:18:38 2013
@@ -181,7 +181,19 @@ public class HRegionServer implements HR
   // of HRegionServer in isolation. We use AtomicBoolean rather than
   // plain boolean so we can pass a reference to Chore threads.  Otherwise,
   // Chore threads need to know about the hosting class.
-  protected final AtomicBoolean stopRequested = new AtomicBoolean(false);
+  // Stop is going to happen in 2 stages:
+  // In Stage 1: we will close all non-essential chores, and start closing the regions.
+  //    To reduce downtime, closing a region involves two flushes: a pre-flush,
+  //    and a second flush. The pre-flush is going to flush all the data in
+  //    memstore so far, while serving client requests in parallel. The second
+  //    flush will flush the puts received during the pre-flush. It is essential
+  //    that the chores corresponding to MemstoreFlusher, and the LogRoller are
+  //    active at this stage.
+  // In Stage 2: we have closed all the regions. So, we will close out the remaining
+  //    threads and chores that need to be up during stage 1 (Memstore flusher,
+  //    and log roller).
+  protected final AtomicBoolean stopRequestedAtStageOne = new AtomicBoolean(false);
+  protected final AtomicBoolean stopRequestedAtStageTwo = new AtomicBoolean(false);
 
   protected final AtomicBoolean quiesced = new AtomicBoolean(false);
 
@@ -463,7 +475,8 @@ public class HRegionServer implements HR
   private void reinitialize() throws IOException {
     this.restartRequested = false;
     this.abortRequested = false;
-    this.stopRequested.set(false);
+    this.stopRequestedAtStageOne.set(false);
+    this.stopRequestedAtStageTwo.set(false);
 
     // Address is giving a default IP for the moment. Will be changed after
     // calling the master.
@@ -569,7 +582,7 @@ public class HRegionServer implements HR
     EventType type = event.getType();
 
     // Ignore events if we're shutting down.
-    if (this.stopRequested.get()) {
+    if (this.stopRequestedAtStageOne.get()) {
       LOG.debug("Ignoring ZooKeeper event while shutting down");
       return;
     }
@@ -621,7 +634,7 @@ public class HRegionServer implements HR
     boolean quiesceRequested = false;
     try {
       MapWritable w = null;
-      while (!stopRequested.get()) {
+      while (!stopRequestedAtStageOne.get()) {
         w = reportForDuty();
         if (w != null) {
           init(w);
@@ -634,7 +647,7 @@ public class HRegionServer implements HR
       List<HMsg> outboundMessages = new ArrayList<HMsg>();
       long lastMsg = 0;
       // Now ask master what it wants us to do and tell it what we have done
-      for (int tries = 0; !stopRequested.get() && isHealthy();) {
+      for (int tries = 0; !stopRequestedAtStageOne.get() && isHealthy();) {
         // Try to get the root region location from the master.
         if (!haveRootRegion.get()) {
           HServerAddress rootServer = zooKeeperWrapper.readRootRegionLocation();
@@ -676,7 +689,7 @@ public class HRegionServer implements HR
               // serving any regions. So set the stop bit and exit.
               LOG.info("Server quiesced and not serving any regions. " +
                 "Starting shutdown");
-              stopRequested.set(true);
+              stopRequestedAtStageOne.set(true);
               this.outboundMsgs.clear();
               continue;
             }
@@ -684,14 +697,14 @@ public class HRegionServer implements HR
             // Queue up the HMaster's instruction stream for processing
             boolean restart = false;
             for(int i = 0;
-                !restart && !stopRequested.get() && i < msgs.length;
+                !restart && !stopRequestedAtStageOne.get() && i < msgs.length;
                 i++) {
               LOG.info(msgs[i].toString());
               this.connection.unsetRootRegionLocation();
               switch(msgs[i].getType()) {
 
               case MSG_REGIONSERVER_STOP:
-                stopRequested.set(true);
+                stopRequestedAtStageOne.set(true);
                 break;
 
               case MSG_REGIONSERVER_QUIESCE:
@@ -718,7 +731,7 @@ public class HRegionServer implements HR
             // Reset tries count if we had a successful transaction.
             tries = 0;
 
-            if (restart || this.stopRequested.get()) {
+            if (restart || this.stopRequestedAtStageOne.get()) {
               toDo.clear();
               continue;
             }
@@ -737,7 +750,7 @@ public class HRegionServer implements HR
               // Check filesystem every so often.
               checkFileSystem();
             }
-            if (this.stopRequested.get()) {
+            if (this.stopRequestedAtStageOne.get()) {
               LOG.info("Stop requested, clearing toDo despite exception");
               toDo.clear();
               continue;
@@ -786,47 +799,6 @@ public class HRegionServer implements HR
     t.setDaemon(true);
     t.start();
 
-
-    // shutdown thriftserver
-    if (thriftServer != null) {
-      thriftServer.shutdown();
-    }
-    this.leases.closeAfterLeasesExpire();
-    isRpcServerRunning = false;
-    this.worker.stop();
-    if (this.server != null) {
-      this.server.stop();
-    }
-    if (this.splitLogWorkers != null) {
-      for(SplitLogWorker splitLogWorker: splitLogWorkers) {
-        splitLogWorker.stop();
-      }
-    }
-    if (this.infoServer != null) {
-      LOG.info("Stopping infoServer");
-      try {
-        this.infoServer.stop();
-      } catch (Exception e) {
-        e.printStackTrace();
-      }
-    }
-    // Send cache a shutdown.
-    if (cacheConfig.isBlockCacheEnabled()) {
-      cacheConfig.getBlockCache().shutdown();
-    }
-
-    // Send interrupts to wake up threads if sleeping so they notice shutdown.
-    // TODO: Should we check they are alive?  If OOME could have exited already
-    cacheFlusher.interruptIfNecessary();
-    compactSplitThread.interruptIfNecessary();
-    for (int i = 0; i < hlogRollers.length; i++) {  
-      hlogRollers[i].interruptIfNecessary();  
-    }
-    this.majorCompactionChecker.interrupt();
-    
-    // shutdown the prefetch threads
-    scanPrefetchThreadPool.shutdownNow();
-
     if (killed) {
       // Just skip out w/o closing regions.
       this.killAllHLogs();
@@ -889,6 +861,10 @@ public class HRegionServer implements HR
       LOG.info("stopping server at: " + this.serverInfo.getServerName());
     }
 
+    // Let us close the server threads after all the regions are closed. This way, we
+    // will have lesser errors (since the server is responsive during the pre-flush).
+    shutdownServers();
+
     // Make sure the proxy is down.
     if (this.hbaseMaster != null) {
       HBaseRPC.stopProxy(this.hbaseMaster);
@@ -898,7 +874,7 @@ public class HRegionServer implements HR
     this.zooKeeperWrapper.close();
     if (!killed) {
       join();
-      if ((this.fs != null) && (stopRequested.get() || abortRequested)) {
+      if ((this.fs != null) && (stopRequestedAtStageOne.get() || abortRequested)) {
         // Finally attempt to close the Filesystem, to flush out any open streams.
         try {
           this.fs.close();
@@ -910,6 +886,60 @@ public class HRegionServer implements HR
     LOG.info(Thread.currentThread().getName() + " exiting");
   }
 
+  private void shutdownServers() {
+    // We should start ignoring client requests now.
+    // So, shutdown the MemstoreFlusher and LogRoller
+    this.stopRequestedAtStageTwo.set(true);
+
+    // shutdown thriftserver
+    if (thriftServer != null) {
+      thriftServer.shutdown();
+    }
+
+    this.leases.closeAfterLeasesExpire();
+    isRpcServerRunning = false;
+    this.worker.stop();
+    // Don't let the worker thread delay us for 10 secs.
+    this.workerThread.interrupt();
+
+    if (this.server != null) {
+      this.server.stop();
+    }
+    if (this.splitLogWorkers != null) {
+      for(SplitLogWorker splitLogWorker: splitLogWorkers) {
+        splitLogWorker.stop();
+      }
+    }
+    if (this.infoServer != null) {
+      LOG.info("Stopping infoServer");
+      try {
+        this.infoServer.stop();
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
+    }
+
+    // interrupt the lease handler thread;
+    this.leases.interrupt();
+
+    // Send cache a shutdown.
+    if (cacheConfig.isBlockCacheEnabled()) {
+      cacheConfig.getBlockCache().shutdown();
+    }
+
+    // Send interrupts to wake up threads if sleeping so they notice shutdown.
+    // TODO: Should we check they are alive?  If OOME could have exited already
+    cacheFlusher.interruptIfNecessary();
+    compactSplitThread.interruptIfNecessary();
+    for (int i = 0; i < hlogRollers.length; i++) {
+      hlogRollers[i].interruptIfNecessary();
+    }
+    this.majorCompactionChecker.interrupt();
+
+    // shutdown the prefetch threads
+    scanPrefetchThreadPool.shutdownNow();
+  }
+
   /**
    * Add to the passed <code>msgs</code> messages to pass to the master.
    *
@@ -1034,7 +1064,7 @@ public class HRegionServer implements HR
       }
     } catch (Throwable e) {
       this.isOnline = false;
-      this.stopRequested.set(true);
+      this.stopRequestedAtStageOne.set(true);
       throw convertThrowableToIOE(cleanup(e, "Failed init"),
         "Region server startup failed");
     }
@@ -1696,7 +1726,7 @@ public class HRegionServer implements HR
    */
   @Override
   public void stop(String why) {
-    this.stopRequested.set(true);
+    this.stopRequestedAtStageOne.set(true);
     stopReason = why;
     synchronized(this) {
       // Wakes run() if it is sleeping
@@ -1781,7 +1811,7 @@ public class HRegionServer implements HR
     HServerAddress prevMasterAddress = null;
     HServerAddress masterAddress = null;
     HMasterRegionInterface master = null;
-    while (!stopRequested.get() && master == null) {
+    while (!stopRequestedAtStageOne.get() && master == null) {
       // Re-read master address from ZK as it might have changed.
       masterAddress = readMasterAddressFromZK();
       if (masterAddress == null) {
@@ -1827,7 +1857,7 @@ public class HRegionServer implements HR
    * Run initialization using parameters passed us by the master.
    */
   private MapWritable reportForDuty() throws YouAreDeadException {
-    while (!stopRequested.get() && !getMaster()) {
+    while (!stopRequestedAtStageOne.get() && !getMaster()) {
       sleeper.sleep();
       LOG.warn("Unable to get master for initialization");
     }
@@ -1835,7 +1865,7 @@ public class HRegionServer implements HR
     MapWritable result = null;
     long lastMsg = 0;
     boolean znodeWritten = false;
-    while(!stopRequested.get()) {
+    while(!stopRequestedAtStageOne.get()) {
       try {
         MemoryUsage memory =
           ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
@@ -1915,11 +1945,11 @@ public class HRegionServer implements HR
     @Override
     public void run() {
       try {
-        while(!stopRequested.get()) {
+        while(!stopRequestedAtStageOne.get()) {
           ToDoEntry e = null;
           try {
             e = toDo.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);
-            if(e == null || stopRequested.get()) {
+            if(e == null || stopRequestedAtStageOne.get()) {
               continue;
             }
             LOG.info("Worker: " + e.msg);
@@ -2008,10 +2038,10 @@ public class HRegionServer implements HR
               break;
 
             case TESTING_MSG_BLOCK_RS:
-              while (!stopRequested.get()) {
+              while (!stopRequestedAtStageOne.get()) {
                 Threads.sleep(1000);
                 LOG.info("Regionserver blocked by " +
-                  HMsg.Type.TESTING_MSG_BLOCK_RS + "; " + stopRequested.get());
+                  HMsg.Type.TESTING_MSG_BLOCK_RS + "; " + stopRequestedAtStageOne.get());
               }
               break;
 
@@ -2907,7 +2937,14 @@ public class HRegionServer implements HR
    * @return true if a stop or abort has been requested.
    */
   public boolean isStopRequested() {
-    return this.stopRequested.get() || abortRequested;
+    return this.stopRequestedAtStageOne.get() || abortRequested;
+  }
+
+  /**
+   * @return true if a stop or abort has been requested.
+   */
+  public boolean isStopRequestedAtStageTwo() {
+    return this.stopRequestedAtStageTwo.get() || abortRequested;
   }
 
   /**
@@ -3186,7 +3223,7 @@ public class HRegionServer implements HR
    * @throws IOException
    */
   protected void checkOpen() throws IOException {
-    if (this.stopRequested.get() || this.abortRequested) {
+    if (this.stopRequestedAtStageTwo.get() || this.abortRequested) {
       throw new IOException("Server not running" +
         (this.abortRequested? ", aborting": ""));
     }
@@ -3584,7 +3621,7 @@ public class HRegionServer implements HR
 
   @Override
   public boolean isStopped() {
-    return stopRequested.get();
+    return stopRequestedAtStageOne.get();
   }
 
   @Override

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java?rev=1462831&r1=1462830&r2=1462831&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java Sat Mar 30 18:18:38 2013
@@ -65,7 +65,7 @@ class LogRoller extends HasThread implem
   public void run() {
     MonitoredTask status = null;
     int retried = -1;
-    while (!server.isStopRequested()) {
+    while (!server.isStopRequestedAtStageTwo()) {
       long now = System.currentTimeMillis();
       boolean periodic = false;
       long modifiedRollPeriod;

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java?rev=1462831&r1=1462830&r2=1462831&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java Sat Mar 30 18:18:38 2013
@@ -152,7 +152,7 @@ class MemStoreFlusher implements FlushRe
 
     @Override
     public void run() {
-      while (!server.isStopRequested()) {
+      while (!server.isStopRequestedAtStageTwo()) {
         FlushQueueEntry fqe = null;
         try {
           fqe = flushQueue.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);