svn commit: r1181540 - /hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

[ns...@apache.org]

Author: nspiegelberg
Date: Tue Oct 11 02:18:04 2011
New Revision: 1181540

URL: http://svn.apache.org/viewvc?rev=1181540&view=rev
Log:
Abort the HRegionServer immediately when run out of memory.

Summary:
Currently, when the HRegionServer runs out of the memory, it will call master,
which will run out of memory again.
The easy way is to abort the RegionServer immediately when it hits the memory
boundary.

Test Plan:
will be tested in the dl cluster

Reviewed By: kannan
Reviewers: kannan, nspiegelberg, gqchen, kranganathan
Commenters: dhruba, nspiegelberg
CC: hbase@lists, , dhruba, kannan, nspiegelberg
Differential Revision: 249713

Modified:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1181540&r1=1181539&r2=1181540&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Oct 11 02:18:04 2011
@@ -842,18 +842,27 @@ public class HRegionServer implements HR
         new IOException(t): new IOException(msg, t));
   }
   /*
-   * Check if an OOME and if so, call abort.
+   * Check if an OOME and if so, call abort immediately and avoid creating more
+   * objects.
    * @param e
    * @return True if we OOME'd and are aborting.
    */
   public boolean checkOOME(final Throwable e) {
     boolean stop = false;
-    if (e instanceof OutOfMemoryError ||
-      (e.getCause() != null && e.getCause() instanceof OutOfMemoryError) ||
-      (e.getMessage() != null &&
-        e.getMessage().contains("java.lang.OutOfMemoryError"))) {
-      abort("OutOfMemoryError, aborting", e);
-      stop = true;
+    try {
+      if (e instanceof OutOfMemoryError ||
+        (e.getCause() != null && e.getCause() instanceof OutOfMemoryError) ||
+        (e.getMessage() != null &&
+          e.getMessage().contains("java.lang.OutOfMemoryError"))) {
+        stop = true;
+        // not sure whether log4j will create more string obj here or not
+        LOG.fatal("Run out of memory;HRegionServer aborts itself immediately",
+            e);
+      }
+    } finally {
+      if (stop) {
+        this.forceAbort();
+      }
     }
     return stop;
   }
@@ -1176,6 +1185,14 @@ public class HRegionServer implements HR
     abort(reason, null);
   }
 
+  /**
+   * when the region server run out of memory, it needs to abort itseft quickly
+   * and avoid creating more objects.
+   */
+  public void forceAbort() {
+    Runtime.getRuntime().halt(1);
+  }
+
   /*
    * Simulate a kill -9 of this server.
    * Exits w/o closing regions or cleaninup logs but it does close socket in