svn commit: r1344055 - /hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java

[st...@apache.org]

Author: stack
Date: Tue May 29 23:58:13 2012
New Revision: 1344055

URL: http://svn.apache.org/viewvc?rev=1344055&view=rev
Log:
HBASE-6107 Distributed log splitting hangs even there is no task under /hbase/splitlog

Modified:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1344055&r1=1344054&r2=1344055&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java Tue May 29 23:58:13 2012
@@ -321,6 +321,23 @@ public class SplitLogManager extends Zoo
               + " scheduled=" + batch.installed
               + " done=" + batch.done
               + " error=" + batch.error);
+          int remaining = batch.installed - (batch.done + batch.error);
+          int actual = activeTasks(batch);
+          if (remaining != actual) {
+            LOG.warn("Expected " + remaining
+              + " active tasks, but actually there are " + actual);
+          }
+          int remainingInZK = remainingTasksInZK();
+          if (remainingInZK >= 0 && actual > remainingInZK) {
+            LOG.warn("Expected at least" + actual
+              + " tasks in ZK, but actually there are " + remainingInZK);
+          }
+          if (remainingInZK == 0 || actual == 0) {
+            LOG.warn("No more task remaining (ZK or task map), splitting "
+              + "should have completed. Remaining tasks in ZK " + remainingInZK
+              + ", active tasks in map " + actual);
+            return;
+          }
           batch.wait(100);
           if (stopper.isStopped()) {
             LOG.warn("Stopped while waiting for log splits to be completed");
@@ -335,6 +352,35 @@ public class SplitLogManager extends Zoo
     }
   }
 
+  private int activeTasks(final TaskBatch batch) {
+    int count = 0;
+    for (Task t: tasks.values()) {
+      if (t.batch == batch && t.status == TerminationStatus.IN_PROGRESS) {
+        count++;
+      }
+    }
+    return count;
+  }
+
+  private int remainingTasksInZK() {
+    int count = 0;
+    try {
+      List<String> tasks =
+        ZKUtil.listChildrenNoWatch(watcher, watcher.splitLogZNode);
+      if (tasks != null) {
+        for (String t: tasks) {
+          if (!ZKSplitLog.isRescanNode(watcher, t)) {
+            count++;
+          }
+        }
+      }
+    } catch (KeeperException ke) {
+      LOG.warn("Failed to check remaining tasks", ke);
+      count = -1;
+    }
+    return count;
+  }
+
   private void setDone(String path, TerminationStatus status) {
     Task task = tasks.get(path);
     if (task == null) {