You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2018/09/07 14:52:44 UTC

[01/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site 1158b2817 -> c548e10d8


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/downloads.html
----------------------------------------------------------------------
diff --git a/downloads.html b/downloads.html
index ad9bdc4..751255b 100644
--- a/downloads.html
+++ b/downloads.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Apache HBase Downloads</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -395,7 +395,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/export_control.html
----------------------------------------------------------------------
diff --git a/export_control.html b/export_control.html
index 736274c..5d525d5 100644
--- a/export_control.html
+++ b/export_control.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; 
       Export Control
@@ -331,7 +331,7 @@ for more details.</p>
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/index.html
----------------------------------------------------------------------
diff --git a/index.html b/index.html
index f5b9c5b..2b433e0 100644
--- a/index.html
+++ b/index.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Apache HBase™ Home</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -411,7 +411,7 @@ Apache HBase is an open-source, distributed, versioned, non-relational database
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/integration.html
----------------------------------------------------------------------
diff --git a/integration.html b/integration.html
index 1c6e52c..45be8db 100644
--- a/integration.html
+++ b/integration.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; CI Management</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -291,7 +291,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/issue-tracking.html
----------------------------------------------------------------------
diff --git a/issue-tracking.html b/issue-tracking.html
index 964a98d..29231c8 100644
--- a/issue-tracking.html
+++ b/issue-tracking.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Issue Management</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -288,7 +288,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/license.html
----------------------------------------------------------------------
diff --git a/license.html b/license.html
index ef84da4..9b09ba7 100644
--- a/license.html
+++ b/license.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Licenses</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -491,7 +491,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/mail-lists.html
----------------------------------------------------------------------
diff --git a/mail-lists.html b/mail-lists.html
index a0d5058..d57ad5a 100644
--- a/mail-lists.html
+++ b/mail-lists.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Mailing Lists</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -341,7 +341,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/metrics.html
----------------------------------------------------------------------
diff --git a/metrics.html b/metrics.html
index f94095e..551a7d1 100644
--- a/metrics.html
+++ b/metrics.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013;  
       Apache HBase (TM) Metrics
@@ -459,7 +459,7 @@ export HBASE_REGIONSERVER_OPTS=&quot;$HBASE_JMX_OPTS -Dcom.sun.management.jmxrem
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/old_news.html
----------------------------------------------------------------------
diff --git a/old_news.html b/old_news.html
index 5dd69ba..7dbb63f 100644
--- a/old_news.html
+++ b/old_news.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; 
       Old Apache HBase (TM) News
@@ -440,7 +440,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/plugin-management.html
----------------------------------------------------------------------
diff --git a/plugin-management.html b/plugin-management.html
index a103c09..a39890b 100644
--- a/plugin-management.html
+++ b/plugin-management.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Plugin Management</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -414,8 +414,8 @@
 <td>2.7</td></tr>
 <tr class="b">
 <td>org.codehaus.mojo</td>
-<td><a class="externalLink" href="http://mojo.codehaus.org/findbugs-maven-plugin">findbugs-maven-plugin</a></td>
-<td>3.0.0</td></tr>
+<td><a class="externalLink" href="http://gleclaire.github.io/findbugs-maven-plugin/">findbugs-maven-plugin</a></td>
+<td>3.0.4</td></tr>
 <tr class="a">
 <td>org.eclipse.m2e</td>
 <td>lifecycle-mapping</td>
@@ -440,7 +440,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/plugins.html
----------------------------------------------------------------------
diff --git a/plugins.html b/plugins.html
index c85ef27..98620fc 100644
--- a/plugins.html
+++ b/plugins.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Plugins</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -330,8 +330,8 @@
 <td>1.4</td></tr>
 <tr class="a">
 <td>org.codehaus.mojo</td>
-<td><a class="externalLink" href="http://mojo.codehaus.org/findbugs-maven-plugin">findbugs-maven-plugin</a></td>
-<td>3.0.0</td></tr>
+<td><a class="externalLink" href="http://gleclaire.github.io/findbugs-maven-plugin/">findbugs-maven-plugin</a></td>
+<td>3.0.4</td></tr>
 <tr class="b">
 <td>org.codehaus.mojo</td>
 <td><a class="externalLink" href="http://www.mojohaus.org/xml-maven-plugin/">xml-maven-plugin</a></td>
@@ -375,7 +375,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/poweredbyhbase.html
----------------------------------------------------------------------
diff --git a/poweredbyhbase.html b/poweredbyhbase.html
index 14a66ac..08b0dc3 100644
--- a/poweredbyhbase.html
+++ b/poweredbyhbase.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Powered By Apache HBase™</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -769,7 +769,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/project-info.html
----------------------------------------------------------------------
diff --git a/project-info.html b/project-info.html
index 8a0a334..5682401 100644
--- a/project-info.html
+++ b/project-info.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Information</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -335,7 +335,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/project-reports.html
----------------------------------------------------------------------
diff --git a/project-reports.html b/project-reports.html
index 6eaccbe..a982c43 100644
--- a/project-reports.html
+++ b/project-reports.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Generated Reports</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -305,7 +305,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/project-summary.html
----------------------------------------------------------------------
diff --git a/project-summary.html b/project-summary.html
index 322b563..64bde6e 100644
--- a/project-summary.html
+++ b/project-summary.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Summary</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -331,7 +331,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/pseudo-distributed.html
----------------------------------------------------------------------
diff --git a/pseudo-distributed.html b/pseudo-distributed.html
index 490f6b1..4df5273 100644
--- a/pseudo-distributed.html
+++ b/pseudo-distributed.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013;  
 Running Apache HBase (TM) in pseudo-distributed mode
@@ -308,7 +308,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/replication.html
----------------------------------------------------------------------
diff --git a/replication.html b/replication.html
index fee073a..c958e98 100644
--- a/replication.html
+++ b/replication.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; 
       Apache HBase (TM) Replication
@@ -303,7 +303,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/resources.html
----------------------------------------------------------------------
diff --git a/resources.html b/resources.html
index ad707a7..8033410 100644
--- a/resources.html
+++ b/resources.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Other Apache HBase (TM) Resources</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -331,7 +331,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/source-repository.html
----------------------------------------------------------------------
diff --git a/source-repository.html b/source-repository.html
index 0858483..9d9c7ed 100644
--- a/source-repository.html
+++ b/source-repository.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Source Code Management</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -299,7 +299,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/sponsors.html
----------------------------------------------------------------------
diff --git a/sponsors.html b/sponsors.html
index 7d5fb7d..496222f 100644
--- a/sponsors.html
+++ b/sponsors.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Apache HBase™ Sponsors</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -333,7 +333,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/supportingprojects.html
----------------------------------------------------------------------
diff --git a/supportingprojects.html b/supportingprojects.html
index 2745c18..a81521a 100644
--- a/supportingprojects.html
+++ b/supportingprojects.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Supporting Projects</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -520,7 +520,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/team-list.html
----------------------------------------------------------------------
diff --git a/team-list.html b/team-list.html
index 332b960..fd803f5 100644
--- a/team-list.html
+++ b/team-list.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Team</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -754,7 +754,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 


[03/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html
index c63e65a..21df2b9 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html
@@ -432,900 +432,907 @@
 <span class="sourceLineNo">424</span>      it.next(); // Skip the current log<a name="line.424"></a>
 <span class="sourceLineNo">425</span><a name="line.425"></a>
 <span class="sourceLineNo">426</span>      ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {<a name="line.426"></a>
-<span class="sourceLineNo">427</span>        @Override<a name="line.427"></a>
-<span class="sourceLineNo">428</span>        public void setMaxProcId(long maxProcId) {<a name="line.428"></a>
-<span class="sourceLineNo">429</span>          loader.setMaxProcId(maxProcId);<a name="line.429"></a>
-<span class="sourceLineNo">430</span>        }<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>        @Override<a name="line.432"></a>
-<span class="sourceLineNo">433</span>        public void load(ProcedureIterator procIter) throws IOException {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>          loader.load(procIter);<a name="line.434"></a>
-<span class="sourceLineNo">435</span>        }<a name="line.435"></a>
-<span class="sourceLineNo">436</span><a name="line.436"></a>
-<span class="sourceLineNo">437</span>        @Override<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.438"></a>
-<span class="sourceLineNo">439</span>          loader.handleCorrupted(procIter);<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>        @Override<a name="line.442"></a>
-<span class="sourceLineNo">443</span>        public void markCorruptedWAL(ProcedureWALFile log, IOException e) {<a name="line.443"></a>
-<span class="sourceLineNo">444</span>          if (corruptedLogs == null) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span>            corruptedLogs = new HashSet&lt;&gt;();<a name="line.445"></a>
-<span class="sourceLineNo">446</span>          }<a name="line.446"></a>
-<span class="sourceLineNo">447</span>          corruptedLogs.add(log);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>          // TODO: sideline corrupted log<a name="line.448"></a>
-<span class="sourceLineNo">449</span>        }<a name="line.449"></a>
-<span class="sourceLineNo">450</span>      });<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    } finally {<a name="line.451"></a>
-<span class="sourceLineNo">452</span>      try {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>        // try to cleanup inactive wals and complete the operation<a name="line.453"></a>
-<span class="sourceLineNo">454</span>        buildHoldingCleanupTracker();<a name="line.454"></a>
-<span class="sourceLineNo">455</span>        tryCleanupLogsOnLoad();<a name="line.455"></a>
-<span class="sourceLineNo">456</span>        loading.set(false);<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      } finally {<a name="line.457"></a>
-<span class="sourceLineNo">458</span>        lock.unlock();<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      }<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>  }<a name="line.461"></a>
-<span class="sourceLineNo">462</span><a name="line.462"></a>
-<span class="sourceLineNo">463</span>  private void tryCleanupLogsOnLoad() {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>    // nothing to cleanup.<a name="line.464"></a>
-<span class="sourceLineNo">465</span>    if (logs.size() &lt;= 1) return;<a name="line.465"></a>
-<span class="sourceLineNo">466</span><a name="line.466"></a>
-<span class="sourceLineNo">467</span>    // the config says to not cleanup wals on load.<a name="line.467"></a>
-<span class="sourceLineNo">468</span>    if (!conf.getBoolean(EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>      DEFAULT_EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY)) {<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      LOG.debug("WALs cleanup on load is not enabled: " + getActiveLogs());<a name="line.470"></a>
-<span class="sourceLineNo">471</span>      return;<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    }<a name="line.472"></a>
+<span class="sourceLineNo">427</span>        long count = 0;<a name="line.427"></a>
+<span class="sourceLineNo">428</span><a name="line.428"></a>
+<span class="sourceLineNo">429</span>        @Override<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        public void setMaxProcId(long maxProcId) {<a name="line.430"></a>
+<span class="sourceLineNo">431</span>          loader.setMaxProcId(maxProcId);<a name="line.431"></a>
+<span class="sourceLineNo">432</span>        }<a name="line.432"></a>
+<span class="sourceLineNo">433</span><a name="line.433"></a>
+<span class="sourceLineNo">434</span>        @Override<a name="line.434"></a>
+<span class="sourceLineNo">435</span>        public void load(ProcedureIterator procIter) throws IOException {<a name="line.435"></a>
+<span class="sourceLineNo">436</span>          if ((++count % 1000) == 0) {<a name="line.436"></a>
+<span class="sourceLineNo">437</span>            // Log every 1000 procedures otherwise it looks like Master is dead if loads of WALs<a name="line.437"></a>
+<span class="sourceLineNo">438</span>            // and procedures to load.<a name="line.438"></a>
+<span class="sourceLineNo">439</span>            LOG.debug("Loaded {} procedures", this.count);<a name="line.439"></a>
+<span class="sourceLineNo">440</span>          }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>          loader.load(procIter);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>        }<a name="line.442"></a>
+<span class="sourceLineNo">443</span><a name="line.443"></a>
+<span class="sourceLineNo">444</span>        @Override<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.445"></a>
+<span class="sourceLineNo">446</span>          loader.handleCorrupted(procIter);<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        }<a name="line.447"></a>
+<span class="sourceLineNo">448</span><a name="line.448"></a>
+<span class="sourceLineNo">449</span>        @Override<a name="line.449"></a>
+<span class="sourceLineNo">450</span>        public void markCorruptedWAL(ProcedureWALFile log, IOException e) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>          if (corruptedLogs == null) {<a name="line.451"></a>
+<span class="sourceLineNo">452</span>            corruptedLogs = new HashSet&lt;&gt;();<a name="line.452"></a>
+<span class="sourceLineNo">453</span>          }<a name="line.453"></a>
+<span class="sourceLineNo">454</span>          corruptedLogs.add(log);<a name="line.454"></a>
+<span class="sourceLineNo">455</span>          // TODO: sideline corrupted log<a name="line.455"></a>
+<span class="sourceLineNo">456</span>        }<a name="line.456"></a>
+<span class="sourceLineNo">457</span>      });<a name="line.457"></a>
+<span class="sourceLineNo">458</span>    } finally {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>      try {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>        // try to cleanup inactive wals and complete the operation<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        buildHoldingCleanupTracker();<a name="line.461"></a>
+<span class="sourceLineNo">462</span>        tryCleanupLogsOnLoad();<a name="line.462"></a>
+<span class="sourceLineNo">463</span>        loading.set(false);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>      } finally {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>        lock.unlock();<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      }<a name="line.466"></a>
+<span class="sourceLineNo">467</span>    }<a name="line.467"></a>
+<span class="sourceLineNo">468</span>  }<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>  private void tryCleanupLogsOnLoad() {<a name="line.470"></a>
+<span class="sourceLineNo">471</span>    // nothing to cleanup.<a name="line.471"></a>
+<span class="sourceLineNo">472</span>    if (logs.size() &lt;= 1) return;<a name="line.472"></a>
 <span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>    try {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>      periodicRoll();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    } catch (IOException e) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      LOG.warn("Unable to cleanup logs on load: " + e.getMessage(), e);<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    }<a name="line.478"></a>
-<span class="sourceLineNo">479</span>  }<a name="line.479"></a>
+<span class="sourceLineNo">474</span>    // the config says to not cleanup wals on load.<a name="line.474"></a>
+<span class="sourceLineNo">475</span>    if (!conf.getBoolean(EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY,<a name="line.475"></a>
+<span class="sourceLineNo">476</span>      DEFAULT_EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY)) {<a name="line.476"></a>
+<span class="sourceLineNo">477</span>      LOG.debug("WALs cleanup on load is not enabled: " + getActiveLogs());<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      return;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
 <span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span>  @Override<a name="line.481"></a>
-<span class="sourceLineNo">482</span>  public void insert(final Procedure proc, final Procedure[] subprocs) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    if (LOG.isTraceEnabled()) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span>      LOG.trace("Insert " + proc + ", subproc=" + Arrays.toString(subprocs));<a name="line.484"></a>
+<span class="sourceLineNo">481</span>    try {<a name="line.481"></a>
+<span class="sourceLineNo">482</span>      periodicRoll();<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    } catch (IOException e) {<a name="line.483"></a>
+<span class="sourceLineNo">484</span>      LOG.warn("Unable to cleanup logs on load: " + e.getMessage(), e);<a name="line.484"></a>
 <span class="sourceLineNo">485</span>    }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>    ByteSlot slot = acquireSlot();<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    try {<a name="line.488"></a>
-<span class="sourceLineNo">489</span>      // Serialize the insert<a name="line.489"></a>
-<span class="sourceLineNo">490</span>      long[] subProcIds = null;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>      if (subprocs != null) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span>        ProcedureWALFormat.writeInsert(slot, proc, subprocs);<a name="line.492"></a>
-<span class="sourceLineNo">493</span>        subProcIds = new long[subprocs.length];<a name="line.493"></a>
-<span class="sourceLineNo">494</span>        for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>          subProcIds[i] = subprocs[i].getProcId();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>        }<a name="line.496"></a>
-<span class="sourceLineNo">497</span>      } else {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>        assert !proc.hasParent();<a name="line.498"></a>
-<span class="sourceLineNo">499</span>        ProcedureWALFormat.writeInsert(slot, proc);<a name="line.499"></a>
-<span class="sourceLineNo">500</span>      }<a name="line.500"></a>
-<span class="sourceLineNo">501</span><a name="line.501"></a>
-<span class="sourceLineNo">502</span>      // Push the transaction data and wait until it is persisted<a name="line.502"></a>
-<span class="sourceLineNo">503</span>      pushData(PushType.INSERT, slot, proc.getProcId(), subProcIds);<a name="line.503"></a>
-<span class="sourceLineNo">504</span>    } catch (IOException e) {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      // We are not able to serialize the procedure.<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      // this is a code error, and we are not able to go on.<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: proc=" +<a name="line.507"></a>
-<span class="sourceLineNo">508</span>          proc + ", subprocs=" + Arrays.toString(subprocs), e);<a name="line.508"></a>
-<span class="sourceLineNo">509</span>      throw new RuntimeException(e);<a name="line.509"></a>
-<span class="sourceLineNo">510</span>    } finally {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      releaseSlot(slot);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>    }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>  }<a name="line.513"></a>
-<span class="sourceLineNo">514</span><a name="line.514"></a>
-<span class="sourceLineNo">515</span>  @Override<a name="line.515"></a>
-<span class="sourceLineNo">516</span>  public void insert(final Procedure[] procs) {<a name="line.516"></a>
-<span class="sourceLineNo">517</span>    if (LOG.isTraceEnabled()) {<a name="line.517"></a>
-<span class="sourceLineNo">518</span>      LOG.trace("Insert " + Arrays.toString(procs));<a name="line.518"></a>
+<span class="sourceLineNo">486</span>  }<a name="line.486"></a>
+<span class="sourceLineNo">487</span><a name="line.487"></a>
+<span class="sourceLineNo">488</span>  @Override<a name="line.488"></a>
+<span class="sourceLineNo">489</span>  public void insert(final Procedure proc, final Procedure[] subprocs) {<a name="line.489"></a>
+<span class="sourceLineNo">490</span>    if (LOG.isTraceEnabled()) {<a name="line.490"></a>
+<span class="sourceLineNo">491</span>      LOG.trace("Insert " + proc + ", subproc=" + Arrays.toString(subprocs));<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    }<a name="line.492"></a>
+<span class="sourceLineNo">493</span><a name="line.493"></a>
+<span class="sourceLineNo">494</span>    ByteSlot slot = acquireSlot();<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    try {<a name="line.495"></a>
+<span class="sourceLineNo">496</span>      // Serialize the insert<a name="line.496"></a>
+<span class="sourceLineNo">497</span>      long[] subProcIds = null;<a name="line.497"></a>
+<span class="sourceLineNo">498</span>      if (subprocs != null) {<a name="line.498"></a>
+<span class="sourceLineNo">499</span>        ProcedureWALFormat.writeInsert(slot, proc, subprocs);<a name="line.499"></a>
+<span class="sourceLineNo">500</span>        subProcIds = new long[subprocs.length];<a name="line.500"></a>
+<span class="sourceLineNo">501</span>        for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.501"></a>
+<span class="sourceLineNo">502</span>          subProcIds[i] = subprocs[i].getProcId();<a name="line.502"></a>
+<span class="sourceLineNo">503</span>        }<a name="line.503"></a>
+<span class="sourceLineNo">504</span>      } else {<a name="line.504"></a>
+<span class="sourceLineNo">505</span>        assert !proc.hasParent();<a name="line.505"></a>
+<span class="sourceLineNo">506</span>        ProcedureWALFormat.writeInsert(slot, proc);<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      }<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      // Push the transaction data and wait until it is persisted<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      pushData(PushType.INSERT, slot, proc.getProcId(), subProcIds);<a name="line.510"></a>
+<span class="sourceLineNo">511</span>    } catch (IOException e) {<a name="line.511"></a>
+<span class="sourceLineNo">512</span>      // We are not able to serialize the procedure.<a name="line.512"></a>
+<span class="sourceLineNo">513</span>      // this is a code error, and we are not able to go on.<a name="line.513"></a>
+<span class="sourceLineNo">514</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: proc=" +<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          proc + ", subprocs=" + Arrays.toString(subprocs), e);<a name="line.515"></a>
+<span class="sourceLineNo">516</span>      throw new RuntimeException(e);<a name="line.516"></a>
+<span class="sourceLineNo">517</span>    } finally {<a name="line.517"></a>
+<span class="sourceLineNo">518</span>      releaseSlot(slot);<a name="line.518"></a>
 <span class="sourceLineNo">519</span>    }<a name="line.519"></a>
-<span class="sourceLineNo">520</span><a name="line.520"></a>
-<span class="sourceLineNo">521</span>    ByteSlot slot = acquireSlot();<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    try {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      // Serialize the insert<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      long[] procIds = new long[procs.length];<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      for (int i = 0; i &lt; procs.length; ++i) {<a name="line.525"></a>
-<span class="sourceLineNo">526</span>        assert !procs[i].hasParent();<a name="line.526"></a>
-<span class="sourceLineNo">527</span>        procIds[i] = procs[i].getProcId();<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        ProcedureWALFormat.writeInsert(slot, procs[i]);<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      }<a name="line.529"></a>
-<span class="sourceLineNo">530</span><a name="line.530"></a>
-<span class="sourceLineNo">531</span>      // Push the transaction data and wait until it is persisted<a name="line.531"></a>
-<span class="sourceLineNo">532</span>      pushData(PushType.INSERT, slot, Procedure.NO_PROC_ID, procIds);<a name="line.532"></a>
-<span class="sourceLineNo">533</span>    } catch (IOException e) {<a name="line.533"></a>
-<span class="sourceLineNo">534</span>      // We are not able to serialize the procedure.<a name="line.534"></a>
-<span class="sourceLineNo">535</span>      // this is a code error, and we are not able to go on.<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: " +<a name="line.536"></a>
-<span class="sourceLineNo">537</span>          Arrays.toString(procs), e);<a name="line.537"></a>
-<span class="sourceLineNo">538</span>      throw new RuntimeException(e);<a name="line.538"></a>
-<span class="sourceLineNo">539</span>    } finally {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      releaseSlot(slot);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>  public void update(final Procedure proc) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>    if (LOG.isTraceEnabled()) {<a name="line.546"></a>
-<span class="sourceLineNo">547</span>      LOG.trace("Update " + proc);<a name="line.547"></a>
+<span class="sourceLineNo">520</span>  }<a name="line.520"></a>
+<span class="sourceLineNo">521</span><a name="line.521"></a>
+<span class="sourceLineNo">522</span>  @Override<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  public void insert(final Procedure[] procs) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    if (LOG.isTraceEnabled()) {<a name="line.524"></a>
+<span class="sourceLineNo">525</span>      LOG.trace("Insert " + Arrays.toString(procs));<a name="line.525"></a>
+<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span>    ByteSlot slot = acquireSlot();<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    try {<a name="line.529"></a>
+<span class="sourceLineNo">530</span>      // Serialize the insert<a name="line.530"></a>
+<span class="sourceLineNo">531</span>      long[] procIds = new long[procs.length];<a name="line.531"></a>
+<span class="sourceLineNo">532</span>      for (int i = 0; i &lt; procs.length; ++i) {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>        assert !procs[i].hasParent();<a name="line.533"></a>
+<span class="sourceLineNo">534</span>        procIds[i] = procs[i].getProcId();<a name="line.534"></a>
+<span class="sourceLineNo">535</span>        ProcedureWALFormat.writeInsert(slot, procs[i]);<a name="line.535"></a>
+<span class="sourceLineNo">536</span>      }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>      // Push the transaction data and wait until it is persisted<a name="line.538"></a>
+<span class="sourceLineNo">539</span>      pushData(PushType.INSERT, slot, Procedure.NO_PROC_ID, procIds);<a name="line.539"></a>
+<span class="sourceLineNo">540</span>    } catch (IOException e) {<a name="line.540"></a>
+<span class="sourceLineNo">541</span>      // We are not able to serialize the procedure.<a name="line.541"></a>
+<span class="sourceLineNo">542</span>      // this is a code error, and we are not able to go on.<a name="line.542"></a>
+<span class="sourceLineNo">543</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: " +<a name="line.543"></a>
+<span class="sourceLineNo">544</span>          Arrays.toString(procs), e);<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      throw new RuntimeException(e);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    } finally {<a name="line.546"></a>
+<span class="sourceLineNo">547</span>      releaseSlot(slot);<a name="line.547"></a>
 <span class="sourceLineNo">548</span>    }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>    ByteSlot slot = acquireSlot();<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    try {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      // Serialize the update<a name="line.552"></a>
-<span class="sourceLineNo">553</span>      ProcedureWALFormat.writeUpdate(slot, proc);<a name="line.553"></a>
-<span class="sourceLineNo">554</span><a name="line.554"></a>
-<span class="sourceLineNo">555</span>      // Push the transaction data and wait until it is persisted<a name="line.555"></a>
-<span class="sourceLineNo">556</span>      pushData(PushType.UPDATE, slot, proc.getProcId(), null);<a name="line.556"></a>
-<span class="sourceLineNo">557</span>    } catch (IOException e) {<a name="line.557"></a>
-<span class="sourceLineNo">558</span>      // We are not able to serialize the procedure.<a name="line.558"></a>
-<span class="sourceLineNo">559</span>      // this is a code error, and we are not able to go on.<a name="line.559"></a>
-<span class="sourceLineNo">560</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.560"></a>
-<span class="sourceLineNo">561</span>      throw new RuntimeException(e);<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    } finally {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      releaseSlot(slot);<a name="line.563"></a>
-<span class="sourceLineNo">564</span>    }<a name="line.564"></a>
-<span class="sourceLineNo">565</span>  }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>  @Override<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  public void delete(final long procId) {<a name="line.568"></a>
-<span class="sourceLineNo">569</span>    if (LOG.isTraceEnabled()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>      LOG.trace("Delete " + procId);<a name="line.570"></a>
+<span class="sourceLineNo">549</span>  }<a name="line.549"></a>
+<span class="sourceLineNo">550</span><a name="line.550"></a>
+<span class="sourceLineNo">551</span>  @Override<a name="line.551"></a>
+<span class="sourceLineNo">552</span>  public void update(final Procedure proc) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    if (LOG.isTraceEnabled()) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      LOG.trace("Update " + proc);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span><a name="line.556"></a>
+<span class="sourceLineNo">557</span>    ByteSlot slot = acquireSlot();<a name="line.557"></a>
+<span class="sourceLineNo">558</span>    try {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>      // Serialize the update<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      ProcedureWALFormat.writeUpdate(slot, proc);<a name="line.560"></a>
+<span class="sourceLineNo">561</span><a name="line.561"></a>
+<span class="sourceLineNo">562</span>      // Push the transaction data and wait until it is persisted<a name="line.562"></a>
+<span class="sourceLineNo">563</span>      pushData(PushType.UPDATE, slot, proc.getProcId(), null);<a name="line.563"></a>
+<span class="sourceLineNo">564</span>    } catch (IOException e) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span>      // We are not able to serialize the procedure.<a name="line.565"></a>
+<span class="sourceLineNo">566</span>      // this is a code error, and we are not able to go on.<a name="line.566"></a>
+<span class="sourceLineNo">567</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.567"></a>
+<span class="sourceLineNo">568</span>      throw new RuntimeException(e);<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    } finally {<a name="line.569"></a>
+<span class="sourceLineNo">570</span>      releaseSlot(slot);<a name="line.570"></a>
 <span class="sourceLineNo">571</span>    }<a name="line.571"></a>
-<span class="sourceLineNo">572</span><a name="line.572"></a>
-<span class="sourceLineNo">573</span>    ByteSlot slot = acquireSlot();<a name="line.573"></a>
-<span class="sourceLineNo">574</span>    try {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>      // Serialize the delete<a name="line.575"></a>
-<span class="sourceLineNo">576</span>      ProcedureWALFormat.writeDelete(slot, procId);<a name="line.576"></a>
-<span class="sourceLineNo">577</span><a name="line.577"></a>
-<span class="sourceLineNo">578</span>      // Push the transaction data and wait until it is persisted<a name="line.578"></a>
-<span class="sourceLineNo">579</span>      pushData(PushType.DELETE, slot, procId, null);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    } catch (IOException e) {<a name="line.580"></a>
-<span class="sourceLineNo">581</span>      // We are not able to serialize the procedure.<a name="line.581"></a>
-<span class="sourceLineNo">582</span>      // this is a code error, and we are not able to go on.<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + procId, e);<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      throw new RuntimeException(e);<a name="line.584"></a>
-<span class="sourceLineNo">585</span>    } finally {<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      releaseSlot(slot);<a name="line.586"></a>
-<span class="sourceLineNo">587</span>    }<a name="line.587"></a>
-<span class="sourceLineNo">588</span>  }<a name="line.588"></a>
-<span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>  @Override<a name="line.590"></a>
-<span class="sourceLineNo">591</span>  public void delete(final Procedure proc, final long[] subProcIds) {<a name="line.591"></a>
-<span class="sourceLineNo">592</span>    assert proc != null : "expected a non-null procedure";<a name="line.592"></a>
-<span class="sourceLineNo">593</span>    assert subProcIds != null &amp;&amp; subProcIds.length &gt; 0 : "expected subProcIds";<a name="line.593"></a>
-<span class="sourceLineNo">594</span>    if (LOG.isTraceEnabled()) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>      LOG.trace("Update " + proc + " and Delete " + Arrays.toString(subProcIds));<a name="line.595"></a>
-<span class="sourceLineNo">596</span>    }<a name="line.596"></a>
-<span class="sourceLineNo">597</span><a name="line.597"></a>
-<span class="sourceLineNo">598</span>    ByteSlot slot = acquireSlot();<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    try {<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // Serialize the delete<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      ProcedureWALFormat.writeDelete(slot, proc, subProcIds);<a name="line.601"></a>
-<span class="sourceLineNo">602</span><a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // Push the transaction data and wait until it is persisted<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      pushData(PushType.DELETE, slot, proc.getProcId(), subProcIds);<a name="line.604"></a>
-<span class="sourceLineNo">605</span>    } catch (IOException e) {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      // We are not able to serialize the procedure.<a name="line.606"></a>
-<span class="sourceLineNo">607</span>      // this is a code error, and we are not able to go on.<a name="line.607"></a>
-<span class="sourceLineNo">608</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.608"></a>
-<span class="sourceLineNo">609</span>      throw new RuntimeException(e);<a name="line.609"></a>
-<span class="sourceLineNo">610</span>    } finally {<a name="line.610"></a>
-<span class="sourceLineNo">611</span>      releaseSlot(slot);<a name="line.611"></a>
-<span class="sourceLineNo">612</span>    }<a name="line.612"></a>
-<span class="sourceLineNo">613</span>  }<a name="line.613"></a>
-<span class="sourceLineNo">614</span><a name="line.614"></a>
-<span class="sourceLineNo">615</span>  @Override<a name="line.615"></a>
-<span class="sourceLineNo">616</span>  public void delete(final long[] procIds, final int offset, final int count) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>    if (count == 0) return;<a name="line.617"></a>
-<span class="sourceLineNo">618</span>    if (offset == 0 &amp;&amp; count == procIds.length) {<a name="line.618"></a>
-<span class="sourceLineNo">619</span>      delete(procIds);<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    } else if (count == 1) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      delete(procIds[offset]);<a name="line.621"></a>
-<span class="sourceLineNo">622</span>    } else {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>      delete(Arrays.copyOfRange(procIds, offset, offset + count));<a name="line.623"></a>
-<span class="sourceLineNo">624</span>    }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>  }<a name="line.625"></a>
-<span class="sourceLineNo">626</span><a name="line.626"></a>
-<span class="sourceLineNo">627</span>  private void delete(final long[] procIds) {<a name="line.627"></a>
-<span class="sourceLineNo">628</span>    if (LOG.isTraceEnabled()) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>      LOG.trace("Delete " + Arrays.toString(procIds));<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    }<a name="line.630"></a>
-<span class="sourceLineNo">631</span><a name="line.631"></a>
-<span class="sourceLineNo">632</span>    final ByteSlot slot = acquireSlot();<a name="line.632"></a>
-<span class="sourceLineNo">633</span>    try {<a name="line.633"></a>
-<span class="sourceLineNo">634</span>      // Serialize the delete<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.635"></a>
-<span class="sourceLineNo">636</span>        ProcedureWALFormat.writeDelete(slot, procIds[i]);<a name="line.636"></a>
-<span class="sourceLineNo">637</span>      }<a name="line.637"></a>
+<span class="sourceLineNo">572</span>  }<a name="line.572"></a>
+<span class="sourceLineNo">573</span><a name="line.573"></a>
+<span class="sourceLineNo">574</span>  @Override<a name="line.574"></a>
+<span class="sourceLineNo">575</span>  public void delete(final long procId) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span>    if (LOG.isTraceEnabled()) {<a name="line.576"></a>
+<span class="sourceLineNo">577</span>      LOG.trace("Delete " + procId);<a name="line.577"></a>
+<span class="sourceLineNo">578</span>    }<a name="line.578"></a>
+<span class="sourceLineNo">579</span><a name="line.579"></a>
+<span class="sourceLineNo">580</span>    ByteSlot slot = acquireSlot();<a name="line.580"></a>
+<span class="sourceLineNo">581</span>    try {<a name="line.581"></a>
+<span class="sourceLineNo">582</span>      // Serialize the delete<a name="line.582"></a>
+<span class="sourceLineNo">583</span>      ProcedureWALFormat.writeDelete(slot, procId);<a name="line.583"></a>
+<span class="sourceLineNo">584</span><a name="line.584"></a>
+<span class="sourceLineNo">585</span>      // Push the transaction data and wait until it is persisted<a name="line.585"></a>
+<span class="sourceLineNo">586</span>      pushData(PushType.DELETE, slot, procId, null);<a name="line.586"></a>
+<span class="sourceLineNo">587</span>    } catch (IOException e) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>      // We are not able to serialize the procedure.<a name="line.588"></a>
+<span class="sourceLineNo">589</span>      // this is a code error, and we are not able to go on.<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + procId, e);<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      throw new RuntimeException(e);<a name="line.591"></a>
+<span class="sourceLineNo">592</span>    } finally {<a name="line.592"></a>
+<span class="sourceLineNo">593</span>      releaseSlot(slot);<a name="line.593"></a>
+<span class="sourceLineNo">594</span>    }<a name="line.594"></a>
+<span class="sourceLineNo">595</span>  }<a name="line.595"></a>
+<span class="sourceLineNo">596</span><a name="line.596"></a>
+<span class="sourceLineNo">597</span>  @Override<a name="line.597"></a>
+<span class="sourceLineNo">598</span>  public void delete(final Procedure proc, final long[] subProcIds) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>    assert proc != null : "expected a non-null procedure";<a name="line.599"></a>
+<span class="sourceLineNo">600</span>    assert subProcIds != null &amp;&amp; subProcIds.length &gt; 0 : "expected subProcIds";<a name="line.600"></a>
+<span class="sourceLineNo">601</span>    if (LOG.isTraceEnabled()) {<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      LOG.trace("Update " + proc + " and Delete " + Arrays.toString(subProcIds));<a name="line.602"></a>
+<span class="sourceLineNo">603</span>    }<a name="line.603"></a>
+<span class="sourceLineNo">604</span><a name="line.604"></a>
+<span class="sourceLineNo">605</span>    ByteSlot slot = acquireSlot();<a name="line.605"></a>
+<span class="sourceLineNo">606</span>    try {<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // Serialize the delete<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      ProcedureWALFormat.writeDelete(slot, proc, subProcIds);<a name="line.608"></a>
+<span class="sourceLineNo">609</span><a name="line.609"></a>
+<span class="sourceLineNo">610</span>      // Push the transaction data and wait until it is persisted<a name="line.610"></a>
+<span class="sourceLineNo">611</span>      pushData(PushType.DELETE, slot, proc.getProcId(), subProcIds);<a name="line.611"></a>
+<span class="sourceLineNo">612</span>    } catch (IOException e) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>      // We are not able to serialize the procedure.<a name="line.613"></a>
+<span class="sourceLineNo">614</span>      // this is a code error, and we are not able to go on.<a name="line.614"></a>
+<span class="sourceLineNo">615</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.615"></a>
+<span class="sourceLineNo">616</span>      throw new RuntimeException(e);<a name="line.616"></a>
+<span class="sourceLineNo">617</span>    } finally {<a name="line.617"></a>
+<span class="sourceLineNo">618</span>      releaseSlot(slot);<a name="line.618"></a>
+<span class="sourceLineNo">619</span>    }<a name="line.619"></a>
+<span class="sourceLineNo">620</span>  }<a name="line.620"></a>
+<span class="sourceLineNo">621</span><a name="line.621"></a>
+<span class="sourceLineNo">622</span>  @Override<a name="line.622"></a>
+<span class="sourceLineNo">623</span>  public void delete(final long[] procIds, final int offset, final int count) {<a name="line.623"></a>
+<span class="sourceLineNo">624</span>    if (count == 0) return;<a name="line.624"></a>
+<span class="sourceLineNo">625</span>    if (offset == 0 &amp;&amp; count == procIds.length) {<a name="line.625"></a>
+<span class="sourceLineNo">626</span>      delete(procIds);<a name="line.626"></a>
+<span class="sourceLineNo">627</span>    } else if (count == 1) {<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      delete(procIds[offset]);<a name="line.628"></a>
+<span class="sourceLineNo">629</span>    } else {<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      delete(Arrays.copyOfRange(procIds, offset, offset + count));<a name="line.630"></a>
+<span class="sourceLineNo">631</span>    }<a name="line.631"></a>
+<span class="sourceLineNo">632</span>  }<a name="line.632"></a>
+<span class="sourceLineNo">633</span><a name="line.633"></a>
+<span class="sourceLineNo">634</span>  private void delete(final long[] procIds) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>    if (LOG.isTraceEnabled()) {<a name="line.635"></a>
+<span class="sourceLineNo">636</span>      LOG.trace("Delete " + Arrays.toString(procIds));<a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
 <span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>      // Push the transaction data and wait until it is persisted<a name="line.639"></a>
-<span class="sourceLineNo">640</span>      pushData(PushType.DELETE, slot, Procedure.NO_PROC_ID, procIds);<a name="line.640"></a>
-<span class="sourceLineNo">641</span>    } catch (IOException e) {<a name="line.641"></a>
-<span class="sourceLineNo">642</span>      // We are not able to serialize the procedure.<a name="line.642"></a>
-<span class="sourceLineNo">643</span>      // this is a code error, and we are not able to go on.<a name="line.643"></a>
-<span class="sourceLineNo">644</span>      LOG.error("Unable to serialize the procedures: " + Arrays.toString(procIds), e);<a name="line.644"></a>
-<span class="sourceLineNo">645</span>      throw new RuntimeException(e);<a name="line.645"></a>
-<span class="sourceLineNo">646</span>    } finally {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      releaseSlot(slot);<a name="line.647"></a>
-<span class="sourceLineNo">648</span>    }<a name="line.648"></a>
-<span class="sourceLineNo">649</span>  }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span>  private ByteSlot acquireSlot() {<a name="line.651"></a>
-<span class="sourceLineNo">652</span>    ByteSlot slot = slotsCache.poll();<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    return slot != null ? slot : new ByteSlot();<a name="line.653"></a>
-<span class="sourceLineNo">654</span>  }<a name="line.654"></a>
-<span class="sourceLineNo">655</span><a name="line.655"></a>
-<span class="sourceLineNo">656</span>  private void releaseSlot(final ByteSlot slot) {<a name="line.656"></a>
-<span class="sourceLineNo">657</span>    slot.reset();<a name="line.657"></a>
-<span class="sourceLineNo">658</span>    slotsCache.offer(slot);<a name="line.658"></a>
-<span class="sourceLineNo">659</span>  }<a name="line.659"></a>
-<span class="sourceLineNo">660</span><a name="line.660"></a>
-<span class="sourceLineNo">661</span>  private enum PushType { INSERT, UPDATE, DELETE }<a name="line.661"></a>
+<span class="sourceLineNo">639</span>    final ByteSlot slot = acquireSlot();<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    try {<a name="line.640"></a>
+<span class="sourceLineNo">641</span>      // Serialize the delete<a name="line.641"></a>
+<span class="sourceLineNo">642</span>      for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.642"></a>
+<span class="sourceLineNo">643</span>        ProcedureWALFormat.writeDelete(slot, procIds[i]);<a name="line.643"></a>
+<span class="sourceLineNo">644</span>      }<a name="line.644"></a>
+<span class="sourceLineNo">645</span><a name="line.645"></a>
+<span class="sourceLineNo">646</span>      // Push the transaction data and wait until it is persisted<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      pushData(PushType.DELETE, slot, Procedure.NO_PROC_ID, procIds);<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    } catch (IOException e) {<a name="line.648"></a>
+<span class="sourceLineNo">649</span>      // We are not able to serialize the procedure.<a name="line.649"></a>
+<span class="sourceLineNo">650</span>      // this is a code error, and we are not able to go on.<a name="line.650"></a>
+<span class="sourceLineNo">651</span>      LOG.error("Unable to serialize the procedures: " + Arrays.toString(procIds), e);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>      throw new RuntimeException(e);<a name="line.652"></a>
+<span class="sourceLineNo">653</span>    } finally {<a name="line.653"></a>
+<span class="sourceLineNo">654</span>      releaseSlot(slot);<a name="line.654"></a>
+<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
+<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
+<span class="sourceLineNo">657</span><a name="line.657"></a>
+<span class="sourceLineNo">658</span>  private ByteSlot acquireSlot() {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    ByteSlot slot = slotsCache.poll();<a name="line.659"></a>
+<span class="sourceLineNo">660</span>    return slot != null ? slot : new ByteSlot();<a name="line.660"></a>
+<span class="sourceLineNo">661</span>  }<a name="line.661"></a>
 <span class="sourceLineNo">662</span><a name="line.662"></a>
-<span class="sourceLineNo">663</span>  private long pushData(final PushType type, final ByteSlot slot,<a name="line.663"></a>
-<span class="sourceLineNo">664</span>      final long procId, final long[] subProcIds) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>    if (!isRunning()) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span>      throw new RuntimeException("the store must be running before inserting data");<a name="line.666"></a>
-<span class="sourceLineNo">667</span>    }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    if (logs.isEmpty()) {<a name="line.668"></a>
-<span class="sourceLineNo">669</span>      throw new RuntimeException("recoverLease() must be called before inserting data");<a name="line.669"></a>
-<span class="sourceLineNo">670</span>    }<a name="line.670"></a>
-<span class="sourceLineNo">671</span><a name="line.671"></a>
-<span class="sourceLineNo">672</span>    long logId = -1;<a name="line.672"></a>
-<span class="sourceLineNo">673</span>    lock.lock();<a name="line.673"></a>
-<span class="sourceLineNo">674</span>    try {<a name="line.674"></a>
-<span class="sourceLineNo">675</span>      // Wait for the sync to be completed<a name="line.675"></a>
-<span class="sourceLineNo">676</span>      while (true) {<a name="line.676"></a>
-<span class="sourceLineNo">677</span>        if (!isRunning()) {<a name="line.677"></a>
-<span class="sourceLineNo">678</span>          throw new RuntimeException("store no longer running");<a name="line.678"></a>
-<span class="sourceLineNo">679</span>        } else if (isSyncAborted()) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span>          throw new RuntimeException("sync aborted", syncException.get());<a name="line.680"></a>
-<span class="sourceLineNo">681</span>        } else if (inSync.get()) {<a name="line.681"></a>
-<span class="sourceLineNo">682</span>          syncCond.await();<a name="line.682"></a>
-<span class="sourceLineNo">683</span>        } else if (slotIndex &gt;= syncMaxSlot) {<a name="line.683"></a>
-<span class="sourceLineNo">684</span>          slotCond.signal();<a name="line.684"></a>
-<span class="sourceLineNo">685</span>          syncCond.await();<a name="line.685"></a>
-<span class="sourceLineNo">686</span>        } else {<a name="line.686"></a>
-<span class="sourceLineNo">687</span>          break;<a name="line.687"></a>
-<span class="sourceLineNo">688</span>        }<a name="line.688"></a>
-<span class="sourceLineNo">689</span>      }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>      final long pushSyncId = syncId.get();<a name="line.691"></a>
-<span class="sourceLineNo">692</span>      updateStoreTracker(type, procId, subProcIds);<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      slots[slotIndex++] = slot;<a name="line.693"></a>
-<span class="sourceLineNo">694</span>      logId = flushLogId;<a name="line.694"></a>
-<span class="sourceLineNo">695</span><a name="line.695"></a>
-<span class="sourceLineNo">696</span>      // Notify that there is new data<a name="line.696"></a>
-<span class="sourceLineNo">697</span>      if (slotIndex == 1) {<a name="line.697"></a>
-<span class="sourceLineNo">698</span>        waitCond.signal();<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      }<a name="line.699"></a>
-<span class="sourceLineNo">700</span><a name="line.700"></a>
-<span class="sourceLineNo">701</span>      // Notify that the slots are full<a name="line.701"></a>
-<span class="sourceLineNo">702</span>      if (slotIndex == syncMaxSlot) {<a name="line.702"></a>
-<span class="sourceLineNo">703</span>        waitCond.signal();<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        slotCond.signal();<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span><a name="line.706"></a>
-<span class="sourceLineNo">707</span>      while (pushSyncId == syncId.get() &amp;&amp; isRunning()) {<a name="line.707"></a>
-<span class="sourceLineNo">708</span>        syncCond.await();<a name="line.708"></a>
-<span class="sourceLineNo">709</span>      }<a name="line.709"></a>
-<span class="sourceLineNo">710</span>    } catch (InterruptedException e) {<a name="line.710"></a>
-<span class="sourceLineNo">711</span>      Thread.currentThread().interrupt();<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      sendAbortProcessSignal();<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      throw new RuntimeException(e);<a name="line.713"></a>
-<span class="sourceLineNo">714</span>    } finally {<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      lock.unlock();<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      if (isSyncAborted()) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span>        throw new RuntimeException("sync aborted", syncException.get());<a name="line.717"></a>
-<span class="sourceLineNo">718</span>      }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>    }<a name="line.719"></a>
-<span class="sourceLineNo">720</span>    return logId;<a name="line.720"></a>
-<span class="sourceLineNo">721</span>  }<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>  private void updateStoreTracker(final PushType type,<a name="line.723"></a>
-<span class="sourceLineNo">724</span>      final long procId, final long[] subProcIds) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span>    switch (type) {<a name="line.725"></a>
-<span class="sourceLineNo">726</span>      case INSERT:<a name="line.726"></a>
-<span class="sourceLineNo">727</span>        if (subProcIds == null) {<a name="line.727"></a>
-<span class="sourceLineNo">728</span>          storeTracker.insert(procId);<a name="line.728"></a>
-<span class="sourceLineNo">729</span>        } else if (procId == Procedure.NO_PROC_ID) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span>          storeTracker.insert(subProcIds);<a name="line.730"></a>
-<span class="sourceLineNo">731</span>        } else {<a name="line.731"></a>
-<span class="sourceLineNo">732</span>          storeTracker.insert(procId, subProcIds);<a name="line.732"></a>
-<span class="sourceLineNo">733</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.733"></a>
-<span class="sourceLineNo">734</span>        }<a name="line.734"></a>
-<span class="sourceLineNo">735</span>        break;<a name="line.735"></a>
-<span class="sourceLineNo">736</span>      case UPDATE:<a name="line.736"></a>
-<span class="sourceLineNo">737</span>        storeTracker.update(procId);<a name="line.737"></a>
-<span class="sourceLineNo">738</span>        holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.738"></a>
-<span class="sourceLineNo">739</span>        break;<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      case DELETE:<a name="line.740"></a>
-<span class="sourceLineNo">741</span>        if (subProcIds != null &amp;&amp; subProcIds.length &gt; 0) {<a name="line.741"></a>
-<span class="sourceLineNo">742</span>          storeTracker.delete(subProcIds);<a name="line.742"></a>
-<span class="sourceLineNo">743</span>          holdingCleanupTracker.setDeletedIfSet(subProcIds);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>        } else {<a name="line.744"></a>
-<span class="sourceLineNo">745</span>          storeTracker.delete(procId);<a name="line.745"></a>
-<span class="sourceLineNo">746</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.746"></a>
-<span class="sourceLineNo">747</span>        }<a name="line.747"></a>
-<span class="sourceLineNo">748</span>        break;<a name="line.748"></a>
-<span class="sourceLineNo">749</span>      default:<a name="line.749"></a>
-<span class="sourceLineNo">750</span>        throw new RuntimeException("invalid push type " + type);<a name="line.750"></a>
-<span class="sourceLineNo">751</span>    }<a name="line.751"></a>
-<span class="sourceLineNo">752</span>  }<a name="line.752"></a>
-<span class="sourceLineNo">753</span><a name="line.753"></a>
-<span class="sourceLineNo">754</span>  private boolean isSyncAborted() {<a name="line.754"></a>
-<span class="sourceLineNo">755</span>    return syncException.get() != null;<a name="line.755"></a>
-<span class="sourceLineNo">756</span>  }<a name="line.756"></a>
-<span class="sourceLineNo">757</span><a name="line.757"></a>
-<span class="sourceLineNo">758</span>  private void syncLoop() throws Throwable {<a name="line.758"></a>
-<span class="sourceLineNo">759</span>    long totalSyncedToStore = 0;<a name="line.759"></a>
-<span class="sourceLineNo">760</span>    inSync.set(false);<a name="line.760"></a>
-<span class="sourceLineNo">761</span>    lock.lock();<a name="line.761"></a>
-<span class="sourceLineNo">762</span>    try {<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (isRunning()) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        try {<a name="line.764"></a>
-<span class="sourceLineNo">765</span>          // Wait until new data is available<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          if (slotIndex == 0) {<a name="line.766"></a>
-<span class="sourceLineNo">767</span>            if (!loading.get()) {<a name="line.767"></a>
-<span class="sourceLineNo">768</span>              periodicRoll();<a name="line.768"></a>
-<span class="sourceLineNo">769</span>            }<a name="line.769"></a>
-<span class="sourceLineNo">770</span><a name="line.770"></a>
-<span class="sourceLineNo">771</span>            if (LOG.isTraceEnabled()) {<a name="line.771"></a>
-<span class="sourceLineNo">772</span>              float rollTsSec = getMillisFromLastRoll() / 1000.0f;<a name="line.772"></a>
-<span class="sourceLineNo">773</span>              LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)",<a name="line.773"></a>
-<span class="sourceLineNo">774</span>                        StringUtils.humanSize(totalSynced.get()),<a name="line.774"></a>
-<span class="sourceLineNo">775</span>                        StringUtils.humanSize(totalSynced.get() / rollTsSec)));<a name="line.775"></a>
+<span class="sourceLineNo">663</span>  private void releaseSlot(final ByteSlot slot) {<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    slot.reset();<a name="line.664"></a>
+<span class="sourceLineNo">665</span>    slotsCache.offer(slot);<a name="line.665"></a>
+<span class="sourceLineNo">666</span>  }<a name="line.666"></a>
+<span class="sourceLineNo">667</span><a name="line.667"></a>
+<span class="sourceLineNo">668</span>  private enum PushType { INSERT, UPDATE, DELETE }<a name="line.668"></a>
+<span class="sourceLineNo">669</span><a name="line.669"></a>
+<span class="sourceLineNo">670</span>  private long pushData(final PushType type, final ByteSlot slot,<a name="line.670"></a>
+<span class="sourceLineNo">671</span>      final long procId, final long[] subProcIds) {<a name="line.671"></a>
+<span class="sourceLineNo">672</span>    if (!isRunning()) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>      throw new RuntimeException("the store must be running before inserting data");<a name="line.673"></a>
+<span class="sourceLineNo">674</span>    }<a name="line.674"></a>
+<span class="sourceLineNo">675</span>    if (logs.isEmpty()) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span>      throw new RuntimeException("recoverLease() must be called before inserting data");<a name="line.676"></a>
+<span class="sourceLineNo">677</span>    }<a name="line.677"></a>
+<span class="sourceLineNo">678</span><a name="line.678"></a>
+<span class="sourceLineNo">679</span>    long logId = -1;<a name="line.679"></a>
+<span class="sourceLineNo">680</span>    lock.lock();<a name="line.680"></a>
+<span class="sourceLineNo">681</span>    try {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>      // Wait for the sync to be completed<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      while (true) {<a name="line.683"></a>
+<span class="sourceLineNo">684</span>        if (!isRunning()) {<a name="line.684"></a>
+<span class="sourceLineNo">685</span>          throw new RuntimeException("store no longer running");<a name="line.685"></a>
+<span class="sourceLineNo">686</span>        } else if (isSyncAborted()) {<a name="line.686"></a>
+<span class="sourceLineNo">687</span>          throw new RuntimeException("sync aborted", syncException.get());<a name="line.687"></a>
+<span class="sourceLineNo">688</span>        } else if (inSync.get()) {<a name="line.688"></a>
+<span class="sourceLineNo">689</span>          syncCond.await();<a name="line.689"></a>
+<span class="sourceLineNo">690</span>        } else if (slotIndex &gt;= syncMaxSlot) {<a name="line.690"></a>
+<span class="sourceLineNo">691</span>          slotCond.signal();<a name="line.691"></a>
+<span class="sourceLineNo">692</span>          syncCond.await();<a name="line.692"></a>
+<span class="sourceLineNo">693</span>        } else {<a name="line.693"></a>
+<span class="sourceLineNo">694</span>          break;<a name="line.694"></a>
+<span class="sourceLineNo">695</span>        }<a name="line.695"></a>
+<span class="sourceLineNo">696</span>      }<a name="line.696"></a>
+<span class="sourceLineNo">697</span><a name="line.697"></a>
+<span class="sourceLineNo">698</span>      final long pushSyncId = syncId.get();<a name="line.698"></a>
+<span class="sourceLineNo">699</span>      updateStoreTracker(type, procId, subProcIds);<a name="line.699"></a>
+<span class="sourceLineNo">700</span>      slots[slotIndex++] = slot;<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      logId = flushLogId;<a name="line.701"></a>
+<span class="sourceLineNo">702</span><a name="line.702"></a>
+<span class="sourceLineNo">703</span>      // Notify that there is new data<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (slotIndex == 1) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        waitCond.signal();<a name="line.705"></a>
+<span class="sourceLineNo">706</span>      }<a name="line.706"></a>
+<span class="sourceLineNo">707</span><a name="line.707"></a>
+<span class="sourceLineNo">708</span>      // Notify that the slots are full<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      if (slotIndex == syncMaxSlot) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        waitCond.signal();<a name="line.710"></a>
+<span class="sourceLineNo">711</span>        slotCond.signal();<a name="line.711"></a>
+<span class="sourceLineNo">712</span>      }<a name="line.712"></a>
+<span class="sourceLineNo">713</span><a name="line.713"></a>
+<span class="sourceLineNo">714</span>      while (pushSyncId == syncId.get() &amp;&amp; isRunning()) {<a name="line.714"></a>
+<span class="sourceLineNo">715</span>        syncCond.await();<a name="line.715"></a>
+<span class="sourceLineNo">716</span>      }<a name="line.716"></a>
+<span class="sourceLineNo">717</span>    } catch (InterruptedException e) {<a name="line.717"></a>
+<span class="sourceLineNo">718</span>      Thread.currentThread().interrupt();<a name="line.718"></a>
+<span class="sourceLineNo">719</span>      sendAbortProcessSignal();<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      throw new RuntimeException(e);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>    } finally {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>      lock.unlock();<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      if (isSyncAborted()) {<a name="line.723"></a>
+<span class="sourceLineNo">724</span>        throw new RuntimeException("sync aborted", syncException.get());<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      }<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>    return logId;<a name="line.727"></a>
+<span class="sourceLineNo">728</span>  }<a name="line.728"></a>
+<span class="sourceLineNo">729</span><a name="line.729"></a>
+<span class="sourceLineNo">730</span>  private void updateStoreTracker(final PushType type,<a name="line.730"></a>
+<span class="sourceLineNo">731</span>      final long procId, final long[] subProcIds) {<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    switch (type) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span>      case INSERT:<a name="line.733"></a>
+<span class="sourceLineNo">734</span>        if (subProcIds == null) {<a name="line.734"></a>
+<span class="sourceLineNo">735</span>          storeTracker.insert(procId);<a name="line.735"></a>
+<span class="sourceLineNo">736</span>        } else if (procId == Procedure.NO_PROC_ID) {<a name="line.736"></a>
+<span class="sourceLineNo">737</span>          storeTracker.insert(subProcIds);<a name="line.737"></a>
+<span class="sourceLineNo">738</span>        } else {<a name="line.738"></a>
+<span class="sourceLineNo">739</span>          storeTracker.insert(procId, subProcIds);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.740"></a>
+<span class="sourceLineNo">741</span>        }<a name="line.741"></a>
+<span class="sourceLineNo">742</span>        break;<a name="line.742"></a>
+<span class="sourceLineNo">743</span>      case UPDATE:<a name="line.743"></a>
+<span class="sourceLineNo">744</span>        storeTracker.update(procId);<a name="line.744"></a>
+<span class="sourceLineNo">745</span>        holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.745"></a>
+<span class="sourceLineNo">746</span>        break;<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      case DELETE:<a name="line.747"></a>
+<span class="sourceLineNo">748</span>        if (subProcIds != null &amp;&amp; subProcIds.length &gt; 0) {<a name="line.748"></a>
+<span class="sourceLineNo">749</span>          storeTracker.delete(subProcIds);<a name="line.749"></a>
+<span class="sourceLineNo">750</span>          holdingCleanupTracker.setDeletedIfSet(subProcIds);<a name="line.750"></a>
+<span class="sourceLineNo">751</span>        } else {<a name="line.751"></a>
+<span class="sourceLineNo">752</span>          storeTracker.delete(procId);<a name="line.752"></a>
+<span class="sourceLineNo">753</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.753"></a>
+<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
+<span class="sourceLineNo">755</span>        break;<a name="line.755"></a>
+<span class="sourceLineNo">756</span>      default:<a name="line.756"></a>
+<span class="sourceLineNo">757</span>        throw new RuntimeException("invalid push type " + type);<a name="line.757"></a>
+<span class="sourceLineNo">758</span>    }<a name="line.758"></a>
+<span class="sourceLineNo">759</span>  }<a name="line.759"></a>
+<span class="sourceLineNo">760</span><a name="line.760"></a>
+<span class="sourceLineNo">761</span>  private boolean isSyncAborted() {<a name="line.761"></a>
+<span class="sourceLineNo">762</span>    return syncException.get() != null;<a name="line.762"></a>
+<span class="sourceLineNo">763</span>  }<a name="line.763"></a>
+<span class="sourceLineNo">764</span><a name="line.764"></a>
+<span class="sourceLineNo">765</span>  private void syncLoop() throws Throwable {<a name="line.765"></a>
+<span class="sourceLineNo">766</span>    long totalSyncedToStore = 0;<a name="line.766"></a>
+<span class="sourceLineNo">767</span>    inSync.set(false);<a name="line.767"></a>
+<span class="sourceLineNo">768</span>    lock.lock();<a name="line.768"></a>
+<span class="sourceLineNo">769</span>    try {<a name="line.769"></a>
+<span class="sourceLineNo">770</span>      while (isRunning()) {<a name="line.770"></a>
+<span class="sourceLineNo">771</span>        try {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>          // Wait until new data is available<a name="line.772"></a>
+<span class="sourceLineNo">773</span>          if (slotIndex == 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>            if (!loading.get()) {<a name="line.774"></a>
+<span class="sourceLineNo">775</span>              periodicRoll();<a name="line.775"></a>
 <span class="sourceLineNo">776</span>            }<a name="line.776"></a>
 <span class="sourceLineNo">777</span><a name="line.777"></a>
-<span class="sourceLineNo">778</span>            waitCond.await(getMillisToNextPeriodicRoll(), TimeUnit.MILLISECONDS);<a name="line.778"></a>
-<span class="sourceLineNo">779</span>            if (slotIndex == 0) {<a name="line.779"></a>
-<span class="sourceLineNo">780</span>              // no data.. probably a stop() or a periodic roll<a name="line.780"></a>
-<span class="sourceLineNo">781</span>              continue;<a name="line.781"></a>
-<span class="sourceLineNo">782</span>            }<a name="line.782"></a>
-<span class="sourceLineNo">783</span>          }<a name="line.783"></a>
-<span class="sourceLineNo">784</span>          // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing<a name="line.784"></a>
-<span class="sourceLineNo">785</span>          syncMaxSlot = runningProcCount;<a name="line.785"></a>
-<span class="sourceLineNo">786</span>          assert syncMaxSlot &gt; 0 : "unexpected syncMaxSlot=" + syncMaxSlot;<a name="line.786"></a>
-<span class="sourceLineNo">787</span>          final long syncWaitSt = System.currentTimeMillis();<a name="line.787"></a>
-<span class="sourceLineNo">788</span>          if (slotIndex != syncMaxSlot) {<a name="line.788"></a>
-<span class="sourceLineNo">789</span>            slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);<a name="line.789"></a>
+<span class="sourceLineNo">778</span>            if (LOG.isTraceEnabled()) {<a name="line.778"></a>
+<span class="sourceLineNo">779</span>              float rollTsSec = getMillisFromLastRoll() / 1000.0f;<a name="line.779"></a>
+<span class="sourceLineNo">780</span>              LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)",<a name="line.780"></a>
+<span class="sourceLineNo">781</span>                        StringUtils.humanSize(totalSynced.get()),<a name="line.781"></a>
+<span class="sourceLineNo">782</span>                        StringUtils.humanSize(totalSynced.get() / rollTsSec)));<a name="line.782"></a>
+<span class="sourceLineNo">783</span>            }<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>            waitCond.await(getMillisToNextPeriodicRoll(), TimeUnit.MILLISECONDS);<a name="line.785"></a>
+<span class="sourceLineNo">786</span>            if (slotIndex == 0) {<a name="line.786"></a>
+<span class="sourceLineNo">787</span>              // no data.. probably a stop() or a periodic roll<a name="line.787"></a>
+<span class="sourceLineNo">788</span>              continue;<a name="line.788"></a>
+<span class="sourceLineNo">789</span>            }<a name="line.789"></a>
 <span class="sourceLineNo">790</span>          }<a name="line.790"></a>
-<span class="sourceLineNo">791</span><a name="line.791"></a>
-<span class="sourceLineNo">792</span>          final long currentTs = System.currentTimeMillis();<a name="line.792"></a>
-<span class="sourceLineNo">793</span>          final long syncWaitMs = currentTs - syncWaitSt;<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          final float rollSec = getMillisFromLastRoll() / 1000.0f;<a name="line.794"></a>
-<span class="sourceLineNo">795</span>          final float syncedPerSec = totalSyncedToStore / rollSec;<a name="line.795"></a>
-<span class="sourceLineNo">796</span>          if (LOG.isTraceEnabled() &amp;&amp; (syncWaitMs &gt; 10 || slotIndex &lt; syncMaxSlot)) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span>            LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)",<a name="line.797"></a>
-<span class="sourceLineNo">798</span>                      StringUtils.humanTimeDiff(syncWaitMs), slotIndex,<a name="line.798"></a>
-<span class="sourceLineNo">799</span>                      StringUtils.humanSize(totalSyncedToStore),<a name="line.799"></a>
-<span class="sourceLineNo">800</span>                      StringUtils.humanSize(syncedPerSec)));<a name="line.800"></a>
-<span class="sourceLineNo">801</span>          }<a name="line.801"></a>
-<span class="sourceLineNo">802</span><a name="line.802"></a>
-<span class="sourceLineNo">803</span>          // update webui circular buffers (TODO: get rid of allocations)<a name="line.803"></a>
-<span class="sourceLineNo">804</span>          final SyncMetrics syncMetrics = new SyncMetrics();<a name="line.804"></a>
-<span class="sourceLineNo">805</span>          syncMetrics.timestamp = currentTs;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>          syncMetrics.syncWaitMs = syncWaitMs;<a name="line.806"></a>
-<span class="sourceLineNo">807</span>          syncMetrics.syncedEntries = slotIndex;<a name="line.807"></a>
-<span class="sourceLineNo">808</span>          syncMetrics.totalSyncedBytes = totalSyncedToStore;<a name="line.808"></a>
-<span class="sourceLineNo">809</span>          syncMetrics.syncedPerSec = syncedPerSec;<a name="line.809"></a>
-<span class="sourceLineNo">810</span>          syncMetricsQueue.add(syncMetrics);<a name="line.810"></a>
-<span class="sourceLineNo">811</span><a name="line.811"></a>
-<span class="sourceLineNo">812</span>          // sync<a name="line.812"></a>
-<span class="sourceLineNo">813</span>          inSync.set(true);<a name="line.813"></a>
-<span class="sourceLineNo">814</span>          long slotSize = syncSlots();<a name="line.814"></a>
-<span class="sourceLineNo">815</span>          logs.getLast().addToSize(slotSize);<a name="line.815"></a>
-<span class="sourceLineNo">816</span>          totalSyncedToStore = totalSynced.addAndGet(slotSize);<a name="line.816"></a>
-<span class="sourceLineNo">817</span>          slotIndex = 0;<a name="line.817"></a>
-<span class="sourceLineNo">818</span>          inSync.set(false);<a name="line.818"></a>
-<span class="sourceLineNo">819</span>          syncId.incrementAndGet();<a name="line.819"></a>
-<span class="sourceLineNo">820</span>        } catch (InterruptedException e) {<a name="line.820"></a>
-<span class="sourceLineNo">821</span>          Thread.currentThread().interrupt();<a name="line.821"></a>
-<span class="sourceLineNo">822</span>          syncException.compareAndSet(null, e);<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          sendAbortProcessSignal();<a name="line.823"></a>
-<span class="sourceLineNo">824</span>          throw e;<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        } catch (Throwable t) {<a name="line.825"></a>
-<span class="sourceLineNo">826</span>          syncException.compareAndSet(null, t);<a name="line.826"></a>
-<span class="sourceLineNo">827</span>          sendAbortProcessSignal();<a name="line.827"></a>
-<span class="sourceLineNo">828</span>          throw t;<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        } finally {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          syncCond.signalAll();<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span>    } finally {<a name="line.833"></a>
-<span class="sourceLineNo">834</span>      lock.unlock();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span>  }<a name="line.836"></a>
-<span class="sourceLineNo">837</span><a name="line.837"></a>
-<span class="sourceLineNo">838</span>  public ArrayList&lt;SyncMetrics&gt; getSyncMetrics() {<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    lock.lock();<a name="line.839"></a>
-<span class="sourceLineNo">840</span>    try {<a name="line.840"></a>
-<span class="sourceLineNo">841</span>      return new ArrayList&lt;&gt;(syncMetricsQueue);<a name="line.841"></a>
-<span class="sourceLineNo">842</span>    } finally {<a name="line.842"></a>
-<span class="sourceLineNo">843</span>      lock.unlock();<a name="line.843"></a>
-<span class="sourceLineNo">844</span>    }<a name="line.844"></a>
-<span class="sourceLineNo">845</span>  }<a name="line.845"></a>
-<span class="sourceLineNo">846</span><a name="line.846"></a>
-<span class="sourceLineNo">847</span>  private long syncSlots() throws Throwable {<a name="line.847"></a>
-<span class="sourceLineNo">848</span>    int retry = 0;<a name="line.848"></a>
-<span class="sourceLineNo">849</span>    int logRolled = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span>    long totalSynced = 0;<a name="line.850"></a>
-<span class="sourceLineNo">851</span>    do {<a name="line.851"></a>
-<span class="sourceLineNo">852</span>      try {<a name="line.852"></a>
-<span class="sourceLineNo">853</span>        totalSynced = syncSlots(stream, slots, 0, slotIndex);<a name="line.853"></a>
-<span class="sourceLineNo">854</span>        break;<a name="line.854"></a>
-<span class="sourceLineNo">855</span>      } catch (Throwable e) {<a name="line.855"></a>
-<span class="sourceLineNo">856</span>        LOG.warn("unable to sync slots, retry=" + retry);<a name="line.856"></a>
-<span class="sourceLineNo">857</span>        if (++retry &gt;= maxRetriesBeforeRoll) {<a name="line.857"></a>
-<span class="sourceLineNo">858</span>          if (logRolled &gt;= maxSyncFailureRoll &amp;&amp; isRunning()) {<a name="line.858"></a>
-<span class="sourceLineNo">859</span>            LOG.error("Sync slots after log roll failed, abort.", e);<a name="line.859"></a>
-<span class="sourceLineNo">860</span>            throw e;<a name="line.860"></a>
-<span class="sourceLineNo">861</span>          }<a name="line.861"></a>
-<span class="sourceLineNo">862</span><a name="line.862"></a>
-<span class="sourceLineNo">863</span>          if (!rollWriterWithRetries()) {<a name="line.863"></a>
-<span class="sourceLineNo">864</span>            throw e;<a name="line.864"></a>
-<span class="sourceLineNo">865</span>          }<a name="line.865"></a>
-<span class="sourceLineNo">866</span><a name="line.866"></a>
-<span class="sourceLineNo">867</span>          logRolled++;<a name="line.867"></a>
-<span class="sourceLineNo">868</span>          retry = 0;<a name="line.868"></a>
-<span class="sourceLineNo">869</span>        }<a name="line.869"></a>
-<span class="sourceLineNo">870</span>      }<a name="line.870"></a>
-<span class="sourceLineNo">871</span>    } while (isRunning());<a name="line.871"></a>
-<span class="sourceLineNo">872</span>    return totalSynced;<a name="line.872"></a>
-<span class="sourceLineNo">873</span>  }<a name="line.873"></a>
-<span class="sourceLineNo">874</span><a name="line.874"></a>
-<span class="sourceLineNo">875</span>  protected long syncSlots(final FSDataOutputStream stream, final ByteSlot[] slots,<a name="line.875"></a>
-<span class="sourceLineNo">876</span>      final int offset, final int count) throws IOException {<a name="line.876"></a>
-<span class="sourceLineNo">877</span>    long totalSynced = 0;<a name="line.877"></a>
-<span class="sourceLineNo">878</span>    for (int i = 0; i &lt; count; ++i) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span>      final ByteSlot data = slots[offset + i];<a name="line.879"></a>
-<span class="sourceLineNo">880</span>      data.writeTo(stream);<a name="line.880"></a>
-<span class="sourceLineNo">881</span>      totalSynced += data.size();<a name="line.881"></a>
-<span class="sourceLineNo">882</span>    }<a name="line.882"></a>
-<span class="sourceLineNo">883</span><a name="line.883"></a>
-<span class="sourceLineNo">884</span>    syncStream(stream);<a name="line.884"></a>
-<span class="sourceLineNo">885</span>    sendPostSyncSignal();<a name="line.885"></a>
-<span class="sourceLineNo">886</span><a name="line.886"></a>
-<span class="sourceLineNo">887</span>    if (LOG.isTraceEnabled()) {<a name="line.887"></a>
-<span class="sourceLineNo">888</span>      LOG.trace("Sync slots=" + count + '/' + syncMaxSlot +<a name="line.888"></a>
-<span class="sourceLineNo">889</span>                ", flushed=" + StringUtils.humanSize(totalSynced));<a name="line.889"></a>
-<span class="sourceLineNo">890</span>    }<a name="line.890"></a>
-<span class="sourceLineNo">891</span>    return totalSynced;<a name="line.891"></a>
-<span class="sourceLineNo">892</span>  }<a name="line.892"></a>
+<span class="sourceLineNo">791</span>          // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing<a name="line.791"></a>
+<span class="sourceLineNo">792</span>          syncMaxSlot = runningProcCount;<a name="line.792"></a>
+<span class="sourceLineNo">793</span>          assert syncMaxSlot &gt; 0 : "unexpected syncMaxSlot=" + syncMaxSlot;<a name="line.793"></a>
+<span class="sourceLineNo">794</span>          final long syncWaitSt = System.currentTimeMillis();<a name="line.794"></a>
+<span class="sourceLineNo">795</span>          if (slotIndex != syncMaxSlot) {<a name="line.795"></a>
+<span class="sourceLineNo">796</span>            slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);<a name="line.796"></a>
+<span class="sourceLineNo">797</span>          }<a name="line.797"></a>
+<span class="sourceLineNo">798</span><a name="line.798"></a>
+<span class="sourceLineNo">799</span>          final long currentTs = System.currentTimeMillis();<a name="line.799"></a>
+<span class="sourceLineNo">800</span>          final long syncWaitMs = currentTs - syncWaitSt;<a name="line.800"></a>
+<span class="sourceLineNo">801</span>          final float rollSec = getMillisFromLastRoll() / 1000.0f;<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          final float syncedPerSec = totalSyncedToStore / rollSec;<a name="line.802"></a>
+<span class="sourceLineNo">803</span>          if (LOG.isTraceEnabled() &amp;&amp; (syncWaitMs &gt; 10 || slotIndex &lt; syncMaxSlot)) {<a name="line.803"></a>
+<span class="sourceLineNo">804</span>            LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)",<a name="line.804"></a>
+<span class="sourceLineNo">805</span>                      StringUtils.humanTimeDiff(syncWaitMs), slotIndex,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>                      StringUtils.humanSize(totalSyncedToStore),<a name="line.806"></a>
+<span class="sourceLineNo">807</span>                      StringUtils.humanSize(syncedPerSec)));<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          }<a name="line.808"></a>
+<span class="sourceLineNo">809</span><a name="line.809"></a>
+<span class="sourceLineNo">810</span>          // update webui circular buffers (TODO: get rid of allocations)<a name="line.810"></a>
+<span class="sourceLineNo">811</span>          final SyncMetrics syncMetrics = new SyncMetrics();<a name="line.811"></a>
+<span class="sourceLineNo">812</span>          syncMetrics.timestamp = currentTs;<a name="line.812"></a>
+<span class="sourceLineNo">813</span>          syncMetrics.syncWaitMs = syncWaitMs;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>          syncMetrics.syncedEntries = slotIndex;<a name="line.814"></a>
+<span class="sourceLineNo">815</span>          syncMetrics.totalSyncedBytes = totalSyncedToStore;<a name="line.815"></a>
+<span class="sourceLineNo">816</span>          syncMetrics.syncedPerSec = syncedPerSec;<a name="line.816"></a>
+<span class="sourceLineNo">817</span>          syncMetricsQueue.add(syncMetrics);<a name="line.817"></a>
+<span class="sourceLineNo">818</span><a name="line.818"></a>
+<span class="sourceLineNo">819</span>          // sync<a name="line.819"></a>
+<span class="sourceLineNo">820</span>          inSync.set(true);<a name="line.820"></a>
+<span class="sourceLineNo">821</span>          long slotSize = syncSlots();<a name="line.821"></a>
+<span class="sourceLineNo">822</span>          logs.getLast().addToSize(slotSize);<a name="line.822"></a>
+<span class="sourceLineNo">823</span>          totalSyncedToStore = totalSynced.addAndGet(slotSize);<a name="line.823"></a>
+<span class="sourceLineNo">824</span>          slotIndex = 0;<a name="line.824"></a>
+<span class="sourceLineNo">825</span>          inSync.set(false);<a name="line.825"></a>
+<span class="sourceLineNo">826</span>          syncId.incrementAndGet();<a name="line.826"></a>
+<span class="sourceLineNo">827</span>        } catch (InterruptedException e) {<a name="line.827"></a>
+<span class="sourceLineNo">828</span>          Thread.currentThread().interrupt();<a name="line.828"></a>
+<span class="sourceLineNo">829</span>          syncException.compareAndSet(null, e);<a name="line.829"></a>
+<span class="sourceLineNo">830</span>          sendAbortProcessSignal();<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          throw e;<a name="line.831"></a>
+<span class="sourceLineNo">832</span>        } catch (Throwable t) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>          syncException.compareAndSet(null, t);<a name="line.833"></a>
+<span class="sourceLineNo">834</span>          sendAbortProcessSignal();<a name="line.834"></a>
+<span class="sourceLineNo">835</span>          throw t;<a name="line.835"></a>
+<span class="sourceLineNo">836</span>        } finally {<a name="line.836"></a>
+<span class="sourceLineNo">837</span>          syncCond.signalAll();<a name="line.837"></a>
+<span class="sourceLineNo">838</span>        }<a name="line.838"></a>
+<span class="sourceLineNo">839</span>      }<a name="line.839"></a>
+<span class="sourceLineNo">840</span>    } finally {<a name="line.840"></a>
+<span class="sourceLineNo">841</span>      lock.unlock();<a name="line.841"></a>
+<span class="sourceLineNo">842</span>    }<a name="line.842"></a>
+<span class="sourceLineNo">843</span>  }<a name="line.843"></a>
+<span class="sourceLineNo">844</span><a name="line.844"></a>
+<span class="sourceLineNo">845</span>  public ArrayList&lt;SyncMetrics&gt; getSyncMetrics() {<a name="line.845"></a>
+<span class="sourceLineNo">846</span>    lock.lock();<a name="line.846"></a>
+<span class="sourceLineNo">847</span>    try {<a name="line.847"></a>
+<span class="sourceLineNo">848</span>      return new ArrayList&lt;&gt;(syncMetricsQueue);<a name="line.848"></a>
+<span class="sourceLineNo">849</span>    } finally {<a name="line.849"></a>
+<span class="sourceLineNo">850</span>      lock.unlock();<a name="line.850"></a>
+<span class="sourceLineNo">851</span>    }<a name="line.851"></a>
+<span class="sourceLineNo">852</span>  }<a name="line.852"></a>
+<span class="sourceLineNo">853</span><a name="line.853"></a>
+<span class="sourceLineNo">854</span>  private long syncSlots() throws Throwable {<a name="line.854"></a>
+<span class="sourceLineNo">855</span>    int retry = 0;<a name="line.855"></a>
+<span class="sourceLineNo">856</span>    int logRolled = 0;<a name="line.856"></a>
+<span class="sourceLineNo">857</span>    long totalSynced = 0;<a name="line.857"></a>
+<span class="sourceLineNo">858</span>    do {<a name="line.858"></a>
+<span class="sourceLineNo">859</span>      try {<a name="line.859"></a>
+<span class="sourceLineNo">860</span>        totalSynced = syncSlots(stream, slots, 0, slotIndex);<a name="line.860"></a>
+<span class="sourceLineNo">861</span>        break;<a name="line.861"></a>
+<span class="sourceLineNo">862</span>      } catch (Throwable e) {<a name="line.862"></a>
+<span class="sourceLineNo">863</span>        LOG.warn("unable to sync slots, retry=" + retry);<a name="line.863"></a>
+<span class="sourceLineNo">864</span>        if (++retry &gt;= maxRetriesBeforeRoll) {<a name="line.864"></a>
+<span class="sourceLineNo">865</span>          if (logRolled &gt;= maxSyncFailureRoll &amp;&amp; isRunning()) {<a name="line.865"></a>
+<span class="sourceLineNo">866</span>            LOG.error("Sync slots after log roll failed, abort.", e);<a name="line.866"></a>
+<span class="sourceLineNo">867</span>            throw e;<a name="line.867"></a>
+<span class="sourceLineNo">868</span>          }<a name="line.868"></a>
+<span class="sourceLineNo">869</span><a name="line.869"></a>
+<span class="sourceLineNo">870</span>          if (!rollWriterWithRetries()) {<a name="line.870"></a>
+<span class="sourceLineNo">871</span>            throw e;<a name="line.871"></a>
+<span class="sourceLineNo">872</span>          }<a name="line.872"></a>
+<span class="sourceLineNo">873</span><a name="line.873"></a>
+<span class="sourceLineNo">874</span>          logRolled++;<a name="line.874"></a>
+<span class="sourceLineNo">875</span>          retry = 0;<a name="line.875"></a>
+<span class="sourceLineNo">876</span>        }<a name="line.876"></a>
+<span class="sourceLineNo">877</span>      }<a name="line.877"></a>
+<span class="sourceLineNo">878</span>    } while (isRunning());<a name="line.878"></a>
+<span class="sourceLineNo">879</span>    return totalSynced;<a name="line.879"></a>
+<span class="sourceLineNo">880</span>  }<a name="line.880"></a>
+<span class="sourceLineNo">881</span><a name="line.881"></a>
+<span class="sourceLineNo">882</span>  protected long syncSlots(final FSDataOutputStream stream, final ByteSlot[] slots,<a name="line.882"></a>
+<span class="sourceLineNo">883</span>      final int offset, final int count) throws IOException {<a name="line.883"></a>
+<span class="sourceLineNo">884</span>    long totalSynced = 0;<a name="line.884"></a>
+<span class="sourceLineNo">885</span>    for (int i = 0; i &lt; count; ++i) {<a name="line.885"></a>
+<span class="sourceLineNo">886</span>      final ByteSlot data = slots[offset + i];<a name="line.886"></a>
+<span class="sourceLineNo">887</span>      data.writeTo(stream);<a name="line.887"></a>
+<span class="sourceLineNo">888</span>      totalSynced += data.size();<a name="line.888"></a>
+<span class="sourceLineNo">889</span>    }<a name="line.889"></a>
+<span class="sourceLineNo">890</span><a name="line.890"></a>
+<span class="sourceLineNo">891</span>    syncStream(stream);<a name="line.891"></a>
+<span class="sourceLineNo">892</span>    sendPostSyncSignal();<a name="line.892"></a>
 <span class="sourceLineNo">893</span><a name="line.893"></a>
-<span class="sourceLineNo">894</span>  protected void syncStream(final FSDataOutputStream stream) throws IOException {<a name="line.894"></a>
-<span class="sourceLineNo">895</span>    if (useHsync) {<a name="line.895"></a>
-<span class="sourceLineNo">896</span>      stream.hsync();<a name="line.896"></a>
-<span class="sourceLineNo">897</span>    } else {<a name="line.897"></a>
-<span class="sourceLineNo">898</span>      stream.hflush();<a name="line.898"></a>
-<span class="sourceLineNo">899</span>    }<a name="line.899"></a>
-<span class="sourceLineNo">900</span>  }<a name="line.900"></a>
-<span class="sourceLineNo">901</span><a name="line.901"></a>
-<span class="sourceLineNo">902</span>  private boolean rollWriterWithRetries() {<a name="line.902"></a>
-<span class="sourceLineNo">903</span>    for (int i = 0; i &lt; rollRetries &amp;&amp; isRunning(); ++i) {<a name="line.903"></a>
-<span class="sourceLineNo">904</span>      if (i &gt; 0) Threads.sleepWithoutInterrupt(waitBeforeRoll * i);<a name="line.904"></a>
-<span class="sourceLineNo">905</span><a name="line.905"></a>
-<span class="sourceLineNo">906</span>      try {<a name="line.906"></a>
-<span class="sourceLineNo">907</span>        if (rollWriter()) {<a name="line.907"></a>
-<span class="sourceLineNo">908</span>          return true;<a name="line.908"></a>
-<span class="sourceLineNo">909</span>        }<a name="line.909"></a>
-<span class="sourceLineNo">910</span>      } catch (IOException e) {<a name="line.910"></a>
-<span class="sourceLineNo">911</span>        LOG.warn("Unable to roll the log, attempt=" + (i + 1), e);<a name="line.911"></a>
-<span class="sourceLineNo">912</span>      }<a name="line.912"></a>
-<span class="sourceLineNo">913</span>    }<a name="line.913"></a>
-<span class="sourceLineNo">914</span>    LOG.error(HBaseMarkers.FATAL, "Unable to roll the log");<a name="line.914"></a>
-<span class="sourceLineNo">915</span>    return false;<a name="line.915"></a>
-<span class="sourceLineNo">916</span>  }<a name="line.916"></a>
-<span class="sourceLineNo">917</span><a name="line.917"></a>
-<span class="sourceLineNo">918</span>  private boolean tryRollWriter() {<a name="line.918"></a>
-<span class="sourceLineNo">919</span>    try {<a name="line.919"></a>
-<span class="sourceLineNo">920</span>      return rollWriter();<a name="line.920"></a>
-<span class="sourceLineNo">921</span>    } catch (IOException e) {<a name="line.921"></a>
-<span class="sourceLineNo">922</span>      LOG.warn("Unable to roll the log", e);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> 

<TRUNCATED>

[22/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/HMaster.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/HMaster.html b/devapidocs/org/apache/hadoop/hbase/master/HMaster.html
index 2815504..4e6d5a3 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/HMaster.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/HMaster.html
@@ -2456,7 +2456,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>updateConfigurationForQuotasObserver</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1145">updateConfigurationForQuotasObserver</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1148">updateConfigurationForQuotasObserver</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 <div class="block">Adds the <code>MasterQuotasObserver</code> to the list of configured Master observers to
  automatically remove quotas for a table when that table is deleted.</div>
 </li>
@@ -2467,7 +2467,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>initMobCleaner</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1162">initMobCleaner</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1165">initMobCleaner</a>()</pre>
 </li>
 </ul>
 <a name="createMetaBootstrap--">
@@ -2476,7 +2476,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>createMetaBootstrap</h4>
-<pre>protected&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterMetaBootstrap.html" title="class in org.apache.hadoop.hbase.master">MasterMetaBootstrap</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1187">createMetaBootstrap</a>()</pre>
+<pre>protected&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterMetaBootstrap.html" title="class in org.apache.hadoop.hbase.master">MasterMetaBootstrap</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1190">createMetaBootstrap</a>()</pre>
 <div class="block"><p>
  Create a <a href="../../../../../org/apache/hadoop/hbase/master/MasterMetaBootstrap.html" title="class in org.apache.hadoop.hbase.master"><code>MasterMetaBootstrap</code></a> instance.
  </p>
@@ -2491,7 +2491,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>createServerManager</h4>
-<pre>protected&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html" title="class in org.apache.hadoop.hbase.master">ServerManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1202">createServerManager</a>(<a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;master)
+<pre>protected&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html" title="class in org.apache.hadoop.hbase.master">ServerManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1205">createServerManager</a>(<a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;master)
                                      throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><p>
  Create a <a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html" title="class in org.apache.hadoop.hbase.master"><code>ServerManager</code></a> instance.
@@ -2511,7 +2511,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>waitForRegionServers</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1209">waitForRegionServers</a>(<a href="../../../../../org/apache/hadoop/hbase/monitoring/MonitoredTask.html" title="interface in org.apache.hadoop.hbase.monitoring">MonitoredTask</a>&nbsp;status)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1212">waitForRegionServers</a>(<a href="../../../../../org/apache/hadoop/hbase/monitoring/MonitoredTask.html" title="interface in org.apache.hadoop.hbase.monitoring">MonitoredTask</a>&nbsp;status)
                            throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
                                   <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a></pre>
 <dl>
@@ -2527,7 +2527,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>initClusterSchemaService</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1216">initClusterSchemaService</a>()
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1219">initClusterSchemaService</a>()
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
                                         <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a></pre>
 <dl>
@@ -2543,7 +2543,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>initQuotaManager</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1228">initQuotaManager</a>()
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1231">initQuotaManager</a>()
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2557,7 +2557,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>createQuotaSnapshotNotifier</h4>
-<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/quotas/SpaceQuotaSnapshotNotifier.html" title="interface in org.apache.hadoop.hbase.quotas">SpaceQuotaSnapshotNotifier</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1234">createQuotaSnapshotNotifier</a>()</pre>
+<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/quotas/SpaceQuotaSnapshotNotifier.html" title="interface in org.apache.hadoop.hbase.quotas">SpaceQuotaSnapshotNotifier</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1237">createQuotaSnapshotNotifier</a>()</pre>
 </li>
 </ul>
 <a name="isCatalogJanitorEnabled--">
@@ -2566,7 +2566,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isCatalogJanitorEnabled</h4>
-<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1240">isCatalogJanitorEnabled</a>()</pre>
+<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1243">isCatalogJanitorEnabled</a>()</pre>
 </li>
 </ul>
 <a name="isCleanerChoreEnabled--">
@@ -2575,7 +2575,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isCleanerChoreEnabled</h4>
-<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1245">isCleanerChoreEnabled</a>()</pre>
+<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1248">isCleanerChoreEnabled</a>()</pre>
 </li>
 </ul>
 <a name="getTableDescriptors--">
@@ -2584,7 +2584,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getTableDescriptors</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/TableDescriptors.html" title="interface in org.apache.hadoop.hbase">TableDescriptors</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1260">getTableDescriptors</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/TableDescriptors.html" title="interface in org.apache.hadoop.hbase">TableDescriptors</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1263">getTableDescriptors</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getTableDescriptors--">getTableDescriptors</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -2601,7 +2601,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getServerManager</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html" title="class in org.apache.hadoop.hbase.master">ServerManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1265">getServerManager</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html" title="class in org.apache.hadoop.hbase.master">ServerManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1268">getServerManager</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getServerManager--">getServerManager</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -2616,7 +2616,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterFileSystem</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterFileSystem.html" title="class in org.apache.hadoop.hbase.master">MasterFileSystem</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1270">getMasterFileSystem</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterFileSystem.html" title="class in org.apache.hadoop.hbase.master">MasterFileSystem</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1273">getMasterFileSystem</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getMasterFileSystem--">getMasterFileSystem</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -2631,7 +2631,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterWalManager</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterWalManager.html" title="class in org.apache.hadoop.hbase.master">MasterWalManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1275">getMasterWalManager</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterWalManager.html" title="class in org.apache.hadoop.hbase.master">MasterWalManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1278">getMasterWalManager</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getMasterWalManager--">getMasterWalManager</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -2646,7 +2646,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getTableStateManager</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/TableStateManager.html" title="class in org.apache.hadoop.hbase.master">TableStateManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1280">getTableStateManager</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/TableStateManager.html" title="class in org.apache.hadoop.hbase.master">TableStateManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1283">getTableStateManager</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getTableStateManager--">getTableStateManager</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -2661,7 +2661,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>startServiceThreads</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1291">startServiceThreads</a>()
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1294">startServiceThreads</a>()
                           throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2675,7 +2675,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>stopServiceThreads</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1341">stopServiceThreads</a>()</pre>
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1344">stopServiceThreads</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegionServer.html#stopServiceThreads--">HRegionServer</a></code></span></div>
 <div class="block">Wait on all threads to finish. Presumption is that all closes and stops
  have already been called.</div>
@@ -2691,7 +2691,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>createProcedureExecutor</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1389">createProcedureExecutor</a>()
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1392">createProcedureExecutor</a>()
                               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2705,7 +2705,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>startProcedureExecutor</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1412">startProcedureExecutor</a>()
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1415">startProcedureExecutor</a>()
                              throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2719,7 +2719,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>stopProcedureExecutor</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1416">stopProcedureExecutor</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1419">stopProcedureExecutor</a>()</pre>
 </li>
 </ul>
 <a name="stopChores--">
@@ -2728,7 +2728,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>stopChores</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1431">stopChores</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1434">stopChores</a>()</pre>
 </li>
 </ul>
 <a name="getRemoteInetAddress-int-long-">
@@ -2737,7 +2737,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getRemoteInetAddress</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/net/InetAddress.html?is-external=true" title="class or interface in java.net">InetAddress</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1451">getRemoteInetAddress</a>(int&nbsp;port,
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/net/InetAddress.html?is-external=true" title="class or interface in java.net">InetAddress</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1454">getRemoteInetAddress</a>(int&nbsp;port,
                                  long&nbsp;serverStartCode)
                           throws <a href="https://docs.oracle.com/javase/8/docs/api/java/net/UnknownHostException.html?is-external=true" title="class or interface in java.net">UnknownHostException</a></pre>
 <dl>
@@ -2754,7 +2754,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMaxBalancingTime</h4>
-<pre>private&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1471">getMaxBalancingTime</a>()</pre>
+<pre>private&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1474">getMaxBalancingTime</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>Maximum time we should run balancer for</dd>
@@ -2767,7 +2767,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMaxRegionsInTransition</h4>
-<pre>private&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1484">getMaxRegionsInTransition</a>()</pre>
+<pre>private&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1487">getMaxRegionsInTransition</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>Maximum number of regions in transition</dd>
@@ -2780,7 +2780,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>balanceThrottling</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1496">balanceThrottling</a>(long&nbsp;nextBalanceStartTime,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1499">balanceThrottling</a>(long&nbsp;nextBalanceStartTime,
                                int&nbsp;maxRegionsInTransition,
                                long&nbsp;cutoffTime)</pre>
 <div class="block">It first sleep to the next balance plan start time. Meanwhile, throttling by the max
@@ -2799,7 +2799,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>balance</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1527">balance</a>()
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1530">balance</a>()
                 throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2813,7 +2813,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>balance</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1531">balance</a>(boolean&nbsp;force)
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1534">balance</a>(boolean&nbsp;force)
                 throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2827,7 +2827,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getRegionNormalizer</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.html" title="interface in org.apache.hadoop.hbase.master.normalizer">RegionNormalizer</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1646">getRegionNormalizer</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.html" title="interface in org.apache.hadoop.hbase.master.normalizer">RegionNormalizer</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1649">getRegionNormalizer</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getRegionNormalizer--">getRegionNormalizer</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -2842,7 +2842,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>normalizeRegions</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1657">normalizeRegions</a>()
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1660">normalizeRegions</a>()
                          throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Perform normalization of cluster (invoked by <a href="../../../../../org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.html" title="class in org.apache.hadoop.hbase.master.normalizer"><code>RegionNormalizerChore</code></a>).</div>
 <dl>
@@ -2861,7 +2861,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClientIdAuditPrefix</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1715">getClientIdAuditPrefix</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1718">getClientIdAuditPrefix</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getClientIdAuditPrefix--">getClientIdAuditPrefix</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -2876,7 +2876,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>setCatalogJanitorEnabled</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1726">setCatalogJanitorEnabled</a>(boolean&nbsp;b)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1729">setCatalogJanitorEnabled</a>(boolean&nbsp;b)</pre>
 <div class="block">Switch for the background CatalogJanitor thread.
  Used for testing.  The thread will continue to run.  It will just be a noop
  if disabled.</div>
@@ -2892,7 +2892,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>mergeRegions</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1731">mergeRegions</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>[]&nbsp;regionsToMerge,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1734">mergeRegions</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>[]&nbsp;regionsToMerge,
                          boolean&nbsp;forcible,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
@@ -2920,7 +2920,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>splitRegion</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1779">splitRegion</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;regionInfo,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1782">splitRegion</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;regionInfo,
                         byte[]&nbsp;splitRow,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
@@ -2948,7 +2948,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>move</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1805">move</a>(byte[]&nbsp;encodedRegionName,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1808">move</a>(byte[]&nbsp;encodedRegionName,
                  byte[]&nbsp;destServerName)
           throws <a href="../../../../../org/apache/hadoop/hbase/HBaseIOException.html" title="class in org.apache.hadoop.hbase">HBaseIOException</a></pre>
 <dl>
@@ -2963,7 +2963,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>createTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1895">createTable</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;tableDescriptor,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1898">createTable</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;tableDescriptor,
                         byte[][]&nbsp;splitKeys,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
@@ -2988,7 +2988,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>createSystemTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1937">createSystemTable</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;tableDescriptor)
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1940">createSystemTable</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;tableDescriptor)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#createSystemTable-org.apache.hadoop.hbase.client.TableDescriptor-">MasterServices</a></code></span></div>
 <div class="block">Create a system table using the given table definition.</div>
@@ -3009,7 +3009,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>sanityCheckTableDescriptor</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1965">sanityCheckTableDescriptor</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1968">sanityCheckTableDescriptor</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Checks whether the table conforms to some sane limits, and configured
  values (compression, etc) work. Throws an exception if something is wrong.</div>
@@ -3025,7 +3025,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkReplicationScope</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2080">checkReplicationScope</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2083">checkReplicationScope</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3039,7 +3039,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkCompactionPolicy</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2092">checkCompactionPolicy</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2095">checkCompactionPolicy</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                                    <a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -3054,7 +3054,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>warnOrThrowExceptionForFailure</h4>
-<pre>private static&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2150">warnOrThrowExceptionForFailure</a>(boolean&nbsp;logWarn,
+<pre>private static&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2153">warnOrThrowExceptionForFailure</a>(boolean&nbsp;logWarn,
                                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;confKey,
                                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;message,
                                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a>&nbsp;cause)
@@ -3071,7 +3071,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>startActiveMasterManager</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2159">startActiveMasterManager</a>(int&nbsp;infoPort)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2162">startActiveMasterManager</a>(int&nbsp;infoPort)
                                throws org.apache.zookeeper.KeeperException</pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3085,7 +3085,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkCompression</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2212">checkCompression</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2215">checkCompression</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3099,7 +3099,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkCompression</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2220">checkCompression</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2223">checkCompression</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3113,7 +3113,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkEncryption</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2227">checkEncryption</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2230">checkEncryption</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                              <a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                       throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -3128,7 +3128,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkEncryption</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2235">checkEncryption</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2238">checkEncryption</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                              <a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
                       throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -3143,7 +3143,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkClassLoading</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2241">checkClassLoading</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2244">checkClassLoading</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                                <a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -3158,7 +3158,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isCatalogTable</h4>
-<pre>private static&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2247">isCatalogTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)</pre>
+<pre>private static&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2250">isCatalogTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)</pre>
 </li>
 </ul>
 <a name="deleteTable-org.apache.hadoop.hbase.TableName-long-long-">
@@ -3167,7 +3167,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>deleteTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2252">deleteTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2255">deleteTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -3189,7 +3189,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>truncateTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2286">truncateTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2289">truncateTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                           boolean&nbsp;preserveSplits,
                           long&nbsp;nonceGroup,
                           long&nbsp;nonce)
@@ -3213,7 +3213,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>addColumn</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2316">addColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2319">addColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                       <a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;column,
                       long&nbsp;nonceGroup,
                       long&nbsp;nonce)
@@ -3237,7 +3237,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>modifyColumn</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2344">modifyColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2347">modifyColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                          <a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;descriptor,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
@@ -3261,7 +3261,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>deleteColumn</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2364">deleteColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2367">deleteColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                          byte[]&nbsp;columnName,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
@@ -3285,7 +3285,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>enableTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2389">enableTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2392">enableTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -3307,7 +3307,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>disableTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2442">disableTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2445">disableTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -3329,7 +3329,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getTableRegionForRow</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2484">getTableRegionForRow</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre><a href="../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2487">getTableRegionForRow</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                                                  byte[]&nbsp;rowKey)
                                           throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Return the region and current deployment for the region containing
@@ -3348,7 +3348,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>modifyTable</h4>
-<pre>private&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2510">modifyTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>private&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2513">modifyTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                          <a href="../../../../../org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html" title="interface in org.apache.hadoop.hbase.master">HMaster.TableDescriptorGetter</a>&nbsp;newDescriptorGetter,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
@@ -3365,7 +3365,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>modifyTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2548">modifyTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2551">modifyTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                         <a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;newDescriptor,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
@@ -3389,7 +3389,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>restoreSnapshot</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2560">restoreSnapshot</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription&nbsp;snapshotDesc,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2563">restoreSnapshot</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription&nbsp;snapshotDesc,
                             long&nbsp;nonceGroup,
                             long&nbsp;nonce,
                             boolean&nbsp;restoreAcl)
@@ -3406,7 +3406,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkTableExists</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2584">checkTableExists</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2587">checkTableExists</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
                               <a href="../../../../../org/apache/hadoop/hbase/TableNotFoundException.html" title="class in org.apache.hadoop.hbase">TableNotFoundException</a></pre>
 <dl>
@@ -3422,7 +3422,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkTableModifiable</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2592">checkTableModifiable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2595">checkTableModifiable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
                           throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
                                  <a href="../../../../../org/apache/hadoop/hbase/TableNotFoundException.html" title="class in org.apache.hadoop.hbase">TableNotFoundException</a>,
                                  <a href="../../../../../org/apache/hadoop/hbase/TableNotDisabledException.html" title="class in org.apache.hadoop.hbase">TableNotDisabledException</a></pre>
@@ -3446,7 +3446,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClusterMetricsWithoutCoprocessor</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2604">getClusterMetricsWithoutCoprocessor</a>()
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2607">getClusterMetricsWithoutCoprocessor</a>()
                                                    throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/InterruptedIOException.html?is-external=true" title="class or interface in java.io">InterruptedIOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3460,7 +3460,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClusterMetricsWithoutCoprocessor</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2608">getClusterMetricsWithoutCoprocessor</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/EnumSet.html?is-external=true" title="class or interface in java.util">EnumSet</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase">ClusterMetrics.Option</a>&gt;&nbsp;options)
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2611">getClusterMetricsWithoutCoprocessor</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/EnumSet.html?is-external=true" title="class or interface in java.util">EnumSet</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase">ClusterMetrics.Option</a>&gt;&nbsp;options)
                                                    throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/InterruptedIOException.html?is-external=true" title="class or interface in java.io">InterruptedIOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3474,7 +3474,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClusterMetrics</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2670">getClusterMetrics</a>()
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2673">getClusterMetrics</a>()
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
@@ -3490,7 +3490,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClusterMetrics</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2674">getClusterMetrics</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/EnumSet.html?is-external=true" title="class or interface in java.util">EnumSet</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase">ClusterMetrics.Option</a>&gt;&nbsp;options)
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2677">getClusterMetrics</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/EnumSet.html?is-external=true" title="class or interface in java.util">EnumSet</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase">ClusterMetrics.Option</a>&gt;&nbsp;options)
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3504,7 +3504,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getBackupMasters</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2685">getBackupMasters</a>()
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2688">getBackupMasters</a>()
                                    throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/InterruptedIOException.html?is-external=true" title="class or interface in java.io">InterruptedIOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3518,7 +3518,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getLoadedCoprocessors</h4>
-<pre>public static&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2738">getLoadedCoprocessors</a>()</pre>
+<pre>public static&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2741">getLoadedCoprocessors</a>()</pre>
 <div class="block">The set of loaded coprocessors is stored in a static set. Since it's
  statically allocated, it does not require that HMaster's cpHost be
  initialized prior to accessing it.</div>
@@ -3534,7 +3534,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterStartTime</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2745">getMasterStartTime</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2748">getMasterStartTime</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>timestamp in millis when HMaster was started.</dd>
@@ -3547,7 +3547,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterActiveTime</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2752">getMasterActiveTime</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2755">getMasterActiveTime</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>timestamp in millis when HMaster became the active master.</dd>
@@ -3560,7 +3560,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterFinishedInitializationTime</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2759">getMasterFinishedInitializationTime</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2762">getMasterFinishedInitializationTime</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>timestamp in millis when HMaster finished becoming the active master</dd>
@@ -3573,7 +3573,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getNumWALFiles</h4>
-<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2763">getNumWALFiles</a>()</pre>
+<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2766">getNumWALFiles</a>()</pre>
 </li>
 </ul>
 <a name="getWalProcedureStore--">
@@ -3582,7 +3582,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getWalProcedureStore</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2767">getWalProcedureStore</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2770">getWalProcedureStore</a>()</pre>
 </li>
 </ul>
 <a name="getRegionServerInfoPort-org.apache.hadoop.hbase.ServerName-">
@@ -3591,7 +3591,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getRegionServerInfoPort</h4>
-<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2771">getRegionServerInfoPort</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;sn)</pre>
+<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2774">getRegionServerInfoPort</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;sn)</pre>
 </li>
 </ul>
 <a name="getRegionServerVersion-org.apache.hadoop.hbase.ServerName-">
@@ -3600,7 +3600,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getRegionServerVersion</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2778">getRegionServerVersion</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;sn)</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2781">getRegionServerVersion</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;sn)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getRegionServerVersion-org.apache.hadoop.hbase.ServerName-">getRegionServerVersion</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3613,7 +3613,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkIfShouldMoveSystemRegionAsync</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2785">checkIfShouldMoveSystemRegionAsync</a>()</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2788">checkIfShouldMoveSystemRegionAsync</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#checkIfShouldMoveSystemRegionAsync--">MasterServices</a></code></span></div>
 <div class="block">Called when a new RegionServer is added to the cluster.
  Checks if new server has a newer version than any existing server and will move system tables
@@ -3630,7 +3630,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterCoprocessors</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2792">getMasterCoprocessors</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2795">getMasterCoprocessors</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>array of coprocessor SimpleNames.</dd>
@@ -3643,7 +3643,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>abort</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2798">abort</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;reason,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2801">abort</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;reason,
                   <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a>&nbsp;cause)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegionServer.html#abort-java.lang.String-java.lang.Throwable-">HRegionServer</a></code></span></div>
 <div class="block">Cause the server to exit without closing the regions it is serving, the log
@@ -3666,7 +3666,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getZooKeeper</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2822">getZooKeeper</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2825">getZooKeeper</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/Server.html#getZooKeeper--">Server</a></code></span></div>
 <div class="block">Gets the ZooKeeper instance for this server.</div>
 <dl>
@@ -3683,7 +3683,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterCoprocessorHost</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterCoprocessorHost.html" title="class in org.apache.hadoop.hbase.master">MasterCoprocessorHost</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2827">getMasterCoprocessorHost</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterCoprocessorHost.html" title="class in org.apache.hadoop.hbase.master">MasterCoprocessorHost</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2830">getMasterCoprocessorHost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getMasterCoprocessorHost--">getMasterCoprocessorHost</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3698,7 +3698,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterQuotaManager</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/quotas/MasterQuotaManager.html" title="class in org.apache.hadoop.hbase.quotas">MasterQuotaManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2832">getMasterQuotaManager</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/quotas/MasterQuotaManager.html" title="class in org.apache.hadoop.hbase.quotas">MasterQuotaManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2835">getMasterQuotaManager</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getMasterQuotaManager--">getMasterQuotaManager</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3713,7 +3713,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterProcedureExecutor</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.html" title="class in org.apache.hadoop.hbase.master.procedure">MasterProcedureEnv</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2837">getMasterProcedureExecutor</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.html" title="class in org.apache.hadoop.hbase.master.procedure">MasterProcedureEnv</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2840">getMasterProcedureExecutor</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getMasterProcedureExecutor--">getMasterProcedureExecutor</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3728,7 +3728,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getServerName</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2842">getServerName</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2845">getServerName</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/Server.html#getServerName--">getServerName</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/Server.html" title="interface in org.apache.hadoop.hbase">Server</a></code></dd>
@@ -3745,7 +3745,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getAssignmentManager</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/assignment/AssignmentManager.html" title="class in org.apache.hadoop.hbase.master.assignment">AssignmentManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2847">getAssignmentManager</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/assignment/AssignmentManager.html" title="class in org.apache.hadoop.hbase.master.assignment">AssignmentManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2850">getAssignmentManager</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getAssignmentManager--">getAssignmentManager</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3760,7 +3760,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getCatalogJanitor</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/CatalogJanitor.html" title="class in org.apache.hadoop.hbase.master">CatalogJanitor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2852">getCatalogJanitor</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/CatalogJanitor.html" title="class in org.apache.hadoop.hbase.master">CatalogJanitor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2855">getCatalogJanitor</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getCatalogJanitor--">getCatalogJanitor</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3775,7 +3775,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getRegionServerFatalLogBuffer</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.html" title="class in org.apache.hadoop.hbase.monitoring">MemoryBoundedLogMessageBuffer</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2856">getRegionServerFatalLogBuffer</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.html" title="class in org.apache.hadoop.hbase.monitoring">MemoryBoundedLogMessageBuffer</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2859">getRegionServerFatalLogBuffer</a>()</pre>
 </li>
 </ul>
 <a name="shutdown--">
@@ -3784,7 +3784,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>shutdown</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2864">shutdown</a>()
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2867">shutdown</a>()
               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Shutdown the cluster.
  Master runs a coordinated stop of all RegionServers and then itself.</div>
@@ -3800,7 +3800,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>stopMaster</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2896">stopMaster</a>()
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2899">stopMaster</a>()
                 throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3814,7 +3814,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>stop</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2904">stop</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;msg)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2907">stop</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;msg)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/Stoppable.html#stop-java.lang.String-">Stoppable</a></code></span></div>
 <div class="block">Stop this service.
  Implementers should favor logging errors over throwing RuntimeExceptions.</div>
@@ -3834,7 +3834,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkServiceStarted</h4>
-<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2913">checkServiceStarted</a>()
+<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2916">checkServiceStarted</a>()
                   throws <a href="../../../../../org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.html" title="class in org.apache.hadoop.hbase.ipc">ServerNotRunningYetException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3848,7 +3848,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkInitialized</h4>
-<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2925">checkInitialized</a>()
+<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2928">checkInitialized</a>()
                throws <a href="../../../../../org/apache/hadoop/hbase/PleaseHoldException.html" title="class in org.apache.hadoop.hbase">PleaseHoldException</a>,
                       <a href="../../../../../org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.html" title="class in org.apache.hadoop.hbase.ipc">ServerNotRunningYetException</a>,
                       <a href="../../../../../org/apache/hadoop/hbase/MasterNotRunningException.html" title="class in org.apache.hadoop.hbase">MasterNotRunningException</a>,
@@ -3868,7 +3868,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isActiveMaster</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2945">isActiveMaster</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2948">isActiveMaster</a>()</pre>
 <div class="block">Report whether this master is currently the active master or not.
  If not active master, we are parked on ZK waiting to become active.
 
@@ -3887,7 +3887,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isInitialized</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2959">isInitialized</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2962">isInitialized</a>()</pre>
 <div class="block">Report whether this master has completed with its initialization and is
  ready.  If ready, the master is also the active master.  A standby master
  is never ready.
@@ -3907,7 +3907,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isInMaintenanceMode</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2969">isInMaintenanceMode</a>()
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2972">isInMaintenanceMode</a>()
                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Report whether this master is in maintenance mode.</div>
 <dl>
@@ -3926,7 +3926,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>setInitialized</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2977">setInitialized</a>(boolean&nbsp;isInitialized)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2980">setInitialized</a>(boolean&nbsp;isInitialized)</pre>
 </li>
 </ul>
 <a name="getInitializedEvent--">
@@ -3935,7 +3935,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getInitializedEvent</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureEvent.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureEvent</a>&lt;?&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2982">getInitializedEvent</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureEvent.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureEvent</a>&lt;?&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2985">getInitializedEvent</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getInitializedEvent--">getInitializedEvent</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterSer

<TRUNCATED>

[12/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html
index 681e263..7cdf584 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html
@@ -182,7 +182,7 @@
 <span class="sourceLineNo">174</span>      }<a name="line.174"></a>
 <span class="sourceLineNo">175</span>      FileStatus[] logfiles = FSUtils.listStatus(fs, logDir, filter);<a name="line.175"></a>
 <span class="sourceLineNo">176</span>      if (logfiles == null || logfiles.length == 0) {<a name="line.176"></a>
-<span class="sourceLineNo">177</span>        LOG.info(logDir + " is empty dir, no logs to split");<a name="line.177"></a>
+<span class="sourceLineNo">177</span>        LOG.info("{} dir is empty, no logs to split.", logDir);<a name="line.177"></a>
 <span class="sourceLineNo">178</span>      } else {<a name="line.178"></a>
 <span class="sourceLineNo">179</span>        Collections.addAll(fileStatus, logfiles);<a name="line.179"></a>
 <span class="sourceLineNo">180</span>      }<a name="line.180"></a>
@@ -243,426 +243,430 @@
 <span class="sourceLineNo">235</span>      PathFilter filter) throws IOException {<a name="line.235"></a>
 <span class="sourceLineNo">236</span>    MonitoredTask status = TaskMonitor.get().createStatus("Doing distributed log split in " +<a name="line.236"></a>
 <span class="sourceLineNo">237</span>      logDirs + " for serverName=" + serverNames);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    status.setStatus("Checking directory contents...");<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.240"></a>
-<span class="sourceLineNo">241</span>    LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.241"></a>
-<span class="sourceLineNo">242</span>      " for " + serverNames);<a name="line.242"></a>
-<span class="sourceLineNo">243</span>    long t = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
-<span class="sourceLineNo">244</span>    long totalSize = 0;<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    TaskBatch batch = new TaskBatch();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    for (FileStatus lf : logfiles) {<a name="line.246"></a>
-<span class="sourceLineNo">247</span>      // TODO If the log file is still being written to - which is most likely<a name="line.247"></a>
-<span class="sourceLineNo">248</span>      // the case for the last log file - then its length will show up here<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      // as zero. The size of such a file can only be retrieved after<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      // recover-lease is done. totalSize will be under in most cases and the<a name="line.250"></a>
-<span class="sourceLineNo">251</span>      // metrics that it drives will also be under-reported.<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      totalSize += lf.getLen();<a name="line.252"></a>
-<span class="sourceLineNo">253</span>      String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.253"></a>
-<span class="sourceLineNo">254</span>      if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.254"></a>
-<span class="sourceLineNo">255</span>        throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.255"></a>
-<span class="sourceLineNo">256</span>      }<a name="line.256"></a>
-<span class="sourceLineNo">257</span>    }<a name="line.257"></a>
-<span class="sourceLineNo">258</span>    waitForSplittingCompletion(batch, status);<a name="line.258"></a>
-<span class="sourceLineNo">259</span><a name="line.259"></a>
-<span class="sourceLineNo">260</span>    if (batch.done != batch.installed) {<a name="line.260"></a>
-<span class="sourceLineNo">261</span>      batch.isDead = true;<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.262"></a>
-<span class="sourceLineNo">263</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.263"></a>
-<span class="sourceLineNo">264</span>          + " but only " + batch.done + " done");<a name="line.264"></a>
-<span class="sourceLineNo">265</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      status.abort(msg);<a name="line.266"></a>
-<span class="sourceLineNo">267</span>      throw new IOException(msg);<a name="line.267"></a>
-<span class="sourceLineNo">268</span>    }<a name="line.268"></a>
-<span class="sourceLineNo">269</span>    for (Path logDir : logDirs) {<a name="line.269"></a>
-<span class="sourceLineNo">270</span>      status.setStatus("Cleaning up log directory...");<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      try {<a name="line.272"></a>
-<span class="sourceLineNo">273</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.273"></a>
-<span class="sourceLineNo">274</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.274"></a>
-<span class="sourceLineNo">275</span>        }<a name="line.275"></a>
-<span class="sourceLineNo">276</span>      } catch (IOException ioe) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.277"></a>
-<span class="sourceLineNo">278</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.278"></a>
-<span class="sourceLineNo">279</span>          LOG.warn("Returning success without actually splitting and "<a name="line.279"></a>
-<span class="sourceLineNo">280</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.280"></a>
-<span class="sourceLineNo">281</span>              + Arrays.toString(files), ioe);<a name="line.281"></a>
-<span class="sourceLineNo">282</span>        } else {<a name="line.282"></a>
-<span class="sourceLineNo">283</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.283"></a>
-<span class="sourceLineNo">284</span>        }<a name="line.284"></a>
-<span class="sourceLineNo">285</span>      }<a name="line.285"></a>
-<span class="sourceLineNo">286</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.286"></a>
-<span class="sourceLineNo">287</span>    }<a name="line.287"></a>
-<span class="sourceLineNo">288</span>    String msg =<a name="line.288"></a>
-<span class="sourceLineNo">289</span>        "finished splitting (more than or equal to) " + totalSize + " bytes in " + batch.installed<a name="line.289"></a>
-<span class="sourceLineNo">290</span>            + " log files in " + logDirs + " in "<a name="line.290"></a>
-<span class="sourceLineNo">291</span>            + (EnvironmentEdgeManager.currentTime() - t) + "ms";<a name="line.291"></a>
-<span class="sourceLineNo">292</span>    status.markComplete(msg);<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    LOG.info(msg);<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    return totalSize;<a name="line.294"></a>
-<span class="sourceLineNo">295</span>  }<a name="line.295"></a>
-<span class="sourceLineNo">296</span><a name="line.296"></a>
-<span class="sourceLineNo">297</span>  /**<a name="line.297"></a>
-<span class="sourceLineNo">298</span>   * Add a task entry to coordination if it is not already there.<a name="line.298"></a>
-<span class="sourceLineNo">299</span>   * @param taskname the path of the log to be split<a name="line.299"></a>
-<span class="sourceLineNo">300</span>   * @param batch the batch this task belongs to<a name="line.300"></a>
-<span class="sourceLineNo">301</span>   * @return true if a new entry is created, false if it is already there.<a name="line.301"></a>
-<span class="sourceLineNo">302</span>   */<a name="line.302"></a>
-<span class="sourceLineNo">303</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.303"></a>
-<span class="sourceLineNo">304</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.304"></a>
-<span class="sourceLineNo">305</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.305"></a>
-<span class="sourceLineNo">306</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.306"></a>
-<span class="sourceLineNo">307</span>    if (oldtask == null) {<a name="line.307"></a>
-<span class="sourceLineNo">308</span>      // publish the task in the coordination engine<a name="line.308"></a>
-<span class="sourceLineNo">309</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.309"></a>
-<span class="sourceLineNo">310</span>      return true;<a name="line.310"></a>
-<span class="sourceLineNo">311</span>    }<a name="line.311"></a>
-<span class="sourceLineNo">312</span>    return false;<a name="line.312"></a>
-<span class="sourceLineNo">313</span>  }<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.315"></a>
-<span class="sourceLineNo">316</span>    synchronized (batch) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.317"></a>
-<span class="sourceLineNo">318</span>        try {<a name="line.318"></a>
-<span class="sourceLineNo">319</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.319"></a>
-<span class="sourceLineNo">320</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.320"></a>
-<span class="sourceLineNo">321</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.321"></a>
-<span class="sourceLineNo">322</span>          int actual = activeTasks(batch);<a name="line.322"></a>
-<span class="sourceLineNo">323</span>          if (remaining != actual) {<a name="line.323"></a>
-<span class="sourceLineNo">324</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.324"></a>
-<span class="sourceLineNo">325</span>          }<a name="line.325"></a>
-<span class="sourceLineNo">326</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.326"></a>
-<span class="sourceLineNo">327</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.327"></a>
-<span class="sourceLineNo">328</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.328"></a>
-<span class="sourceLineNo">329</span>                + remainingTasks);<a name="line.329"></a>
-<span class="sourceLineNo">330</span>          }<a name="line.330"></a>
-<span class="sourceLineNo">331</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span>            LOG.warn("No more task remaining, splitting "<a name="line.332"></a>
-<span class="sourceLineNo">333</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.333"></a>
-<span class="sourceLineNo">334</span>                + ", active tasks in map " + actual);<a name="line.334"></a>
-<span class="sourceLineNo">335</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.335"></a>
-<span class="sourceLineNo">336</span>              return;<a name="line.336"></a>
-<span class="sourceLineNo">337</span>            }<a name="line.337"></a>
-<span class="sourceLineNo">338</span>          }<a name="line.338"></a>
-<span class="sourceLineNo">339</span>          batch.wait(100);<a name="line.339"></a>
-<span class="sourceLineNo">340</span>          if (server.isStopped()) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.341"></a>
-<span class="sourceLineNo">342</span>            return;<a name="line.342"></a>
-<span class="sourceLineNo">343</span>          }<a name="line.343"></a>
-<span class="sourceLineNo">344</span>        } catch (InterruptedException e) {<a name="line.344"></a>
-<span class="sourceLineNo">345</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.345"></a>
-<span class="sourceLineNo">346</span>          Thread.currentThread().interrupt();<a name="line.346"></a>
-<span class="sourceLineNo">347</span>          return;<a name="line.347"></a>
-<span class="sourceLineNo">348</span>        }<a name="line.348"></a>
-<span class="sourceLineNo">349</span>      }<a name="line.349"></a>
-<span class="sourceLineNo">350</span>    }<a name="line.350"></a>
-<span class="sourceLineNo">351</span>  }<a name="line.351"></a>
-<span class="sourceLineNo">352</span><a name="line.352"></a>
-<span class="sourceLineNo">353</span>  @VisibleForTesting<a name="line.353"></a>
-<span class="sourceLineNo">354</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.354"></a>
-<span class="sourceLineNo">355</span>    return tasks;<a name="line.355"></a>
-<span class="sourceLineNo">356</span>  }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span>  private int activeTasks(final TaskBatch batch) {<a name="line.358"></a>
-<span class="sourceLineNo">359</span>    int count = 0;<a name="line.359"></a>
-<span class="sourceLineNo">360</span>    for (Task t : tasks.values()) {<a name="line.360"></a>
-<span class="sourceLineNo">361</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.361"></a>
-<span class="sourceLineNo">362</span>        count++;<a name="line.362"></a>
-<span class="sourceLineNo">363</span>      }<a name="line.363"></a>
-<span class="sourceLineNo">364</span>    }<a name="line.364"></a>
-<span class="sourceLineNo">365</span>    return count;<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span>  }<a name="line.367"></a>
-<span class="sourceLineNo">368</span><a name="line.368"></a>
-<span class="sourceLineNo">369</span>  /**<a name="line.369"></a>
-<span class="sourceLineNo">370</span>   * @param path<a name="line.370"></a>
-<span class="sourceLineNo">371</span>   * @param batch<a name="line.371"></a>
-<span class="sourceLineNo">372</span>   * @return null on success, existing task on error<a name="line.372"></a>
-<span class="sourceLineNo">373</span>   */<a name="line.373"></a>
-<span class="sourceLineNo">374</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span>    Task oldtask;<a name="line.375"></a>
-<span class="sourceLineNo">376</span>    // batch.installed is only changed via this function and<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    // a single thread touches batch.installed.<a name="line.377"></a>
-<span class="sourceLineNo">378</span>    Task newtask = new Task();<a name="line.378"></a>
-<span class="sourceLineNo">379</span>    newtask.batch = batch;<a name="line.379"></a>
-<span class="sourceLineNo">380</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    if (oldtask == null) {<a name="line.381"></a>
-<span class="sourceLineNo">382</span>      batch.installed++;<a name="line.382"></a>
-<span class="sourceLineNo">383</span>      return null;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    }<a name="line.384"></a>
-<span class="sourceLineNo">385</span>    // new task was not used.<a name="line.385"></a>
-<span class="sourceLineNo">386</span>    synchronized (oldtask) {<a name="line.386"></a>
-<span class="sourceLineNo">387</span>      if (oldtask.isOrphan()) {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>        if (oldtask.status == SUCCESS) {<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          // The task is already done. Do not install the batch for this<a name="line.389"></a>
-<span class="sourceLineNo">390</span>          // task because it might be too late for setDone() to update<a name="line.390"></a>
-<span class="sourceLineNo">391</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.391"></a>
-<span class="sourceLineNo">392</span>          // this task to complete.<a name="line.392"></a>
-<span class="sourceLineNo">393</span>          return (null);<a name="line.393"></a>
-<span class="sourceLineNo">394</span>        }<a name="line.394"></a>
-<span class="sourceLineNo">395</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.395"></a>
-<span class="sourceLineNo">396</span>          oldtask.batch = batch;<a name="line.396"></a>
-<span class="sourceLineNo">397</span>          batch.installed++;<a name="line.397"></a>
-<span class="sourceLineNo">398</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.398"></a>
-<span class="sourceLineNo">399</span>          return null;<a name="line.399"></a>
-<span class="sourceLineNo">400</span>        }<a name="line.400"></a>
-<span class="sourceLineNo">401</span>        while (oldtask.status == FAILURE) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.402"></a>
-<span class="sourceLineNo">403</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.403"></a>
-<span class="sourceLineNo">404</span>          try {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>            oldtask.wait();<a name="line.405"></a>
-<span class="sourceLineNo">406</span>          } catch (InterruptedException e) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span>            Thread.currentThread().interrupt();<a name="line.407"></a>
-<span class="sourceLineNo">408</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.408"></a>
-<span class="sourceLineNo">409</span>            // fall through to return failure<a name="line.409"></a>
-<span class="sourceLineNo">410</span>            break;<a name="line.410"></a>
-<span class="sourceLineNo">411</span>          }<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        }<a name="line.412"></a>
-<span class="sourceLineNo">413</span>        if (oldtask.status != DELETED) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span>          LOG.warn("Failure because previously failed task"<a name="line.414"></a>
-<span class="sourceLineNo">415</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.415"></a>
-<span class="sourceLineNo">416</span>          return oldtask;<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        }<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        // reinsert the newTask and it must succeed this time<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        if (t == null) {<a name="line.420"></a>
-<span class="sourceLineNo">421</span>          batch.installed++;<a name="line.421"></a>
-<span class="sourceLineNo">422</span>          return null;<a name="line.422"></a>
-<span class="sourceLineNo">423</span>        }<a name="line.423"></a>
-<span class="sourceLineNo">424</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        assert false : "Deleted task still present in tasks map";<a name="line.425"></a>
-<span class="sourceLineNo">426</span>        return t;<a name="line.426"></a>
-<span class="sourceLineNo">427</span>      }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.428"></a>
-<span class="sourceLineNo">429</span>      return oldtask;<a name="line.429"></a>
-<span class="sourceLineNo">430</span>    }<a name="line.430"></a>
-<span class="sourceLineNo">431</span>  }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span>  public void stop() {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (choreService != null) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      choreService.shutdown();<a name="line.435"></a>
-<span class="sourceLineNo">436</span>    }<a name="line.436"></a>
-<span class="sourceLineNo">437</span>    if (timeoutMonitor != null) {<a name="line.437"></a>
-<span class="sourceLineNo">438</span>      timeoutMonitor.cancel(true);<a name="line.438"></a>
-<span class="sourceLineNo">439</span>    }<a name="line.439"></a>
-<span class="sourceLineNo">440</span>  }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>  void handleDeadWorker(ServerName workerName) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.443"></a>
-<span class="sourceLineNo">444</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.444"></a>
-<span class="sourceLineNo">445</span>    synchronized (deadWorkersLock) {<a name="line.445"></a>
-<span class="sourceLineNo">446</span>      if (deadWorkers == null) {<a name="line.446"></a>
-<span class="sourceLineNo">447</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>      }<a name="line.448"></a>
-<span class="sourceLineNo">449</span>      deadWorkers.add(workerName);<a name="line.449"></a>
-<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    LOG.info("dead splitlog worker " + workerName);<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  }<a name="line.452"></a>
-<span class="sourceLineNo">453</span><a name="line.453"></a>
-<span class="sourceLineNo">454</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.454"></a>
-<span class="sourceLineNo">455</span>    synchronized (deadWorkersLock) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span>      if (deadWorkers == null) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      }<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      deadWorkers.addAll(serverNames);<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.461"></a>
-<span class="sourceLineNo">462</span>  }<a name="line.462"></a>
-<span class="sourceLineNo">463</span><a name="line.463"></a>
-<span class="sourceLineNo">464</span>  /**<a name="line.464"></a>
-<span class="sourceLineNo">465</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.465"></a>
-<span class="sourceLineNo">466</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.466"></a>
-<span class="sourceLineNo">467</span>   * &lt;p&gt;<a name="line.467"></a>
-<span class="sourceLineNo">468</span>   * All access is synchronized.<a name="line.468"></a>
-<span class="sourceLineNo">469</span>   */<a name="line.469"></a>
-<span class="sourceLineNo">470</span>  @InterfaceAudience.Private<a name="line.470"></a>
-<span class="sourceLineNo">471</span>  public static class TaskBatch {<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    public int installed = 0;<a name="line.472"></a>
-<span class="sourceLineNo">473</span>    public int done = 0;<a name="line.473"></a>
-<span class="sourceLineNo">474</span>    public int error = 0;<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    public volatile boolean isDead = false;<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span>    @Override<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    public String toString() {<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.479"></a>
-<span class="sourceLineNo">480</span>    }<a name="line.480"></a>
-<span class="sourceLineNo">481</span>  }<a name="line.481"></a>
-<span class="sourceLineNo">482</span><a name="line.482"></a>
-<span class="sourceLineNo">483</span>  /**<a name="line.483"></a>
-<span class="sourceLineNo">484</span>   * in memory state of an active task.<a name="line.484"></a>
-<span class="sourceLineNo">485</span>   */<a name="line.485"></a>
-<span class="sourceLineNo">486</span>  @InterfaceAudience.Private<a name="line.486"></a>
-<span class="sourceLineNo">487</span>  public static class Task {<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    public volatile long last_update;<a name="line.488"></a>
-<span class="sourceLineNo">489</span>    public volatile int last_version;<a name="line.489"></a>
-<span class="sourceLineNo">490</span>    public volatile ServerName cur_worker_name;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>    public volatile TaskBatch batch;<a name="line.491"></a>
-<span class="sourceLineNo">492</span>    public volatile TerminationStatus status;<a name="line.492"></a>
-<span class="sourceLineNo">493</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.493"></a>
-<span class="sourceLineNo">494</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    public volatile boolean resubmitThresholdReached;<a name="line.495"></a>
-<span class="sourceLineNo">496</span><a name="line.496"></a>
-<span class="sourceLineNo">497</span>    @Override<a name="line.497"></a>
-<span class="sourceLineNo">498</span>    public String toString() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.499"></a>
-<span class="sourceLineNo">500</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.500"></a>
-<span class="sourceLineNo">501</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.501"></a>
-<span class="sourceLineNo">502</span>    }<a name="line.502"></a>
-<span class="sourceLineNo">503</span><a name="line.503"></a>
-<span class="sourceLineNo">504</span>    public Task() {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      last_version = -1;<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      status = IN_PROGRESS;<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      setUnassigned();<a name="line.507"></a>
-<span class="sourceLineNo">508</span>    }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span>    public boolean isOrphan() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      return (batch == null || batch.isDead);<a name="line.511"></a>
+<span class="sourceLineNo">238</span>    long totalSize = 0;<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    TaskBatch batch = null;<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    long startTime = 0;<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.241"></a>
+<span class="sourceLineNo">242</span>    if (logfiles.length != 0) {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>      status.setStatus("Checking directory contents...");<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.245"></a>
+<span class="sourceLineNo">246</span>          " for " + serverNames);<a name="line.246"></a>
+<span class="sourceLineNo">247</span>      startTime = EnvironmentEdgeManager.currentTime();<a name="line.247"></a>
+<span class="sourceLineNo">248</span>      batch = new TaskBatch();<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      for (FileStatus lf : logfiles) {<a name="line.249"></a>
+<span class="sourceLineNo">250</span>        // TODO If the log file is still being written to - which is most likely<a name="line.250"></a>
+<span class="sourceLineNo">251</span>        // the case for the last log file - then its length will show up here<a name="line.251"></a>
+<span class="sourceLineNo">252</span>        // as zero. The size of such a file can only be retrieved after<a name="line.252"></a>
+<span class="sourceLineNo">253</span>        // recover-lease is done. totalSize will be under in most cases and the<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        // metrics that it drives will also be under-reported.<a name="line.254"></a>
+<span class="sourceLineNo">255</span>        totalSize += lf.getLen();<a name="line.255"></a>
+<span class="sourceLineNo">256</span>        String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.257"></a>
+<span class="sourceLineNo">258</span>          throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.258"></a>
+<span class="sourceLineNo">259</span>        }<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      }<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      waitForSplittingCompletion(batch, status);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    if (batch != null &amp;&amp; batch.done != batch.installed) {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      batch.isDead = true;<a name="line.265"></a>
+<span class="sourceLineNo">266</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.267"></a>
+<span class="sourceLineNo">268</span>          + " but only " + batch.done + " done");<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.269"></a>
+<span class="sourceLineNo">270</span>      status.abort(msg);<a name="line.270"></a>
+<span class="sourceLineNo">271</span>      throw new IOException(msg);<a name="line.271"></a>
+<span class="sourceLineNo">272</span>    }<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    for (Path logDir : logDirs) {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      status.setStatus("Cleaning up log directory...");<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.275"></a>
+<span class="sourceLineNo">276</span>      try {<a name="line.276"></a>
+<span class="sourceLineNo">277</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.277"></a>
+<span class="sourceLineNo">278</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.278"></a>
+<span class="sourceLineNo">279</span>        }<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      } catch (IOException ioe) {<a name="line.280"></a>
+<span class="sourceLineNo">281</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.281"></a>
+<span class="sourceLineNo">282</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.282"></a>
+<span class="sourceLineNo">283</span>          LOG.warn("Returning success without actually splitting and "<a name="line.283"></a>
+<span class="sourceLineNo">284</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.284"></a>
+<span class="sourceLineNo">285</span>              + Arrays.toString(files), ioe);<a name="line.285"></a>
+<span class="sourceLineNo">286</span>        } else {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.287"></a>
+<span class="sourceLineNo">288</span>        }<a name="line.288"></a>
+<span class="sourceLineNo">289</span>      }<a name="line.289"></a>
+<span class="sourceLineNo">290</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.290"></a>
+<span class="sourceLineNo">291</span>    }<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    String msg = "Finished splitting (more than or equal to) " + totalSize +<a name="line.292"></a>
+<span class="sourceLineNo">293</span>        " bytes in " + ((batch == null)? 0: batch.installed) +<a name="line.293"></a>
+<span class="sourceLineNo">294</span>        " log files in " + logDirs + " in " +<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        ((startTime == 0)? startTime: (EnvironmentEdgeManager.currentTime() - startTime)) + "ms";<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    status.markComplete(msg);<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    LOG.info(msg);<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    return totalSize;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
+<span class="sourceLineNo">300</span><a name="line.300"></a>
+<span class="sourceLineNo">301</span>  /**<a name="line.301"></a>
+<span class="sourceLineNo">302</span>   * Add a task entry to coordination if it is not already there.<a name="line.302"></a>
+<span class="sourceLineNo">303</span>   * @param taskname the path of the log to be split<a name="line.303"></a>
+<span class="sourceLineNo">304</span>   * @param batch the batch this task belongs to<a name="line.304"></a>
+<span class="sourceLineNo">305</span>   * @return true if a new entry is created, false if it is already there.<a name="line.305"></a>
+<span class="sourceLineNo">306</span>   */<a name="line.306"></a>
+<span class="sourceLineNo">307</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.307"></a>
+<span class="sourceLineNo">308</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.309"></a>
+<span class="sourceLineNo">310</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.310"></a>
+<span class="sourceLineNo">311</span>    if (oldtask == null) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span>      // publish the task in the coordination engine<a name="line.312"></a>
+<span class="sourceLineNo">313</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.313"></a>
+<span class="sourceLineNo">314</span>      return true;<a name="line.314"></a>
+<span class="sourceLineNo">315</span>    }<a name="line.315"></a>
+<span class="sourceLineNo">316</span>    return false;<a name="line.316"></a>
+<span class="sourceLineNo">317</span>  }<a name="line.317"></a>
+<span class="sourceLineNo">318</span><a name="line.318"></a>
+<span class="sourceLineNo">319</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    synchronized (batch) {<a name="line.320"></a>
+<span class="sourceLineNo">321</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.321"></a>
+<span class="sourceLineNo">322</span>        try {<a name="line.322"></a>
+<span class="sourceLineNo">323</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.323"></a>
+<span class="sourceLineNo">324</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.324"></a>
+<span class="sourceLineNo">325</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.325"></a>
+<span class="sourceLineNo">326</span>          int actual = activeTasks(batch);<a name="line.326"></a>
+<span class="sourceLineNo">327</span>          if (remaining != actual) {<a name="line.327"></a>
+<span class="sourceLineNo">328</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.328"></a>
+<span class="sourceLineNo">329</span>          }<a name="line.329"></a>
+<span class="sourceLineNo">330</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.330"></a>
+<span class="sourceLineNo">331</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.331"></a>
+<span class="sourceLineNo">332</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.332"></a>
+<span class="sourceLineNo">333</span>                + remainingTasks);<a name="line.333"></a>
+<span class="sourceLineNo">334</span>          }<a name="line.334"></a>
+<span class="sourceLineNo">335</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.335"></a>
+<span class="sourceLineNo">336</span>            LOG.warn("No more task remaining, splitting "<a name="line.336"></a>
+<span class="sourceLineNo">337</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.337"></a>
+<span class="sourceLineNo">338</span>                + ", active tasks in map " + actual);<a name="line.338"></a>
+<span class="sourceLineNo">339</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.339"></a>
+<span class="sourceLineNo">340</span>              return;<a name="line.340"></a>
+<span class="sourceLineNo">341</span>            }<a name="line.341"></a>
+<span class="sourceLineNo">342</span>          }<a name="line.342"></a>
+<span class="sourceLineNo">343</span>          batch.wait(100);<a name="line.343"></a>
+<span class="sourceLineNo">344</span>          if (server.isStopped()) {<a name="line.344"></a>
+<span class="sourceLineNo">345</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.345"></a>
+<span class="sourceLineNo">346</span>            return;<a name="line.346"></a>
+<span class="sourceLineNo">347</span>          }<a name="line.347"></a>
+<span class="sourceLineNo">348</span>        } catch (InterruptedException e) {<a name="line.348"></a>
+<span class="sourceLineNo">349</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.349"></a>
+<span class="sourceLineNo">350</span>          Thread.currentThread().interrupt();<a name="line.350"></a>
+<span class="sourceLineNo">351</span>          return;<a name="line.351"></a>
+<span class="sourceLineNo">352</span>        }<a name="line.352"></a>
+<span class="sourceLineNo">353</span>      }<a name="line.353"></a>
+<span class="sourceLineNo">354</span>    }<a name="line.354"></a>
+<span class="sourceLineNo">355</span>  }<a name="line.355"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span>  @VisibleForTesting<a name="line.357"></a>
+<span class="sourceLineNo">358</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.358"></a>
+<span class="sourceLineNo">359</span>    return tasks;<a name="line.359"></a>
+<span class="sourceLineNo">360</span>  }<a name="line.360"></a>
+<span class="sourceLineNo">361</span><a name="line.361"></a>
+<span class="sourceLineNo">362</span>  private int activeTasks(final TaskBatch batch) {<a name="line.362"></a>
+<span class="sourceLineNo">363</span>    int count = 0;<a name="line.363"></a>
+<span class="sourceLineNo">364</span>    for (Task t : tasks.values()) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.365"></a>
+<span class="sourceLineNo">366</span>        count++;<a name="line.366"></a>
+<span class="sourceLineNo">367</span>      }<a name="line.367"></a>
+<span class="sourceLineNo">368</span>    }<a name="line.368"></a>
+<span class="sourceLineNo">369</span>    return count;<a name="line.369"></a>
+<span class="sourceLineNo">370</span><a name="line.370"></a>
+<span class="sourceLineNo">371</span>  }<a name="line.371"></a>
+<span class="sourceLineNo">372</span><a name="line.372"></a>
+<span class="sourceLineNo">373</span>  /**<a name="line.373"></a>
+<span class="sourceLineNo">374</span>   * @param path<a name="line.374"></a>
+<span class="sourceLineNo">375</span>   * @param batch<a name="line.375"></a>
+<span class="sourceLineNo">376</span>   * @return null on success, existing task on error<a name="line.376"></a>
+<span class="sourceLineNo">377</span>   */<a name="line.377"></a>
+<span class="sourceLineNo">378</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.378"></a>
+<span class="sourceLineNo">379</span>    Task oldtask;<a name="line.379"></a>
+<span class="sourceLineNo">380</span>    // batch.installed is only changed via this function and<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    // a single thread touches batch.installed.<a name="line.381"></a>
+<span class="sourceLineNo">382</span>    Task newtask = new Task();<a name="line.382"></a>
+<span class="sourceLineNo">383</span>    newtask.batch = batch;<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.384"></a>
+<span class="sourceLineNo">385</span>    if (oldtask == null) {<a name="line.385"></a>
+<span class="sourceLineNo">386</span>      batch.installed++;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      return null;<a name="line.387"></a>
+<span class="sourceLineNo">388</span>    }<a name="line.388"></a>
+<span class="sourceLineNo">389</span>    // new task was not used.<a name="line.389"></a>
+<span class="sourceLineNo">390</span>    synchronized (oldtask) {<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      if (oldtask.isOrphan()) {<a name="line.391"></a>
+<span class="sourceLineNo">392</span>        if (oldtask.status == SUCCESS) {<a name="line.392"></a>
+<span class="sourceLineNo">393</span>          // The task is already done. Do not install the batch for this<a name="line.393"></a>
+<span class="sourceLineNo">394</span>          // task because it might be too late for setDone() to update<a name="line.394"></a>
+<span class="sourceLineNo">395</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.395"></a>
+<span class="sourceLineNo">396</span>          // this task to complete.<a name="line.396"></a>
+<span class="sourceLineNo">397</span>          return (null);<a name="line.397"></a>
+<span class="sourceLineNo">398</span>        }<a name="line.398"></a>
+<span class="sourceLineNo">399</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.399"></a>
+<span class="sourceLineNo">400</span>          oldtask.batch = batch;<a name="line.400"></a>
+<span class="sourceLineNo">401</span>          batch.installed++;<a name="line.401"></a>
+<span class="sourceLineNo">402</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.402"></a>
+<span class="sourceLineNo">403</span>          return null;<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        }<a name="line.404"></a>
+<span class="sourceLineNo">405</span>        while (oldtask.status == FAILURE) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.406"></a>
+<span class="sourceLineNo">407</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.407"></a>
+<span class="sourceLineNo">408</span>          try {<a name="line.408"></a>
+<span class="sourceLineNo">409</span>            oldtask.wait();<a name="line.409"></a>
+<span class="sourceLineNo">410</span>          } catch (InterruptedException e) {<a name="line.410"></a>
+<span class="sourceLineNo">411</span>            Thread.currentThread().interrupt();<a name="line.411"></a>
+<span class="sourceLineNo">412</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.412"></a>
+<span class="sourceLineNo">413</span>            // fall through to return failure<a name="line.413"></a>
+<span class="sourceLineNo">414</span>            break;<a name="line.414"></a>
+<span class="sourceLineNo">415</span>          }<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        }<a name="line.416"></a>
+<span class="sourceLineNo">417</span>        if (oldtask.status != DELETED) {<a name="line.417"></a>
+<span class="sourceLineNo">418</span>          LOG.warn("Failure because previously failed task"<a name="line.418"></a>
+<span class="sourceLineNo">419</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.419"></a>
+<span class="sourceLineNo">420</span>          return oldtask;<a name="line.420"></a>
+<span class="sourceLineNo">421</span>        }<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        // reinsert the newTask and it must succeed this time<a name="line.422"></a>
+<span class="sourceLineNo">423</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        if (t == null) {<a name="line.424"></a>
+<span class="sourceLineNo">425</span>          batch.installed++;<a name="line.425"></a>
+<span class="sourceLineNo">426</span>          return null;<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        }<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.428"></a>
+<span class="sourceLineNo">429</span>        assert false : "Deleted task still present in tasks map";<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        return t;<a name="line.430"></a>
+<span class="sourceLineNo">431</span>      }<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.432"></a>
+<span class="sourceLineNo">433</span>      return oldtask;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>    }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>  }<a name="line.435"></a>
+<span class="sourceLineNo">436</span><a name="line.436"></a>
+<span class="sourceLineNo">437</span>  public void stop() {<a name="line.437"></a>
+<span class="sourceLineNo">438</span>    if (choreService != null) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span>      choreService.shutdown();<a name="line.439"></a>
+<span class="sourceLineNo">440</span>    }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    if (timeoutMonitor != null) {<a name="line.441"></a>
+<span class="sourceLineNo">442</span>      timeoutMonitor.cancel(true);<a name="line.442"></a>
+<span class="sourceLineNo">443</span>    }<a name="line.443"></a>
+<span class="sourceLineNo">444</span>  }<a name="line.444"></a>
+<span class="sourceLineNo">445</span><a name="line.445"></a>
+<span class="sourceLineNo">446</span>  void handleDeadWorker(ServerName workerName) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.447"></a>
+<span class="sourceLineNo">448</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.448"></a>
+<span class="sourceLineNo">449</span>    synchronized (deadWorkersLock) {<a name="line.449"></a>
+<span class="sourceLineNo">450</span>      if (deadWorkers == null) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      }<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      deadWorkers.add(workerName);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    }<a name="line.454"></a>
+<span class="sourceLineNo">455</span>    LOG.info("Dead splitlog worker {}", workerName);<a name="line.455"></a>
+<span class="sourceLineNo">456</span>  }<a name="line.456"></a>
+<span class="sourceLineNo">457</span><a name="line.457"></a>
+<span class="sourceLineNo">458</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>    synchronized (deadWorkersLock) {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>      if (deadWorkers == null) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.461"></a>
+<span class="sourceLineNo">462</span>      }<a name="line.462"></a>
+<span class="sourceLineNo">463</span>      deadWorkers.addAll(serverNames);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    }<a name="line.464"></a>
+<span class="sourceLineNo">465</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.465"></a>
+<span class="sourceLineNo">466</span>  }<a name="line.466"></a>
+<span class="sourceLineNo">467</span><a name="line.467"></a>
+<span class="sourceLineNo">468</span>  /**<a name="line.468"></a>
+<span class="sourceLineNo">469</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.469"></a>
+<span class="sourceLineNo">470</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.470"></a>
+<span class="sourceLineNo">471</span>   * &lt;p&gt;<a name="line.471"></a>
+<span class="sourceLineNo">472</span>   * All access is synchronized.<a name="line.472"></a>
+<span class="sourceLineNo">473</span>   */<a name="line.473"></a>
+<span class="sourceLineNo">474</span>  @InterfaceAudience.Private<a name="line.474"></a>
+<span class="sourceLineNo">475</span>  public static class TaskBatch {<a name="line.475"></a>
+<span class="sourceLineNo">476</span>    public int installed = 0;<a name="line.476"></a>
+<span class="sourceLineNo">477</span>    public int done = 0;<a name="line.477"></a>
+<span class="sourceLineNo">478</span>    public int error = 0;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    public volatile boolean isDead = false;<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span>    @Override<a name="line.481"></a>
+<span class="sourceLineNo">482</span>    public String toString() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    }<a name="line.484"></a>
+<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
+<span class="sourceLineNo">486</span><a name="line.486"></a>
+<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
+<span class="sourceLineNo">488</span>   * in memory state of an active task.<a name="line.488"></a>
+<span class="sourceLineNo">489</span>   */<a name="line.489"></a>
+<span class="sourceLineNo">490</span>  @InterfaceAudience.Private<a name="line.490"></a>
+<span class="sourceLineNo">491</span>  public static class Task {<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    public volatile long last_update;<a name="line.492"></a>
+<span class="sourceLineNo">493</span>    public volatile int last_version;<a name="line.493"></a>
+<span class="sourceLineNo">494</span>    public volatile ServerName cur_worker_name;<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    public volatile TaskBatch batch;<a name="line.495"></a>
+<span class="sourceLineNo">496</span>    public volatile TerminationStatus status;<a name="line.496"></a>
+<span class="sourceLineNo">497</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.497"></a>
+<span class="sourceLineNo">498</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.498"></a>
+<span class="sourceLineNo">499</span>    public volatile boolean resubmitThresholdReached;<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span>    @Override<a name="line.501"></a>
+<span class="sourceLineNo">502</span>    public String toString() {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.503"></a>
+<span class="sourceLineNo">504</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.504"></a>
+<span class="sourceLineNo">505</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.505"></a>
+<span class="sourceLineNo">506</span>    }<a name="line.506"></a>
+<span class="sourceLineNo">507</span><a name="line.507"></a>
+<span class="sourceLineNo">508</span>    public Task() {<a name="line.508"></a>
+<span class="sourceLineNo">509</span>      last_version = -1;<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      status = IN_PROGRESS;<a name="line.510"></a>
+<span class="sourceLineNo">511</span>      setUnassigned();<a name="line.511"></a>
 <span class="sourceLineNo">512</span>    }<a name="line.512"></a>
 <span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span>    public boolean isUnassigned() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span>      return (cur_worker_name == null);<a name="line.515"></a>
+<span class="sourceLineNo">514</span>    public boolean isOrphan() {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>      return (batch == null || batch.isDead);<a name="line.515"></a>
 <span class="sourceLineNo">516</span>    }<a name="line.516"></a>
 <span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span>    public void heartbeatNoDetails(long time) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span>      last_update = time;<a name="line.519"></a>
+<span class="sourceLineNo">518</span>    public boolean isUnassigned() {<a name="line.518"></a>
+<span class="sourceLineNo">519</span>      return (cur_worker_name == null);<a name="line.519"></a>
 <span class="sourceLineNo">520</span>    }<a name="line.520"></a>
 <span class="sourceLineNo">521</span><a name="line.521"></a>
-<span class="sourceLineNo">522</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      last_version = version;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      last_update = time;<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      cur_worker_name = worker;<a name="line.525"></a>
-<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span>    public void setUnassigned() {<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      cur_worker_name = null;<a name="line.529"></a>
-<span class="sourceLineNo">530</span>      last_update = -1;<a name="line.530"></a>
-<span class="sourceLineNo">531</span>    }<a name="line.531"></a>
-<span class="sourceLineNo">532</span>  }<a name="line.532"></a>
-<span class="sourceLineNo">533</span><a name="line.533"></a>
-<span class="sourceLineNo">534</span>  /**<a name="line.534"></a>
-<span class="sourceLineNo">535</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.535"></a>
-<span class="sourceLineNo">536</span>   */<a name="line.536"></a>
-<span class="sourceLineNo">537</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.537"></a>
-<span class="sourceLineNo">538</span>    private long lastLog = 0;<a name="line.538"></a>
-<span class="sourceLineNo">539</span><a name="line.539"></a>
-<span class="sourceLineNo">540</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.540"></a>
-<span class="sourceLineNo">541</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.541"></a>
-<span class="sourceLineNo">542</span>    }<a name="line.542"></a>
+<span class="sourceLineNo">522</span>    public void heartbeatNoDetails(long time) {<a name="line.522"></a>
+<span class="sourceLineNo">523</span>      last_update = time;<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    }<a name="line.524"></a>
+<span class="sourceLineNo">525</span><a name="line.525"></a>
+<span class="sourceLineNo">526</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span>      last_version = version;<a name="line.527"></a>
+<span class="sourceLineNo">528</span>      last_update = time;<a name="line.528"></a>
+<span class="sourceLineNo">529</span>      cur_worker_name = worker;<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    }<a name="line.530"></a>
+<span class="sourceLineNo">531</span><a name="line.531"></a>
+<span class="sourceLineNo">532</span>    public void setUnassigned() {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>      cur_worker_name = null;<a name="line.533"></a>
+<span class="sourceLineNo">534</span>      last_update = -1;<a name="line.534"></a>
+<span class="sourceLineNo">535</span>    }<a name="line.535"></a>
+<span class="sourceLineNo">536</span>  }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>  /**<a name="line.538"></a>
+<span class="sourceLineNo">539</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.539"></a>
+<span class="sourceLineNo">540</span>   */<a name="line.540"></a>
+<span class="sourceLineNo">541</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.541"></a>
+<span class="sourceLineNo">542</span>    private long lastLog = 0;<a name="line.542"></a>
 <span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>    @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    protected void chore() {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.546"></a>
+<span class="sourceLineNo">544</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    }<a name="line.546"></a>
 <span class="sourceLineNo">547</span><a name="line.547"></a>
-<span class="sourceLineNo">548</span>      int resubmitted = 0;<a name="line.548"></a>
-<span class="sourceLineNo">549</span>      int unassigned = 0;<a name="line.549"></a>
-<span class="sourceLineNo">550</span>      int tot = 0;<a name="line.550"></a>
-<span class="sourceLineNo">551</span>      boolean found_assigned_task = false;<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.552"></a>
-<span class="sourceLineNo">553</span><a name="line.553"></a>
-<span class="sourceLineNo">554</span>      synchronized (deadWorkersLock) {<a name="line.554"></a>
-<span class="sourceLineNo">555</span>        localDeadWorkers = deadWorkers;<a name="line.555"></a>
-<span class="sourceLineNo">556</span>        deadWorkers = null;<a name="line.556"></a>
-<span class="sourceLineNo">557</span>      }<a name="line.557"></a>
-<span class="sourceLineNo">558</span><a name="line.558"></a>
-<span class="sourceLineNo">559</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>        String path = e.getKey();<a name="line.560"></a>
-<span class="sourceLineNo">561</span>        Task task = e.getValue();<a name="line.561"></a>
-<span class="sourceLineNo">562</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.562"></a>
-<span class="sourceLineNo">563</span>        tot++;<a name="line.563"></a>
-<span class="sourceLineNo">564</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.564"></a>
-<span class="sourceLineNo">565</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.565"></a>
-<span class="sourceLineNo">566</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.566"></a>
-<span class="sourceLineNo">567</span>        // time. If we want progress when there are no region servers then we<a name="line.567"></a>
-<span class="sourceLineNo">568</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.568"></a>
-<span class="sourceLineNo">569</span>        if (task.isUnassigned()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>          unassigned++;<a name="line.570"></a>
-<span class="sourceLineNo">571</span>          continue;<a name="line.571"></a>
-<span class="sourceLineNo">572</span>        }<a name="line.572"></a>
-<span class="sourceLineNo">573</span>        found_assigned_task = true;<a name="line.573"></a>
-<span class="sourceLineNo">574</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.575"></a>
-<span class="sourceLineNo">576</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.576"></a>
-<span class="sourceLineNo">577</span>            resubmitted++;<a name="line.577"></a>
-<span class="sourceLineNo">578</span>          } else {<a name="line.578"></a>
-<span class="sourceLineNo">579</span>            handleDeadWorker(cur_worker);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.580"></a>
-<span class="sourceLineNo">581</span>                + ", will retry.");<a name="line.581"></a>
-<span class="sourceLineNo">582</span>          }<a name="line.582"></a>
-<span class="sourceLineNo">583</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.583"></a>
-<span class="sourceLineNo">584</span>          resubmitted++;<a name="line.584"></a>
-<span class="sourceLineNo">585</span>        }<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      }<a name="line.586"></a>
-<span class="sourceLineNo">587</span>      if (tot &gt; 0) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.588"></a>
-<span class="sourceLineNo">589</span>        if (now &gt; lastLog + 5000) {<a name="line.589"></a>
-<span class="sourceLineNo">590</span>          lastLog = now;<a name="line.590"></a>
-<span class="sourceLineNo">591</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.591"></a>
-<span class="sourceLineNo">592</span>        }<a name="line.592"></a>
-<span class="sourceLineNo">593</span>      }<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      if (resubmitted &gt; 0) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.595"></a>
-<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
-<span class="sourceLineNo">597</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.598"></a>
-<span class="sourceLineNo">599</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // because a. it is very unlikely that every worker had a<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      // transient error when trying to grab the task b. if there are no<a name="line.601"></a>
-<span class="sourceLineNo">602</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.604"></a>
-<span class="sourceLineNo">605</span>      // that there is always one worker in the system<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      if (tot &gt; 0<a name="line.606"></a>
-<span class="sourceLineNo">607</span>          &amp;&amp; !found_assigned_task<a name="line.607"></a>
-<span class="sourceLineNo">608</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.608"></a>
-<span class="sourceLineNo">609</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.609"></a>
-<span class="sourceLineNo">610</span>          String key = e.getKey();<a name="line.610"></a>
-<span class="sourceLineNo">611</span>          Task task = e.getValue();<a name="line.611"></a>
-<span class="sourceLineNo">612</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.612"></a>
-<span class="sourceLineNo">613</span>          // have been asynchronously assigned. There is no locking required<a name="line.613"></a>
-<span class="sourceLineNo">614</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.614"></a>
-<span class="sourceLineNo">615</span>          // called unnecessarily for a taskpath<a name="line.615"></a>
-<span class="sourceLineNo">616</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>            // We just touch the znode to make sure its still there<a name="line.617"></a>
-<span class="sourceLineNo">618</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.618"></a>
-<span class="sourceLineNo">619</span>          }<a name="line.619"></a>
-<span class="sourceLineNo">620</span>        }<a name="line.620"></a>
-<span class="sourceLineNo">621</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.621"></a>
-<span class="sourceLineNo">622</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.623"></a>
-<span class="sourceLineNo">624</span>      }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      Set&lt;String&gt; failedDeletions =<a name="line.625"></a>
-<span class="sourceLineNo">626</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      // Retry previously failed deletes<a name="line.627"></a>
-<span class="sourceLineNo">628</span>      if (failedDeletions.size() &gt; 0) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.629"></a>
-<span class="sourceLineNo">630</span>        for (String tmpPath : tmpPaths) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>          // deleteNode is an async call<a name="line.631"></a>
-<span class="sourceLineNo">632</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.632"></a>
-<span class="sourceLineNo">633</span>        }<a name="line.633"></a>
-<span class="sourceLineNo">634</span>        failedDeletions.removeAll(tmpPaths);<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      }<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span>  }<a name="line.637"></a>
-<span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>  public enum ResubmitDirective {<a name="line.639"></a>
-<span class="sourceLineNo">640</span>    CHECK(), FORCE()<a name="line.640"></a>
+<span class="sourceLineNo">548</span>    @Override<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    protected void chore() {<a name="line.549"></a>
+<span class="sourceLineNo">550</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>      int resubmitted = 0;<a name="line.552"></a>
+<span class="sourceLineNo">553</span>      int unassigned = 0;<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      int tot = 0;<a name="line.554"></a>
+<span class="sourceLineNo">555</span>      boolean found_assigned_task = false;<a name="line.555"></a>
+<span class="sourceLineNo">556</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>      synchronized (deadWorkersLock) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>        localDeadWorkers = deadWorkers;<a name="line.559"></a>
+<span class="sourceLineNo">560</span>        deadWorkers = null;<a name="line.560"></a>
+<span class="sourceLineNo">561</span>      }<a name="line.561"></a>
+<span class="sourceLineNo">562</span><a name="line.562"></a>
+<span class="sourceLineNo">563</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.563"></a>
+<span class="sourceLineNo">564</span>        String path = e.getKey();<a name="line.564"></a>
+<span class="sourceLineNo">565</span>        Task task = e.getValue();<a name="line.565"></a>
+<span class="sourceLineNo">566</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.566"></a>
+<span class="sourceLineNo">567</span>        tot++;<a name="line.567"></a>
+<span class="sourceLineNo">568</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.568"></a>
+<span class="sourceLineNo">569</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.569"></a>
+<span class="sourceLineNo">570</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.570"></a>
+<span class="sourceLineNo">571</span>        // time. If we want progress when there are no region servers then we<a name="line.571"></a>
+<span class="sourceLineNo">572</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.572"></a>
+<span class="sourceLineNo">573</span>        if (task.isUnassigned()) {<a name="line.573"></a>
+<span class="sourceLineNo">574</span>          unassigned++;<a name="line.574"></a>
+<span class="sourceLineNo">575</span>          continue;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>        }<a name="line.576"></a>
+<span class="sourceLineNo">577</span>        found_assigned_task = true;<a name="line.577"></a>
+<span class="sourceLineNo">578</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.578"></a>
+<span class="sourceLineNo">579</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.579"></a>
+<span class="sourceLineNo">580</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.580"></a>
+<span class="sourceLineNo">581</span>            resubmitted++;<a name="line.581"></a>
+<span class="sourceLineNo">582</span>          } else {<a name="line.582"></a>
+<span class="sourceLineNo">583</span>            handleDeadWorker(cur_worker);<a name="line.583"></a>
+<span class="sourceLineNo">584</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.584"></a>
+<span class="sourceLineNo">585</span>                + ", will retry.");<a name="line.585"></a>
+<span class="sourceLineNo">586</span>          }<a name="line.586"></a>
+<span class="sourceLineNo">587</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>          resubmitted++;<a name="line.588"></a>
+<span class="sourceLineNo">589</span>        }<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      }<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      if (tot &gt; 0) {<a name="line.591"></a>
+<span class="sourceLineNo">592</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.592"></a>
+<span class="sourceLineNo">593</span>        if (now &gt; lastLog + 5000) {<a name="line.593"></a>
+<span class="sourceLineNo">594</span>          lastLog = now;<a name="line.594"></a>
+<span class="sourceLineNo">595</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.595"></a>
+<span class="sourceLineNo">596</span>        }<a name="line.596"></a>
+<span class="sourceLineNo">597</span>      }<a name="line.597"></a>
+<span class="sourceLineNo">598</span>      if (resubmitted &gt; 0) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.599"></a>
+<span class="sourceLineNo">600</span>      }<a name="line.600"></a>
+<span class="sourceLineNo">601</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.602"></a>
+<span class="sourceLineNo">603</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.603"></a>
+<span class="sourceLineNo">604</span>      // because a. it is very unlikely that every worker had a<a name="line.604"></a>
+<span class="sourceLineNo">605</span>      // transient error when trying to grab the task b. if there are no<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.608"></a>
+<span class="sourceLineNo">609</span>      // that there is always one worker in the system<a name="line.609"></a>
+<span class="sourceLineNo">610</span>      if (tot &gt; 0<a name="line.610"></a>
+<span class="sourceLineNo">611</span>          &amp;&amp; !found_assigned_task<a name="line.611"></a>
+<span class="sourceLineNo">612</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>          String key = e.getKey();<a name="line.614"></a>
+<span class="sourceLineNo">615</span>          Task task = e.getValue();<a name="line.615"></a>
+<span class="sourceLineNo">616</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.616"></a>
+<span class="sourceLineNo">617</span>          // have been asynchronously assigned. There is no locking required<a name="line.617"></a>
+<span class="sourceLineNo">618</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.618"></a>
+<span class="sourceLineNo">619</span>          // called unnecessarily for a taskpath<a name="line.619"></a>
+<span class="sourceLineNo">620</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.620"></a>
+<span class="sourceLineNo">621</span>            // We just touch the znode to make sure its still there<a name="line.621"></a>
+<span class="sourceLineNo">622</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.622"></a>
+<span class="sourceLineNo">623</span>          }<a name="line.623"></a>
+<span class="sourceLineNo">624</span>        }<a name="line.624"></a>
+<span class="sourceLineNo">625</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.625"></a>
+<span class="sourceLineNo">626</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.626"></a>
+<span class="sourceLineNo">627</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      }<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      Set&lt;String&gt; failedDeletions =<a name="line.629"></a>
+<span class="sourceLineNo">630</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.630"></a>
+<span class="sourceLineNo">631</span>      // Retry previously failed deletes<a name="line.631"></a>
+<span class="sourceLineNo">632</span>      if (failedDeletions.size() &gt; 0) {<a name="line.632"></a>
+<span class="sourceLineNo">633</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.633"></a>
+<span class="sourceLineNo">634</span>        for (String tmpPath : tmpPaths) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>          // deleteNode is an async call<a name="line.635"></a>
+<span class="sourceLineNo">636</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.636"></a>
+<span class="sourceLineNo">637</span>        }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>        failedDeletions.removeAll(tmpPaths);<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      }<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    }<a name="line.640"></a>
 <span class="sourceLineNo">641</span>  }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>  public enum TerminationStatus {<a name="line.643"></a>
-<span class="sourceLineNo">644</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.644"></a>
-<span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    final String statusMsg;<a name="line.646"></a>
-<span class="sourceLineNo">647</span><a name="line.647"></a>
-<span class="sourceLineNo">648</span>    TerminationStatus(String msg) {<a name="line.648"></a>
-<span class="sourceLineNo">649</span>      statusMsg = msg;<a name="line.649"></a>
-<span class="sourceLineNo">650</span>    }<a name="line.650"></a>
+<span class="sourceLineNo">643</span>  public enum ResubmitDirective {<a name="line.643"></a>
+<span class="sourceLineNo">644</span>    CHECK(), FORCE()<a name="line.644"></a>
+<span class="sourceLineNo">645</span>  }<a name="line.645"></a>
+<span class="sourceLineNo">646</span><a name="line.646"></a>
+<span class="sourceLineNo">647</span>  public enum TerminationStatus {<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.648"></a>
+<span class="sourceLineNo">649</span><a name="line.649"></a>
+<span class="sourceLineNo">650</span>    final String statusMsg;<a name="line.650"></a>
 <span class="sourceLineNo">651</span><a name="line.651"></a>
-<span class="sourceLineNo">652</span>    @Override<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    public String toString() {<a name="line.653"></a>
-<span class="sourceLineNo">654</span>      return statusMsg;<a name="line.654"></a>
-<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
-<span class="sourceLineNo">657</span>}<a name="line.657"></a>
+<span class="sourceLineNo">652</span>    TerminationStatus(String msg) {<a name="line.652"></a>
+<span class="sourceLineNo">653</span>      statusMsg = msg;<a name="line.653"></a>
+<span class="sourceLineNo">654</span>    }<a name="line.654"></a>
+<span class="sourceLineNo">655</span><a name="line.655"></a>
+<span class="sourceLineNo">656</span>    @Override<a name="line.656"></a>
+<span class="sourceLineNo">657</span>    public String toString() {<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      return statusMsg;<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    }<a name="line.659"></a>
+<span class="sourceLineNo">660</span>  }<a name="line.660"></a>
+<span class="sourceLineNo">661</span>}<a name="line.661"></a>
 
 
 


[11/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html
index 681e263..7cdf584 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html
@@ -182,7 +182,7 @@
 <span class="sourceLineNo">174</span>      }<a name="line.174"></a>
 <span class="sourceLineNo">175</span>      FileStatus[] logfiles = FSUtils.listStatus(fs, logDir, filter);<a name="line.175"></a>
 <span class="sourceLineNo">176</span>      if (logfiles == null || logfiles.length == 0) {<a name="line.176"></a>
-<span class="sourceLineNo">177</span>        LOG.info(logDir + " is empty dir, no logs to split");<a name="line.177"></a>
+<span class="sourceLineNo">177</span>        LOG.info("{} dir is empty, no logs to split.", logDir);<a name="line.177"></a>
 <span class="sourceLineNo">178</span>      } else {<a name="line.178"></a>
 <span class="sourceLineNo">179</span>        Collections.addAll(fileStatus, logfiles);<a name="line.179"></a>
 <span class="sourceLineNo">180</span>      }<a name="line.180"></a>
@@ -243,426 +243,430 @@
 <span class="sourceLineNo">235</span>      PathFilter filter) throws IOException {<a name="line.235"></a>
 <span class="sourceLineNo">236</span>    MonitoredTask status = TaskMonitor.get().createStatus("Doing distributed log split in " +<a name="line.236"></a>
 <span class="sourceLineNo">237</span>      logDirs + " for serverName=" + serverNames);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    status.setStatus("Checking directory contents...");<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.240"></a>
-<span class="sourceLineNo">241</span>    LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.241"></a>
-<span class="sourceLineNo">242</span>      " for " + serverNames);<a name="line.242"></a>
-<span class="sourceLineNo">243</span>    long t = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
-<span class="sourceLineNo">244</span>    long totalSize = 0;<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    TaskBatch batch = new TaskBatch();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    for (FileStatus lf : logfiles) {<a name="line.246"></a>
-<span class="sourceLineNo">247</span>      // TODO If the log file is still being written to - which is most likely<a name="line.247"></a>
-<span class="sourceLineNo">248</span>      // the case for the last log file - then its length will show up here<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      // as zero. The size of such a file can only be retrieved after<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      // recover-lease is done. totalSize will be under in most cases and the<a name="line.250"></a>
-<span class="sourceLineNo">251</span>      // metrics that it drives will also be under-reported.<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      totalSize += lf.getLen();<a name="line.252"></a>
-<span class="sourceLineNo">253</span>      String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.253"></a>
-<span class="sourceLineNo">254</span>      if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.254"></a>
-<span class="sourceLineNo">255</span>        throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.255"></a>
-<span class="sourceLineNo">256</span>      }<a name="line.256"></a>
-<span class="sourceLineNo">257</span>    }<a name="line.257"></a>
-<span class="sourceLineNo">258</span>    waitForSplittingCompletion(batch, status);<a name="line.258"></a>
-<span class="sourceLineNo">259</span><a name="line.259"></a>
-<span class="sourceLineNo">260</span>    if (batch.done != batch.installed) {<a name="line.260"></a>
-<span class="sourceLineNo">261</span>      batch.isDead = true;<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.262"></a>
-<span class="sourceLineNo">263</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.263"></a>
-<span class="sourceLineNo">264</span>          + " but only " + batch.done + " done");<a name="line.264"></a>
-<span class="sourceLineNo">265</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      status.abort(msg);<a name="line.266"></a>
-<span class="sourceLineNo">267</span>      throw new IOException(msg);<a name="line.267"></a>
-<span class="sourceLineNo">268</span>    }<a name="line.268"></a>
-<span class="sourceLineNo">269</span>    for (Path logDir : logDirs) {<a name="line.269"></a>
-<span class="sourceLineNo">270</span>      status.setStatus("Cleaning up log directory...");<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      try {<a name="line.272"></a>
-<span class="sourceLineNo">273</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.273"></a>
-<span class="sourceLineNo">274</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.274"></a>
-<span class="sourceLineNo">275</span>        }<a name="line.275"></a>
-<span class="sourceLineNo">276</span>      } catch (IOException ioe) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.277"></a>
-<span class="sourceLineNo">278</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.278"></a>
-<span class="sourceLineNo">279</span>          LOG.warn("Returning success without actually splitting and "<a name="line.279"></a>
-<span class="sourceLineNo">280</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.280"></a>
-<span class="sourceLineNo">281</span>              + Arrays.toString(files), ioe);<a name="line.281"></a>
-<span class="sourceLineNo">282</span>        } else {<a name="line.282"></a>
-<span class="sourceLineNo">283</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.283"></a>
-<span class="sourceLineNo">284</span>        }<a name="line.284"></a>
-<span class="sourceLineNo">285</span>      }<a name="line.285"></a>
-<span class="sourceLineNo">286</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.286"></a>
-<span class="sourceLineNo">287</span>    }<a name="line.287"></a>
-<span class="sourceLineNo">288</span>    String msg =<a name="line.288"></a>
-<span class="sourceLineNo">289</span>        "finished splitting (more than or equal to) " + totalSize + " bytes in " + batch.installed<a name="line.289"></a>
-<span class="sourceLineNo">290</span>            + " log files in " + logDirs + " in "<a name="line.290"></a>
-<span class="sourceLineNo">291</span>            + (EnvironmentEdgeManager.currentTime() - t) + "ms";<a name="line.291"></a>
-<span class="sourceLineNo">292</span>    status.markComplete(msg);<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    LOG.info(msg);<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    return totalSize;<a name="line.294"></a>
-<span class="sourceLineNo">295</span>  }<a name="line.295"></a>
-<span class="sourceLineNo">296</span><a name="line.296"></a>
-<span class="sourceLineNo">297</span>  /**<a name="line.297"></a>
-<span class="sourceLineNo">298</span>   * Add a task entry to coordination if it is not already there.<a name="line.298"></a>
-<span class="sourceLineNo">299</span>   * @param taskname the path of the log to be split<a name="line.299"></a>
-<span class="sourceLineNo">300</span>   * @param batch the batch this task belongs to<a name="line.300"></a>
-<span class="sourceLineNo">301</span>   * @return true if a new entry is created, false if it is already there.<a name="line.301"></a>
-<span class="sourceLineNo">302</span>   */<a name="line.302"></a>
-<span class="sourceLineNo">303</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.303"></a>
-<span class="sourceLineNo">304</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.304"></a>
-<span class="sourceLineNo">305</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.305"></a>
-<span class="sourceLineNo">306</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.306"></a>
-<span class="sourceLineNo">307</span>    if (oldtask == null) {<a name="line.307"></a>
-<span class="sourceLineNo">308</span>      // publish the task in the coordination engine<a name="line.308"></a>
-<span class="sourceLineNo">309</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.309"></a>
-<span class="sourceLineNo">310</span>      return true;<a name="line.310"></a>
-<span class="sourceLineNo">311</span>    }<a name="line.311"></a>
-<span class="sourceLineNo">312</span>    return false;<a name="line.312"></a>
-<span class="sourceLineNo">313</span>  }<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.315"></a>
-<span class="sourceLineNo">316</span>    synchronized (batch) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.317"></a>
-<span class="sourceLineNo">318</span>        try {<a name="line.318"></a>
-<span class="sourceLineNo">319</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.319"></a>
-<span class="sourceLineNo">320</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.320"></a>
-<span class="sourceLineNo">321</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.321"></a>
-<span class="sourceLineNo">322</span>          int actual = activeTasks(batch);<a name="line.322"></a>
-<span class="sourceLineNo">323</span>          if (remaining != actual) {<a name="line.323"></a>
-<span class="sourceLineNo">324</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.324"></a>
-<span class="sourceLineNo">325</span>          }<a name="line.325"></a>
-<span class="sourceLineNo">326</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.326"></a>
-<span class="sourceLineNo">327</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.327"></a>
-<span class="sourceLineNo">328</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.328"></a>
-<span class="sourceLineNo">329</span>                + remainingTasks);<a name="line.329"></a>
-<span class="sourceLineNo">330</span>          }<a name="line.330"></a>
-<span class="sourceLineNo">331</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span>            LOG.warn("No more task remaining, splitting "<a name="line.332"></a>
-<span class="sourceLineNo">333</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.333"></a>
-<span class="sourceLineNo">334</span>                + ", active tasks in map " + actual);<a name="line.334"></a>
-<span class="sourceLineNo">335</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.335"></a>
-<span class="sourceLineNo">336</span>              return;<a name="line.336"></a>
-<span class="sourceLineNo">337</span>            }<a name="line.337"></a>
-<span class="sourceLineNo">338</span>          }<a name="line.338"></a>
-<span class="sourceLineNo">339</span>          batch.wait(100);<a name="line.339"></a>
-<span class="sourceLineNo">340</span>          if (server.isStopped()) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.341"></a>
-<span class="sourceLineNo">342</span>            return;<a name="line.342"></a>
-<span class="sourceLineNo">343</span>          }<a name="line.343"></a>
-<span class="sourceLineNo">344</span>        } catch (InterruptedException e) {<a name="line.344"></a>
-<span class="sourceLineNo">345</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.345"></a>
-<span class="sourceLineNo">346</span>          Thread.currentThread().interrupt();<a name="line.346"></a>
-<span class="sourceLineNo">347</span>          return;<a name="line.347"></a>
-<span class="sourceLineNo">348</span>        }<a name="line.348"></a>
-<span class="sourceLineNo">349</span>      }<a name="line.349"></a>
-<span class="sourceLineNo">350</span>    }<a name="line.350"></a>
-<span class="sourceLineNo">351</span>  }<a name="line.351"></a>
-<span class="sourceLineNo">352</span><a name="line.352"></a>
-<span class="sourceLineNo">353</span>  @VisibleForTesting<a name="line.353"></a>
-<span class="sourceLineNo">354</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.354"></a>
-<span class="sourceLineNo">355</span>    return tasks;<a name="line.355"></a>
-<span class="sourceLineNo">356</span>  }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span>  private int activeTasks(final TaskBatch batch) {<a name="line.358"></a>
-<span class="sourceLineNo">359</span>    int count = 0;<a name="line.359"></a>
-<span class="sourceLineNo">360</span>    for (Task t : tasks.values()) {<a name="line.360"></a>
-<span class="sourceLineNo">361</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.361"></a>
-<span class="sourceLineNo">362</span>        count++;<a name="line.362"></a>
-<span class="sourceLineNo">363</span>      }<a name="line.363"></a>
-<span class="sourceLineNo">364</span>    }<a name="line.364"></a>
-<span class="sourceLineNo">365</span>    return count;<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span>  }<a name="line.367"></a>
-<span class="sourceLineNo">368</span><a name="line.368"></a>
-<span class="sourceLineNo">369</span>  /**<a name="line.369"></a>
-<span class="sourceLineNo">370</span>   * @param path<a name="line.370"></a>
-<span class="sourceLineNo">371</span>   * @param batch<a name="line.371"></a>
-<span class="sourceLineNo">372</span>   * @return null on success, existing task on error<a name="line.372"></a>
-<span class="sourceLineNo">373</span>   */<a name="line.373"></a>
-<span class="sourceLineNo">374</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span>    Task oldtask;<a name="line.375"></a>
-<span class="sourceLineNo">376</span>    // batch.installed is only changed via this function and<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    // a single thread touches batch.installed.<a name="line.377"></a>
-<span class="sourceLineNo">378</span>    Task newtask = new Task();<a name="line.378"></a>
-<span class="sourceLineNo">379</span>    newtask.batch = batch;<a name="line.379"></a>
-<span class="sourceLineNo">380</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    if (oldtask == null) {<a name="line.381"></a>
-<span class="sourceLineNo">382</span>      batch.installed++;<a name="line.382"></a>
-<span class="sourceLineNo">383</span>      return null;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    }<a name="line.384"></a>
-<span class="sourceLineNo">385</span>    // new task was not used.<a name="line.385"></a>
-<span class="sourceLineNo">386</span>    synchronized (oldtask) {<a name="line.386"></a>
-<span class="sourceLineNo">387</span>      if (oldtask.isOrphan()) {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>        if (oldtask.status == SUCCESS) {<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          // The task is already done. Do not install the batch for this<a name="line.389"></a>
-<span class="sourceLineNo">390</span>          // task because it might be too late for setDone() to update<a name="line.390"></a>
-<span class="sourceLineNo">391</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.391"></a>
-<span class="sourceLineNo">392</span>          // this task to complete.<a name="line.392"></a>
-<span class="sourceLineNo">393</span>          return (null);<a name="line.393"></a>
-<span class="sourceLineNo">394</span>        }<a name="line.394"></a>
-<span class="sourceLineNo">395</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.395"></a>
-<span class="sourceLineNo">396</span>          oldtask.batch = batch;<a name="line.396"></a>
-<span class="sourceLineNo">397</span>          batch.installed++;<a name="line.397"></a>
-<span class="sourceLineNo">398</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.398"></a>
-<span class="sourceLineNo">399</span>          return null;<a name="line.399"></a>
-<span class="sourceLineNo">400</span>        }<a name="line.400"></a>
-<span class="sourceLineNo">401</span>        while (oldtask.status == FAILURE) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.402"></a>
-<span class="sourceLineNo">403</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.403"></a>
-<span class="sourceLineNo">404</span>          try {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>            oldtask.wait();<a name="line.405"></a>
-<span class="sourceLineNo">406</span>          } catch (InterruptedException e) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span>            Thread.currentThread().interrupt();<a name="line.407"></a>
-<span class="sourceLineNo">408</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.408"></a>
-<span class="sourceLineNo">409</span>            // fall through to return failure<a name="line.409"></a>
-<span class="sourceLineNo">410</span>            break;<a name="line.410"></a>
-<span class="sourceLineNo">411</span>          }<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        }<a name="line.412"></a>
-<span class="sourceLineNo">413</span>        if (oldtask.status != DELETED) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span>          LOG.warn("Failure because previously failed task"<a name="line.414"></a>
-<span class="sourceLineNo">415</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.415"></a>
-<span class="sourceLineNo">416</span>          return oldtask;<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        }<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        // reinsert the newTask and it must succeed this time<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        if (t == null) {<a name="line.420"></a>
-<span class="sourceLineNo">421</span>          batch.installed++;<a name="line.421"></a>
-<span class="sourceLineNo">422</span>          return null;<a name="line.422"></a>
-<span class="sourceLineNo">423</span>        }<a name="line.423"></a>
-<span class="sourceLineNo">424</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        assert false : "Deleted task still present in tasks map";<a name="line.425"></a>
-<span class="sourceLineNo">426</span>        return t;<a name="line.426"></a>
-<span class="sourceLineNo">427</span>      }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.428"></a>
-<span class="sourceLineNo">429</span>      return oldtask;<a name="line.429"></a>
-<span class="sourceLineNo">430</span>    }<a name="line.430"></a>
-<span class="sourceLineNo">431</span>  }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span>  public void stop() {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (choreService != null) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      choreService.shutdown();<a name="line.435"></a>
-<span class="sourceLineNo">436</span>    }<a name="line.436"></a>
-<span class="sourceLineNo">437</span>    if (timeoutMonitor != null) {<a name="line.437"></a>
-<span class="sourceLineNo">438</span>      timeoutMonitor.cancel(true);<a name="line.438"></a>
-<span class="sourceLineNo">439</span>    }<a name="line.439"></a>
-<span class="sourceLineNo">440</span>  }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>  void handleDeadWorker(ServerName workerName) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.443"></a>
-<span class="sourceLineNo">444</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.444"></a>
-<span class="sourceLineNo">445</span>    synchronized (deadWorkersLock) {<a name="line.445"></a>
-<span class="sourceLineNo">446</span>      if (deadWorkers == null) {<a name="line.446"></a>
-<span class="sourceLineNo">447</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>      }<a name="line.448"></a>
-<span class="sourceLineNo">449</span>      deadWorkers.add(workerName);<a name="line.449"></a>
-<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    LOG.info("dead splitlog worker " + workerName);<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  }<a name="line.452"></a>
-<span class="sourceLineNo">453</span><a name="line.453"></a>
-<span class="sourceLineNo">454</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.454"></a>
-<span class="sourceLineNo">455</span>    synchronized (deadWorkersLock) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span>      if (deadWorkers == null) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      }<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      deadWorkers.addAll(serverNames);<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.461"></a>
-<span class="sourceLineNo">462</span>  }<a name="line.462"></a>
-<span class="sourceLineNo">463</span><a name="line.463"></a>
-<span class="sourceLineNo">464</span>  /**<a name="line.464"></a>
-<span class="sourceLineNo">465</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.465"></a>
-<span class="sourceLineNo">466</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.466"></a>
-<span class="sourceLineNo">467</span>   * &lt;p&gt;<a name="line.467"></a>
-<span class="sourceLineNo">468</span>   * All access is synchronized.<a name="line.468"></a>
-<span class="sourceLineNo">469</span>   */<a name="line.469"></a>
-<span class="sourceLineNo">470</span>  @InterfaceAudience.Private<a name="line.470"></a>
-<span class="sourceLineNo">471</span>  public static class TaskBatch {<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    public int installed = 0;<a name="line.472"></a>
-<span class="sourceLineNo">473</span>    public int done = 0;<a name="line.473"></a>
-<span class="sourceLineNo">474</span>    public int error = 0;<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    public volatile boolean isDead = false;<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span>    @Override<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    public String toString() {<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.479"></a>
-<span class="sourceLineNo">480</span>    }<a name="line.480"></a>
-<span class="sourceLineNo">481</span>  }<a name="line.481"></a>
-<span class="sourceLineNo">482</span><a name="line.482"></a>
-<span class="sourceLineNo">483</span>  /**<a name="line.483"></a>
-<span class="sourceLineNo">484</span>   * in memory state of an active task.<a name="line.484"></a>
-<span class="sourceLineNo">485</span>   */<a name="line.485"></a>
-<span class="sourceLineNo">486</span>  @InterfaceAudience.Private<a name="line.486"></a>
-<span class="sourceLineNo">487</span>  public static class Task {<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    public volatile long last_update;<a name="line.488"></a>
-<span class="sourceLineNo">489</span>    public volatile int last_version;<a name="line.489"></a>
-<span class="sourceLineNo">490</span>    public volatile ServerName cur_worker_name;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>    public volatile TaskBatch batch;<a name="line.491"></a>
-<span class="sourceLineNo">492</span>    public volatile TerminationStatus status;<a name="line.492"></a>
-<span class="sourceLineNo">493</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.493"></a>
-<span class="sourceLineNo">494</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    public volatile boolean resubmitThresholdReached;<a name="line.495"></a>
-<span class="sourceLineNo">496</span><a name="line.496"></a>
-<span class="sourceLineNo">497</span>    @Override<a name="line.497"></a>
-<span class="sourceLineNo">498</span>    public String toString() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.499"></a>
-<span class="sourceLineNo">500</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.500"></a>
-<span class="sourceLineNo">501</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.501"></a>
-<span class="sourceLineNo">502</span>    }<a name="line.502"></a>
-<span class="sourceLineNo">503</span><a name="line.503"></a>
-<span class="sourceLineNo">504</span>    public Task() {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      last_version = -1;<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      status = IN_PROGRESS;<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      setUnassigned();<a name="line.507"></a>
-<span class="sourceLineNo">508</span>    }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span>    public boolean isOrphan() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      return (batch == null || batch.isDead);<a name="line.511"></a>
+<span class="sourceLineNo">238</span>    long totalSize = 0;<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    TaskBatch batch = null;<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    long startTime = 0;<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.241"></a>
+<span class="sourceLineNo">242</span>    if (logfiles.length != 0) {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>      status.setStatus("Checking directory contents...");<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.245"></a>
+<span class="sourceLineNo">246</span>          " for " + serverNames);<a name="line.246"></a>
+<span class="sourceLineNo">247</span>      startTime = EnvironmentEdgeManager.currentTime();<a name="line.247"></a>
+<span class="sourceLineNo">248</span>      batch = new TaskBatch();<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      for (FileStatus lf : logfiles) {<a name="line.249"></a>
+<span class="sourceLineNo">250</span>        // TODO If the log file is still being written to - which is most likely<a name="line.250"></a>
+<span class="sourceLineNo">251</span>        // the case for the last log file - then its length will show up here<a name="line.251"></a>
+<span class="sourceLineNo">252</span>        // as zero. The size of such a file can only be retrieved after<a name="line.252"></a>
+<span class="sourceLineNo">253</span>        // recover-lease is done. totalSize will be under in most cases and the<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        // metrics that it drives will also be under-reported.<a name="line.254"></a>
+<span class="sourceLineNo">255</span>        totalSize += lf.getLen();<a name="line.255"></a>
+<span class="sourceLineNo">256</span>        String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.257"></a>
+<span class="sourceLineNo">258</span>          throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.258"></a>
+<span class="sourceLineNo">259</span>        }<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      }<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      waitForSplittingCompletion(batch, status);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    if (batch != null &amp;&amp; batch.done != batch.installed) {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      batch.isDead = true;<a name="line.265"></a>
+<span class="sourceLineNo">266</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.267"></a>
+<span class="sourceLineNo">268</span>          + " but only " + batch.done + " done");<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.269"></a>
+<span class="sourceLineNo">270</span>      status.abort(msg);<a name="line.270"></a>
+<span class="sourceLineNo">271</span>      throw new IOException(msg);<a name="line.271"></a>
+<span class="sourceLineNo">272</span>    }<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    for (Path logDir : logDirs) {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      status.setStatus("Cleaning up log directory...");<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.275"></a>
+<span class="sourceLineNo">276</span>      try {<a name="line.276"></a>
+<span class="sourceLineNo">277</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.277"></a>
+<span class="sourceLineNo">278</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.278"></a>
+<span class="sourceLineNo">279</span>        }<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      } catch (IOException ioe) {<a name="line.280"></a>
+<span class="sourceLineNo">281</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.281"></a>
+<span class="sourceLineNo">282</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.282"></a>
+<span class="sourceLineNo">283</span>          LOG.warn("Returning success without actually splitting and "<a name="line.283"></a>
+<span class="sourceLineNo">284</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.284"></a>
+<span class="sourceLineNo">285</span>              + Arrays.toString(files), ioe);<a name="line.285"></a>
+<span class="sourceLineNo">286</span>        } else {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.287"></a>
+<span class="sourceLineNo">288</span>        }<a name="line.288"></a>
+<span class="sourceLineNo">289</span>      }<a name="line.289"></a>
+<span class="sourceLineNo">290</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.290"></a>
+<span class="sourceLineNo">291</span>    }<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    String msg = "Finished splitting (more than or equal to) " + totalSize +<a name="line.292"></a>
+<span class="sourceLineNo">293</span>        " bytes in " + ((batch == null)? 0: batch.installed) +<a name="line.293"></a>
+<span class="sourceLineNo">294</span>        " log files in " + logDirs + " in " +<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        ((startTime == 0)? startTime: (EnvironmentEdgeManager.currentTime() - startTime)) + "ms";<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    status.markComplete(msg);<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    LOG.info(msg);<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    return totalSize;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
+<span class="sourceLineNo">300</span><a name="line.300"></a>
+<span class="sourceLineNo">301</span>  /**<a name="line.301"></a>
+<span class="sourceLineNo">302</span>   * Add a task entry to coordination if it is not already there.<a name="line.302"></a>
+<span class="sourceLineNo">303</span>   * @param taskname the path of the log to be split<a name="line.303"></a>
+<span class="sourceLineNo">304</span>   * @param batch the batch this task belongs to<a name="line.304"></a>
+<span class="sourceLineNo">305</span>   * @return true if a new entry is created, false if it is already there.<a name="line.305"></a>
+<span class="sourceLineNo">306</span>   */<a name="line.306"></a>
+<span class="sourceLineNo">307</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.307"></a>
+<span class="sourceLineNo">308</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.309"></a>
+<span class="sourceLineNo">310</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.310"></a>
+<span class="sourceLineNo">311</span>    if (oldtask == null) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span>      // publish the task in the coordination engine<a name="line.312"></a>
+<span class="sourceLineNo">313</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.313"></a>
+<span class="sourceLineNo">314</span>      return true;<a name="line.314"></a>
+<span class="sourceLineNo">315</span>    }<a name="line.315"></a>
+<span class="sourceLineNo">316</span>    return false;<a name="line.316"></a>
+<span class="sourceLineNo">317</span>  }<a name="line.317"></a>
+<span class="sourceLineNo">318</span><a name="line.318"></a>
+<span class="sourceLineNo">319</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    synchronized (batch) {<a name="line.320"></a>
+<span class="sourceLineNo">321</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.321"></a>
+<span class="sourceLineNo">322</span>        try {<a name="line.322"></a>
+<span class="sourceLineNo">323</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.323"></a>
+<span class="sourceLineNo">324</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.324"></a>
+<span class="sourceLineNo">325</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.325"></a>
+<span class="sourceLineNo">326</span>          int actual = activeTasks(batch);<a name="line.326"></a>
+<span class="sourceLineNo">327</span>          if (remaining != actual) {<a name="line.327"></a>
+<span class="sourceLineNo">328</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.328"></a>
+<span class="sourceLineNo">329</span>          }<a name="line.329"></a>
+<span class="sourceLineNo">330</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.330"></a>
+<span class="sourceLineNo">331</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.331"></a>
+<span class="sourceLineNo">332</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.332"></a>
+<span class="sourceLineNo">333</span>                + remainingTasks);<a name="line.333"></a>
+<span class="sourceLineNo">334</span>          }<a name="line.334"></a>
+<span class="sourceLineNo">335</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.335"></a>
+<span class="sourceLineNo">336</span>            LOG.warn("No more task remaining, splitting "<a name="line.336"></a>
+<span class="sourceLineNo">337</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.337"></a>
+<span class="sourceLineNo">338</span>                + ", active tasks in map " + actual);<a name="line.338"></a>
+<span class="sourceLineNo">339</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.339"></a>
+<span class="sourceLineNo">340</span>              return;<a name="line.340"></a>
+<span class="sourceLineNo">341</span>            }<a name="line.341"></a>
+<span class="sourceLineNo">342</span>          }<a name="line.342"></a>
+<span class="sourceLineNo">343</span>          batch.wait(100);<a name="line.343"></a>
+<span class="sourceLineNo">344</span>          if (server.isStopped()) {<a name="line.344"></a>
+<span class="sourceLineNo">345</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.345"></a>
+<span class="sourceLineNo">346</span>            return;<a name="line.346"></a>
+<span class="sourceLineNo">347</span>          }<a name="line.347"></a>
+<span class="sourceLineNo">348</span>        } catch (InterruptedException e) {<a name="line.348"></a>
+<span class="sourceLineNo">349</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.349"></a>
+<span class="sourceLineNo">350</span>          Thread.currentThread().interrupt();<a name="line.350"></a>
+<span class="sourceLineNo">351</span>          return;<a name="line.351"></a>
+<span class="sourceLineNo">352</span>        }<a name="line.352"></a>
+<span class="sourceLineNo">353</span>      }<a name="line.353"></a>
+<span class="sourceLineNo">354</span>    }<a name="line.354"></a>
+<span class="sourceLineNo">355</span>  }<a name="line.355"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span>  @VisibleForTesting<a name="line.357"></a>
+<span class="sourceLineNo">358</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.358"></a>
+<span class="sourceLineNo">359</span>    return tasks;<a name="line.359"></a>
+<span class="sourceLineNo">360</span>  }<a name="line.360"></a>
+<span class="sourceLineNo">361</span><a name="line.361"></a>
+<span class="sourceLineNo">362</span>  private int activeTasks(final TaskBatch batch) {<a name="line.362"></a>
+<span class="sourceLineNo">363</span>    int count = 0;<a name="line.363"></a>
+<span class="sourceLineNo">364</span>    for (Task t : tasks.values()) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.365"></a>
+<span class="sourceLineNo">366</span>        count++;<a name="line.366"></a>
+<span class="sourceLineNo">367</span>      }<a name="line.367"></a>
+<span class="sourceLineNo">368</span>    }<a name="line.368"></a>
+<span class="sourceLineNo">369</span>    return count;<a name="line.369"></a>
+<span class="sourceLineNo">370</span><a name="line.370"></a>
+<span class="sourceLineNo">371</span>  }<a name="line.371"></a>
+<span class="sourceLineNo">372</span><a name="line.372"></a>
+<span class="sourceLineNo">373</span>  /**<a name="line.373"></a>
+<span class="sourceLineNo">374</span>   * @param path<a name="line.374"></a>
+<span class="sourceLineNo">375</span>   * @param batch<a name="line.375"></a>
+<span class="sourceLineNo">376</span>   * @return null on success, existing task on error<a name="line.376"></a>
+<span class="sourceLineNo">377</span>   */<a name="line.377"></a>
+<span class="sourceLineNo">378</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.378"></a>
+<span class="sourceLineNo">379</span>    Task oldtask;<a name="line.379"></a>
+<span class="sourceLineNo">380</span>    // batch.installed is only changed via this function and<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    // a single thread touches batch.installed.<a name="line.381"></a>
+<span class="sourceLineNo">382</span>    Task newtask = new Task();<a name="line.382"></a>
+<span class="sourceLineNo">383</span>    newtask.batch = batch;<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.384"></a>
+<span class="sourceLineNo">385</span>    if (oldtask == null) {<a name="line.385"></a>
+<span class="sourceLineNo">386</span>      batch.installed++;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      return null;<a name="line.387"></a>
+<span class="sourceLineNo">388</span>    }<a name="line.388"></a>
+<span class="sourceLineNo">389</span>    // new task was not used.<a name="line.389"></a>
+<span class="sourceLineNo">390</span>    synchronized (oldtask) {<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      if (oldtask.isOrphan()) {<a name="line.391"></a>
+<span class="sourceLineNo">392</span>        if (oldtask.status == SUCCESS) {<a name="line.392"></a>
+<span class="sourceLineNo">393</span>          // The task is already done. Do not install the batch for this<a name="line.393"></a>
+<span class="sourceLineNo">394</span>          // task because it might be too late for setDone() to update<a name="line.394"></a>
+<span class="sourceLineNo">395</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.395"></a>
+<span class="sourceLineNo">396</span>          // this task to complete.<a name="line.396"></a>
+<span class="sourceLineNo">397</span>          return (null);<a name="line.397"></a>
+<span class="sourceLineNo">398</span>        }<a name="line.398"></a>
+<span class="sourceLineNo">399</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.399"></a>
+<span class="sourceLineNo">400</span>          oldtask.batch = batch;<a name="line.400"></a>
+<span class="sourceLineNo">401</span>          batch.installed++;<a name="line.401"></a>
+<span class="sourceLineNo">402</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.402"></a>
+<span class="sourceLineNo">403</span>          return null;<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        }<a name="line.404"></a>
+<span class="sourceLineNo">405</span>        while (oldtask.status == FAILURE) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.406"></a>
+<span class="sourceLineNo">407</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.407"></a>
+<span class="sourceLineNo">408</span>          try {<a name="line.408"></a>
+<span class="sourceLineNo">409</span>            oldtask.wait();<a name="line.409"></a>
+<span class="sourceLineNo">410</span>          } catch (InterruptedException e) {<a name="line.410"></a>
+<span class="sourceLineNo">411</span>            Thread.currentThread().interrupt();<a name="line.411"></a>
+<span class="sourceLineNo">412</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.412"></a>
+<span class="sourceLineNo">413</span>            // fall through to return failure<a name="line.413"></a>
+<span class="sourceLineNo">414</span>            break;<a name="line.414"></a>
+<span class="sourceLineNo">415</span>          }<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        }<a name="line.416"></a>
+<span class="sourceLineNo">417</span>        if (oldtask.status != DELETED) {<a name="line.417"></a>
+<span class="sourceLineNo">418</span>          LOG.warn("Failure because previously failed task"<a name="line.418"></a>
+<span class="sourceLineNo">419</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.419"></a>
+<span class="sourceLineNo">420</span>          return oldtask;<a name="line.420"></a>
+<span class="sourceLineNo">421</span>        }<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        // reinsert the newTask and it must succeed this time<a name="line.422"></a>
+<span class="sourceLineNo">423</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        if (t == null) {<a name="line.424"></a>
+<span class="sourceLineNo">425</span>          batch.installed++;<a name="line.425"></a>
+<span class="sourceLineNo">426</span>          return null;<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        }<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.428"></a>
+<span class="sourceLineNo">429</span>        assert false : "Deleted task still present in tasks map";<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        return t;<a name="line.430"></a>
+<span class="sourceLineNo">431</span>      }<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.432"></a>
+<span class="sourceLineNo">433</span>      return oldtask;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>    }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>  }<a name="line.435"></a>
+<span class="sourceLineNo">436</span><a name="line.436"></a>
+<span class="sourceLineNo">437</span>  public void stop() {<a name="line.437"></a>
+<span class="sourceLineNo">438</span>    if (choreService != null) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span>      choreService.shutdown();<a name="line.439"></a>
+<span class="sourceLineNo">440</span>    }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    if (timeoutMonitor != null) {<a name="line.441"></a>
+<span class="sourceLineNo">442</span>      timeoutMonitor.cancel(true);<a name="line.442"></a>
+<span class="sourceLineNo">443</span>    }<a name="line.443"></a>
+<span class="sourceLineNo">444</span>  }<a name="line.444"></a>
+<span class="sourceLineNo">445</span><a name="line.445"></a>
+<span class="sourceLineNo">446</span>  void handleDeadWorker(ServerName workerName) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.447"></a>
+<span class="sourceLineNo">448</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.448"></a>
+<span class="sourceLineNo">449</span>    synchronized (deadWorkersLock) {<a name="line.449"></a>
+<span class="sourceLineNo">450</span>      if (deadWorkers == null) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      }<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      deadWorkers.add(workerName);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    }<a name="line.454"></a>
+<span class="sourceLineNo">455</span>    LOG.info("Dead splitlog worker {}", workerName);<a name="line.455"></a>
+<span class="sourceLineNo">456</span>  }<a name="line.456"></a>
+<span class="sourceLineNo">457</span><a name="line.457"></a>
+<span class="sourceLineNo">458</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>    synchronized (deadWorkersLock) {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>      if (deadWorkers == null) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.461"></a>
+<span class="sourceLineNo">462</span>      }<a name="line.462"></a>
+<span class="sourceLineNo">463</span>      deadWorkers.addAll(serverNames);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    }<a name="line.464"></a>
+<span class="sourceLineNo">465</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.465"></a>
+<span class="sourceLineNo">466</span>  }<a name="line.466"></a>
+<span class="sourceLineNo">467</span><a name="line.467"></a>
+<span class="sourceLineNo">468</span>  /**<a name="line.468"></a>
+<span class="sourceLineNo">469</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.469"></a>
+<span class="sourceLineNo">470</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.470"></a>
+<span class="sourceLineNo">471</span>   * &lt;p&gt;<a name="line.471"></a>
+<span class="sourceLineNo">472</span>   * All access is synchronized.<a name="line.472"></a>
+<span class="sourceLineNo">473</span>   */<a name="line.473"></a>
+<span class="sourceLineNo">474</span>  @InterfaceAudience.Private<a name="line.474"></a>
+<span class="sourceLineNo">475</span>  public static class TaskBatch {<a name="line.475"></a>
+<span class="sourceLineNo">476</span>    public int installed = 0;<a name="line.476"></a>
+<span class="sourceLineNo">477</span>    public int done = 0;<a name="line.477"></a>
+<span class="sourceLineNo">478</span>    public int error = 0;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    public volatile boolean isDead = false;<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span>    @Override<a name="line.481"></a>
+<span class="sourceLineNo">482</span>    public String toString() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    }<a name="line.484"></a>
+<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
+<span class="sourceLineNo">486</span><a name="line.486"></a>
+<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
+<span class="sourceLineNo">488</span>   * in memory state of an active task.<a name="line.488"></a>
+<span class="sourceLineNo">489</span>   */<a name="line.489"></a>
+<span class="sourceLineNo">490</span>  @InterfaceAudience.Private<a name="line.490"></a>
+<span class="sourceLineNo">491</span>  public static class Task {<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    public volatile long last_update;<a name="line.492"></a>
+<span class="sourceLineNo">493</span>    public volatile int last_version;<a name="line.493"></a>
+<span class="sourceLineNo">494</span>    public volatile ServerName cur_worker_name;<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    public volatile TaskBatch batch;<a name="line.495"></a>
+<span class="sourceLineNo">496</span>    public volatile TerminationStatus status;<a name="line.496"></a>
+<span class="sourceLineNo">497</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.497"></a>
+<span class="sourceLineNo">498</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.498"></a>
+<span class="sourceLineNo">499</span>    public volatile boolean resubmitThresholdReached;<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span>    @Override<a name="line.501"></a>
+<span class="sourceLineNo">502</span>    public String toString() {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.503"></a>
+<span class="sourceLineNo">504</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.504"></a>
+<span class="sourceLineNo">505</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.505"></a>
+<span class="sourceLineNo">506</span>    }<a name="line.506"></a>
+<span class="sourceLineNo">507</span><a name="line.507"></a>
+<span class="sourceLineNo">508</span>    public Task() {<a name="line.508"></a>
+<span class="sourceLineNo">509</span>      last_version = -1;<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      status = IN_PROGRESS;<a name="line.510"></a>
+<span class="sourceLineNo">511</span>      setUnassigned();<a name="line.511"></a>
 <span class="sourceLineNo">512</span>    }<a name="line.512"></a>
 <span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span>    public boolean isUnassigned() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span>      return (cur_worker_name == null);<a name="line.515"></a>
+<span class="sourceLineNo">514</span>    public boolean isOrphan() {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>      return (batch == null || batch.isDead);<a name="line.515"></a>
 <span class="sourceLineNo">516</span>    }<a name="line.516"></a>
 <span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span>    public void heartbeatNoDetails(long time) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span>      last_update = time;<a name="line.519"></a>
+<span class="sourceLineNo">518</span>    public boolean isUnassigned() {<a name="line.518"></a>
+<span class="sourceLineNo">519</span>      return (cur_worker_name == null);<a name="line.519"></a>
 <span class="sourceLineNo">520</span>    }<a name="line.520"></a>
 <span class="sourceLineNo">521</span><a name="line.521"></a>
-<span class="sourceLineNo">522</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      last_version = version;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      last_update = time;<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      cur_worker_name = worker;<a name="line.525"></a>
-<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span>    public void setUnassigned() {<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      cur_worker_name = null;<a name="line.529"></a>
-<span class="sourceLineNo">530</span>      last_update = -1;<a name="line.530"></a>
-<span class="sourceLineNo">531</span>    }<a name="line.531"></a>
-<span class="sourceLineNo">532</span>  }<a name="line.532"></a>
-<span class="sourceLineNo">533</span><a name="line.533"></a>
-<span class="sourceLineNo">534</span>  /**<a name="line.534"></a>
-<span class="sourceLineNo">535</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.535"></a>
-<span class="sourceLineNo">536</span>   */<a name="line.536"></a>
-<span class="sourceLineNo">537</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.537"></a>
-<span class="sourceLineNo">538</span>    private long lastLog = 0;<a name="line.538"></a>
-<span class="sourceLineNo">539</span><a name="line.539"></a>
-<span class="sourceLineNo">540</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.540"></a>
-<span class="sourceLineNo">541</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.541"></a>
-<span class="sourceLineNo">542</span>    }<a name="line.542"></a>
+<span class="sourceLineNo">522</span>    public void heartbeatNoDetails(long time) {<a name="line.522"></a>
+<span class="sourceLineNo">523</span>      last_update = time;<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    }<a name="line.524"></a>
+<span class="sourceLineNo">525</span><a name="line.525"></a>
+<span class="sourceLineNo">526</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span>      last_version = version;<a name="line.527"></a>
+<span class="sourceLineNo">528</span>      last_update = time;<a name="line.528"></a>
+<span class="sourceLineNo">529</span>      cur_worker_name = worker;<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    }<a name="line.530"></a>
+<span class="sourceLineNo">531</span><a name="line.531"></a>
+<span class="sourceLineNo">532</span>    public void setUnassigned() {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>      cur_worker_name = null;<a name="line.533"></a>
+<span class="sourceLineNo">534</span>      last_update = -1;<a name="line.534"></a>
+<span class="sourceLineNo">535</span>    }<a name="line.535"></a>
+<span class="sourceLineNo">536</span>  }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>  /**<a name="line.538"></a>
+<span class="sourceLineNo">539</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.539"></a>
+<span class="sourceLineNo">540</span>   */<a name="line.540"></a>
+<span class="sourceLineNo">541</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.541"></a>
+<span class="sourceLineNo">542</span>    private long lastLog = 0;<a name="line.542"></a>
 <span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>    @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    protected void chore() {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.546"></a>
+<span class="sourceLineNo">544</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    }<a name="line.546"></a>
 <span class="sourceLineNo">547</span><a name="line.547"></a>
-<span class="sourceLineNo">548</span>      int resubmitted = 0;<a name="line.548"></a>
-<span class="sourceLineNo">549</span>      int unassigned = 0;<a name="line.549"></a>
-<span class="sourceLineNo">550</span>      int tot = 0;<a name="line.550"></a>
-<span class="sourceLineNo">551</span>      boolean found_assigned_task = false;<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.552"></a>
-<span class="sourceLineNo">553</span><a name="line.553"></a>
-<span class="sourceLineNo">554</span>      synchronized (deadWorkersLock) {<a name="line.554"></a>
-<span class="sourceLineNo">555</span>        localDeadWorkers = deadWorkers;<a name="line.555"></a>
-<span class="sourceLineNo">556</span>        deadWorkers = null;<a name="line.556"></a>
-<span class="sourceLineNo">557</span>      }<a name="line.557"></a>
-<span class="sourceLineNo">558</span><a name="line.558"></a>
-<span class="sourceLineNo">559</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>        String path = e.getKey();<a name="line.560"></a>
-<span class="sourceLineNo">561</span>        Task task = e.getValue();<a name="line.561"></a>
-<span class="sourceLineNo">562</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.562"></a>
-<span class="sourceLineNo">563</span>        tot++;<a name="line.563"></a>
-<span class="sourceLineNo">564</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.564"></a>
-<span class="sourceLineNo">565</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.565"></a>
-<span class="sourceLineNo">566</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.566"></a>
-<span class="sourceLineNo">567</span>        // time. If we want progress when there are no region servers then we<a name="line.567"></a>
-<span class="sourceLineNo">568</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.568"></a>
-<span class="sourceLineNo">569</span>        if (task.isUnassigned()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>          unassigned++;<a name="line.570"></a>
-<span class="sourceLineNo">571</span>          continue;<a name="line.571"></a>
-<span class="sourceLineNo">572</span>        }<a name="line.572"></a>
-<span class="sourceLineNo">573</span>        found_assigned_task = true;<a name="line.573"></a>
-<span class="sourceLineNo">574</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.575"></a>
-<span class="sourceLineNo">576</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.576"></a>
-<span class="sourceLineNo">577</span>            resubmitted++;<a name="line.577"></a>
-<span class="sourceLineNo">578</span>          } else {<a name="line.578"></a>
-<span class="sourceLineNo">579</span>            handleDeadWorker(cur_worker);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.580"></a>
-<span class="sourceLineNo">581</span>                + ", will retry.");<a name="line.581"></a>
-<span class="sourceLineNo">582</span>          }<a name="line.582"></a>
-<span class="sourceLineNo">583</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.583"></a>
-<span class="sourceLineNo">584</span>          resubmitted++;<a name="line.584"></a>
-<span class="sourceLineNo">585</span>        }<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      }<a name="line.586"></a>
-<span class="sourceLineNo">587</span>      if (tot &gt; 0) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.588"></a>
-<span class="sourceLineNo">589</span>        if (now &gt; lastLog + 5000) {<a name="line.589"></a>
-<span class="sourceLineNo">590</span>          lastLog = now;<a name="line.590"></a>
-<span class="sourceLineNo">591</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.591"></a>
-<span class="sourceLineNo">592</span>        }<a name="line.592"></a>
-<span class="sourceLineNo">593</span>      }<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      if (resubmitted &gt; 0) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.595"></a>
-<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
-<span class="sourceLineNo">597</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.598"></a>
-<span class="sourceLineNo">599</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // because a. it is very unlikely that every worker had a<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      // transient error when trying to grab the task b. if there are no<a name="line.601"></a>
-<span class="sourceLineNo">602</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.604"></a>
-<span class="sourceLineNo">605</span>      // that there is always one worker in the system<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      if (tot &gt; 0<a name="line.606"></a>
-<span class="sourceLineNo">607</span>          &amp;&amp; !found_assigned_task<a name="line.607"></a>
-<span class="sourceLineNo">608</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.608"></a>
-<span class="sourceLineNo">609</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.609"></a>
-<span class="sourceLineNo">610</span>          String key = e.getKey();<a name="line.610"></a>
-<span class="sourceLineNo">611</span>          Task task = e.getValue();<a name="line.611"></a>
-<span class="sourceLineNo">612</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.612"></a>
-<span class="sourceLineNo">613</span>          // have been asynchronously assigned. There is no locking required<a name="line.613"></a>
-<span class="sourceLineNo">614</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.614"></a>
-<span class="sourceLineNo">615</span>          // called unnecessarily for a taskpath<a name="line.615"></a>
-<span class="sourceLineNo">616</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>            // We just touch the znode to make sure its still there<a name="line.617"></a>
-<span class="sourceLineNo">618</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.618"></a>
-<span class="sourceLineNo">619</span>          }<a name="line.619"></a>
-<span class="sourceLineNo">620</span>        }<a name="line.620"></a>
-<span class="sourceLineNo">621</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.621"></a>
-<span class="sourceLineNo">622</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.623"></a>
-<span class="sourceLineNo">624</span>      }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      Set&lt;String&gt; failedDeletions =<a name="line.625"></a>
-<span class="sourceLineNo">626</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      // Retry previously failed deletes<a name="line.627"></a>
-<span class="sourceLineNo">628</span>      if (failedDeletions.size() &gt; 0) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.629"></a>
-<span class="sourceLineNo">630</span>        for (String tmpPath : tmpPaths) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>          // deleteNode is an async call<a name="line.631"></a>
-<span class="sourceLineNo">632</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.632"></a>
-<span class="sourceLineNo">633</span>        }<a name="line.633"></a>
-<span class="sourceLineNo">634</span>        failedDeletions.removeAll(tmpPaths);<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      }<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span>  }<a name="line.637"></a>
-<span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>  public enum ResubmitDirective {<a name="line.639"></a>
-<span class="sourceLineNo">640</span>    CHECK(), FORCE()<a name="line.640"></a>
+<span class="sourceLineNo">548</span>    @Override<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    protected void chore() {<a name="line.549"></a>
+<span class="sourceLineNo">550</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>      int resubmitted = 0;<a name="line.552"></a>
+<span class="sourceLineNo">553</span>      int unassigned = 0;<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      int tot = 0;<a name="line.554"></a>
+<span class="sourceLineNo">555</span>      boolean found_assigned_task = false;<a name="line.555"></a>
+<span class="sourceLineNo">556</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>      synchronized (deadWorkersLock) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>        localDeadWorkers = deadWorkers;<a name="line.559"></a>
+<span class="sourceLineNo">560</span>        deadWorkers = null;<a name="line.560"></a>
+<span class="sourceLineNo">561</span>      }<a name="line.561"></a>
+<span class="sourceLineNo">562</span><a name="line.562"></a>
+<span class="sourceLineNo">563</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.563"></a>
+<span class="sourceLineNo">564</span>        String path = e.getKey();<a name="line.564"></a>
+<span class="sourceLineNo">565</span>        Task task = e.getValue();<a name="line.565"></a>
+<span class="sourceLineNo">566</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.566"></a>
+<span class="sourceLineNo">567</span>        tot++;<a name="line.567"></a>
+<span class="sourceLineNo">568</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.568"></a>
+<span class="sourceLineNo">569</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.569"></a>
+<span class="sourceLineNo">570</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.570"></a>
+<span class="sourceLineNo">571</span>        // time. If we want progress when there are no region servers then we<a name="line.571"></a>
+<span class="sourceLineNo">572</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.572"></a>
+<span class="sourceLineNo">573</span>        if (task.isUnassigned()) {<a name="line.573"></a>
+<span class="sourceLineNo">574</span>          unassigned++;<a name="line.574"></a>
+<span class="sourceLineNo">575</span>          continue;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>        }<a name="line.576"></a>
+<span class="sourceLineNo">577</span>        found_assigned_task = true;<a name="line.577"></a>
+<span class="sourceLineNo">578</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.578"></a>
+<span class="sourceLineNo">579</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.579"></a>
+<span class="sourceLineNo">580</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.580"></a>
+<span class="sourceLineNo">581</span>            resubmitted++;<a name="line.581"></a>
+<span class="sourceLineNo">582</span>          } else {<a name="line.582"></a>
+<span class="sourceLineNo">583</span>            handleDeadWorker(cur_worker);<a name="line.583"></a>
+<span class="sourceLineNo">584</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.584"></a>
+<span class="sourceLineNo">585</span>                + ", will retry.");<a name="line.585"></a>
+<span class="sourceLineNo">586</span>          }<a name="line.586"></a>
+<span class="sourceLineNo">587</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>          resubmitted++;<a name="line.588"></a>
+<span class="sourceLineNo">589</span>        }<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      }<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      if (tot &gt; 0) {<a name="line.591"></a>
+<span class="sourceLineNo">592</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.592"></a>
+<span class="sourceLineNo">593</span>        if (now &gt; lastLog + 5000) {<a name="line.593"></a>
+<span class="sourceLineNo">594</span>          lastLog = now;<a name="line.594"></a>
+<span class="sourceLineNo">595</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.595"></a>
+<span class="sourceLineNo">596</span>        }<a name="line.596"></a>
+<span class="sourceLineNo">597</span>      }<a name="line.597"></a>
+<span class="sourceLineNo">598</span>      if (resubmitted &gt; 0) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.599"></a>
+<span class="sourceLineNo">600</span>      }<a name="line.600"></a>
+<span class="sourceLineNo">601</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.602"></a>
+<span class="sourceLineNo">603</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.603"></a>
+<span class="sourceLineNo">604</span>      // because a. it is very unlikely that every worker had a<a name="line.604"></a>
+<span class="sourceLineNo">605</span>      // transient error when trying to grab the task b. if there are no<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.608"></a>
+<span class="sourceLineNo">609</span>      // that there is always one worker in the system<a name="line.609"></a>
+<span class="sourceLineNo">610</span>      if (tot &gt; 0<a name="line.610"></a>
+<span class="sourceLineNo">611</span>          &amp;&amp; !found_assigned_task<a name="line.611"></a>
+<span class="sourceLineNo">612</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>          String key = e.getKey();<a name="line.614"></a>
+<span class="sourceLineNo">615</span>          Task task = e.getValue();<a name="line.615"></a>
+<span class="sourceLineNo">616</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.616"></a>
+<span class="sourceLineNo">617</span>          // have been asynchronously assigned. There is no locking required<a name="line.617"></a>
+<span class="sourceLineNo">618</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.618"></a>
+<span class="sourceLineNo">619</span>          // called unnecessarily for a taskpath<a name="line.619"></a>
+<span class="sourceLineNo">620</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.620"></a>
+<span class="sourceLineNo">621</span>            // We just touch the znode to make sure its still there<a name="line.621"></a>
+<span class="sourceLineNo">622</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.622"></a>
+<span class="sourceLineNo">623</span>          }<a name="line.623"></a>
+<span class="sourceLineNo">624</span>        }<a name="line.624"></a>
+<span class="sourceLineNo">625</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.625"></a>
+<span class="sourceLineNo">626</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.626"></a>
+<span class="sourceLineNo">627</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      }<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      Set&lt;String&gt; failedDeletions =<a name="line.629"></a>
+<span class="sourceLineNo">630</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.630"></a>
+<span class="sourceLineNo">631</span>      // Retry previously failed deletes<a name="line.631"></a>
+<span class="sourceLineNo">632</span>      if (failedDeletions.size() &gt; 0) {<a name="line.632"></a>
+<span class="sourceLineNo">633</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.633"></a>
+<span class="sourceLineNo">634</span>        for (String tmpPath : tmpPaths) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>          // deleteNode is an async call<a name="line.635"></a>
+<span class="sourceLineNo">636</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.636"></a>
+<span class="sourceLineNo">637</span>        }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>        failedDeletions.removeAll(tmpPaths);<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      }<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    }<a name="line.640"></a>
 <span class="sourceLineNo">641</span>  }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>  public enum TerminationStatus {<a name="line.643"></a>
-<span class="sourceLineNo">644</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.644"></a>
-<span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    final String statusMsg;<a name="line.646"></a>
-<span class="sourceLineNo">647</span><a name="line.647"></a>
-<span class="sourceLineNo">648</span>    TerminationStatus(String msg) {<a name="line.648"></a>
-<span class="sourceLineNo">649</span>      statusMsg = msg;<a name="line.649"></a>
-<span class="sourceLineNo">650</span>    }<a name="line.650"></a>
+<span class="sourceLineNo">643</span>  public enum ResubmitDirective {<a name="line.643"></a>
+<span class="sourceLineNo">644</span>    CHECK(), FORCE()<a name="line.644"></a>
+<span class="sourceLineNo">645</span>  }<a name="line.645"></a>
+<span class="sourceLineNo">646</span><a name="line.646"></a>
+<span class="sourceLineNo">647</span>  public enum TerminationStatus {<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.648"></a>
+<span class="sourceLineNo">649</span><a name="line.649"></a>
+<span class="sourceLineNo">650</span>    final String statusMsg;<a name="line.650"></a>
 <span class="sourceLineNo">651</span><a name="line.651"></a>
-<span class="sourceLineNo">652</span>    @Override<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    public String toString() {<a name="line.653"></a>
-<span class="sourceLineNo">654</span>      return statusMsg;<a name="line.654"></a>
-<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
-<span class="sourceLineNo">657</span>}<a name="line.657"></a>
+<span class="sourceLineNo">652</span>    TerminationStatus(String msg) {<a name="line.652"></a>
+<span class="sourceLineNo">653</span>      statusMsg = msg;<a name="line.653"></a>
+<span class="sourceLineNo">654</span>    }<a name="line.654"></a>
+<span class="sourceLineNo">655</span><a name="line.655"></a>
+<span class="sourceLineNo">656</span>    @Override<a name="line.656"></a>
+<span class="sourceLineNo">657</span>    public String toString() {<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      return statusMsg;<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    }<a name="line.659"></a>
+<span class="sourceLineNo">660</span>  }<a name="line.660"></a>
+<span class="sourceLineNo">661</span>}<a name="line.661"></a>
 
 
 


[02/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
index c63e65a..21df2b9 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
@@ -432,900 +432,907 @@
 <span class="sourceLineNo">424</span>      it.next(); // Skip the current log<a name="line.424"></a>
 <span class="sourceLineNo">425</span><a name="line.425"></a>
 <span class="sourceLineNo">426</span>      ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {<a name="line.426"></a>
-<span class="sourceLineNo">427</span>        @Override<a name="line.427"></a>
-<span class="sourceLineNo">428</span>        public void setMaxProcId(long maxProcId) {<a name="line.428"></a>
-<span class="sourceLineNo">429</span>          loader.setMaxProcId(maxProcId);<a name="line.429"></a>
-<span class="sourceLineNo">430</span>        }<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>        @Override<a name="line.432"></a>
-<span class="sourceLineNo">433</span>        public void load(ProcedureIterator procIter) throws IOException {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>          loader.load(procIter);<a name="line.434"></a>
-<span class="sourceLineNo">435</span>        }<a name="line.435"></a>
-<span class="sourceLineNo">436</span><a name="line.436"></a>
-<span class="sourceLineNo">437</span>        @Override<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.438"></a>
-<span class="sourceLineNo">439</span>          loader.handleCorrupted(procIter);<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>        @Override<a name="line.442"></a>
-<span class="sourceLineNo">443</span>        public void markCorruptedWAL(ProcedureWALFile log, IOException e) {<a name="line.443"></a>
-<span class="sourceLineNo">444</span>          if (corruptedLogs == null) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span>            corruptedLogs = new HashSet&lt;&gt;();<a name="line.445"></a>
-<span class="sourceLineNo">446</span>          }<a name="line.446"></a>
-<span class="sourceLineNo">447</span>          corruptedLogs.add(log);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>          // TODO: sideline corrupted log<a name="line.448"></a>
-<span class="sourceLineNo">449</span>        }<a name="line.449"></a>
-<span class="sourceLineNo">450</span>      });<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    } finally {<a name="line.451"></a>
-<span class="sourceLineNo">452</span>      try {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>        // try to cleanup inactive wals and complete the operation<a name="line.453"></a>
-<span class="sourceLineNo">454</span>        buildHoldingCleanupTracker();<a name="line.454"></a>
-<span class="sourceLineNo">455</span>        tryCleanupLogsOnLoad();<a name="line.455"></a>
-<span class="sourceLineNo">456</span>        loading.set(false);<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      } finally {<a name="line.457"></a>
-<span class="sourceLineNo">458</span>        lock.unlock();<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      }<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>  }<a name="line.461"></a>
-<span class="sourceLineNo">462</span><a name="line.462"></a>
-<span class="sourceLineNo">463</span>  private void tryCleanupLogsOnLoad() {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>    // nothing to cleanup.<a name="line.464"></a>
-<span class="sourceLineNo">465</span>    if (logs.size() &lt;= 1) return;<a name="line.465"></a>
-<span class="sourceLineNo">466</span><a name="line.466"></a>
-<span class="sourceLineNo">467</span>    // the config says to not cleanup wals on load.<a name="line.467"></a>
-<span class="sourceLineNo">468</span>    if (!conf.getBoolean(EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>      DEFAULT_EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY)) {<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      LOG.debug("WALs cleanup on load is not enabled: " + getActiveLogs());<a name="line.470"></a>
-<span class="sourceLineNo">471</span>      return;<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    }<a name="line.472"></a>
+<span class="sourceLineNo">427</span>        long count = 0;<a name="line.427"></a>
+<span class="sourceLineNo">428</span><a name="line.428"></a>
+<span class="sourceLineNo">429</span>        @Override<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        public void setMaxProcId(long maxProcId) {<a name="line.430"></a>
+<span class="sourceLineNo">431</span>          loader.setMaxProcId(maxProcId);<a name="line.431"></a>
+<span class="sourceLineNo">432</span>        }<a name="line.432"></a>
+<span class="sourceLineNo">433</span><a name="line.433"></a>
+<span class="sourceLineNo">434</span>        @Override<a name="line.434"></a>
+<span class="sourceLineNo">435</span>        public void load(ProcedureIterator procIter) throws IOException {<a name="line.435"></a>
+<span class="sourceLineNo">436</span>          if ((++count % 1000) == 0) {<a name="line.436"></a>
+<span class="sourceLineNo">437</span>            // Log every 1000 procedures otherwise it looks like Master is dead if loads of WALs<a name="line.437"></a>
+<span class="sourceLineNo">438</span>            // and procedures to load.<a name="line.438"></a>
+<span class="sourceLineNo">439</span>            LOG.debug("Loaded {} procedures", this.count);<a name="line.439"></a>
+<span class="sourceLineNo">440</span>          }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>          loader.load(procIter);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>        }<a name="line.442"></a>
+<span class="sourceLineNo">443</span><a name="line.443"></a>
+<span class="sourceLineNo">444</span>        @Override<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.445"></a>
+<span class="sourceLineNo">446</span>          loader.handleCorrupted(procIter);<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        }<a name="line.447"></a>
+<span class="sourceLineNo">448</span><a name="line.448"></a>
+<span class="sourceLineNo">449</span>        @Override<a name="line.449"></a>
+<span class="sourceLineNo">450</span>        public void markCorruptedWAL(ProcedureWALFile log, IOException e) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>          if (corruptedLogs == null) {<a name="line.451"></a>
+<span class="sourceLineNo">452</span>            corruptedLogs = new HashSet&lt;&gt;();<a name="line.452"></a>
+<span class="sourceLineNo">453</span>          }<a name="line.453"></a>
+<span class="sourceLineNo">454</span>          corruptedLogs.add(log);<a name="line.454"></a>
+<span class="sourceLineNo">455</span>          // TODO: sideline corrupted log<a name="line.455"></a>
+<span class="sourceLineNo">456</span>        }<a name="line.456"></a>
+<span class="sourceLineNo">457</span>      });<a name="line.457"></a>
+<span class="sourceLineNo">458</span>    } finally {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>      try {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>        // try to cleanup inactive wals and complete the operation<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        buildHoldingCleanupTracker();<a name="line.461"></a>
+<span class="sourceLineNo">462</span>        tryCleanupLogsOnLoad();<a name="line.462"></a>
+<span class="sourceLineNo">463</span>        loading.set(false);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>      } finally {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>        lock.unlock();<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      }<a name="line.466"></a>
+<span class="sourceLineNo">467</span>    }<a name="line.467"></a>
+<span class="sourceLineNo">468</span>  }<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>  private void tryCleanupLogsOnLoad() {<a name="line.470"></a>
+<span class="sourceLineNo">471</span>    // nothing to cleanup.<a name="line.471"></a>
+<span class="sourceLineNo">472</span>    if (logs.size() &lt;= 1) return;<a name="line.472"></a>
 <span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>    try {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>      periodicRoll();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    } catch (IOException e) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      LOG.warn("Unable to cleanup logs on load: " + e.getMessage(), e);<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    }<a name="line.478"></a>
-<span class="sourceLineNo">479</span>  }<a name="line.479"></a>
+<span class="sourceLineNo">474</span>    // the config says to not cleanup wals on load.<a name="line.474"></a>
+<span class="sourceLineNo">475</span>    if (!conf.getBoolean(EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY,<a name="line.475"></a>
+<span class="sourceLineNo">476</span>      DEFAULT_EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY)) {<a name="line.476"></a>
+<span class="sourceLineNo">477</span>      LOG.debug("WALs cleanup on load is not enabled: " + getActiveLogs());<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      return;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
 <span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span>  @Override<a name="line.481"></a>
-<span class="sourceLineNo">482</span>  public void insert(final Procedure proc, final Procedure[] subprocs) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    if (LOG.isTraceEnabled()) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span>      LOG.trace("Insert " + proc + ", subproc=" + Arrays.toString(subprocs));<a name="line.484"></a>
+<span class="sourceLineNo">481</span>    try {<a name="line.481"></a>
+<span class="sourceLineNo">482</span>      periodicRoll();<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    } catch (IOException e) {<a name="line.483"></a>
+<span class="sourceLineNo">484</span>      LOG.warn("Unable to cleanup logs on load: " + e.getMessage(), e);<a name="line.484"></a>
 <span class="sourceLineNo">485</span>    }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>    ByteSlot slot = acquireSlot();<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    try {<a name="line.488"></a>
-<span class="sourceLineNo">489</span>      // Serialize the insert<a name="line.489"></a>
-<span class="sourceLineNo">490</span>      long[] subProcIds = null;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>      if (subprocs != null) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span>        ProcedureWALFormat.writeInsert(slot, proc, subprocs);<a name="line.492"></a>
-<span class="sourceLineNo">493</span>        subProcIds = new long[subprocs.length];<a name="line.493"></a>
-<span class="sourceLineNo">494</span>        for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>          subProcIds[i] = subprocs[i].getProcId();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>        }<a name="line.496"></a>
-<span class="sourceLineNo">497</span>      } else {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>        assert !proc.hasParent();<a name="line.498"></a>
-<span class="sourceLineNo">499</span>        ProcedureWALFormat.writeInsert(slot, proc);<a name="line.499"></a>
-<span class="sourceLineNo">500</span>      }<a name="line.500"></a>
-<span class="sourceLineNo">501</span><a name="line.501"></a>
-<span class="sourceLineNo">502</span>      // Push the transaction data and wait until it is persisted<a name="line.502"></a>
-<span class="sourceLineNo">503</span>      pushData(PushType.INSERT, slot, proc.getProcId(), subProcIds);<a name="line.503"></a>
-<span class="sourceLineNo">504</span>    } catch (IOException e) {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      // We are not able to serialize the procedure.<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      // this is a code error, and we are not able to go on.<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: proc=" +<a name="line.507"></a>
-<span class="sourceLineNo">508</span>          proc + ", subprocs=" + Arrays.toString(subprocs), e);<a name="line.508"></a>
-<span class="sourceLineNo">509</span>      throw new RuntimeException(e);<a name="line.509"></a>
-<span class="sourceLineNo">510</span>    } finally {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      releaseSlot(slot);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>    }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>  }<a name="line.513"></a>
-<span class="sourceLineNo">514</span><a name="line.514"></a>
-<span class="sourceLineNo">515</span>  @Override<a name="line.515"></a>
-<span class="sourceLineNo">516</span>  public void insert(final Procedure[] procs) {<a name="line.516"></a>
-<span class="sourceLineNo">517</span>    if (LOG.isTraceEnabled()) {<a name="line.517"></a>
-<span class="sourceLineNo">518</span>      LOG.trace("Insert " + Arrays.toString(procs));<a name="line.518"></a>
+<span class="sourceLineNo">486</span>  }<a name="line.486"></a>
+<span class="sourceLineNo">487</span><a name="line.487"></a>
+<span class="sourceLineNo">488</span>  @Override<a name="line.488"></a>
+<span class="sourceLineNo">489</span>  public void insert(final Procedure proc, final Procedure[] subprocs) {<a name="line.489"></a>
+<span class="sourceLineNo">490</span>    if (LOG.isTraceEnabled()) {<a name="line.490"></a>
+<span class="sourceLineNo">491</span>      LOG.trace("Insert " + proc + ", subproc=" + Arrays.toString(subprocs));<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    }<a name="line.492"></a>
+<span class="sourceLineNo">493</span><a name="line.493"></a>
+<span class="sourceLineNo">494</span>    ByteSlot slot = acquireSlot();<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    try {<a name="line.495"></a>
+<span class="sourceLineNo">496</span>      // Serialize the insert<a name="line.496"></a>
+<span class="sourceLineNo">497</span>      long[] subProcIds = null;<a name="line.497"></a>
+<span class="sourceLineNo">498</span>      if (subprocs != null) {<a name="line.498"></a>
+<span class="sourceLineNo">499</span>        ProcedureWALFormat.writeInsert(slot, proc, subprocs);<a name="line.499"></a>
+<span class="sourceLineNo">500</span>        subProcIds = new long[subprocs.length];<a name="line.500"></a>
+<span class="sourceLineNo">501</span>        for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.501"></a>
+<span class="sourceLineNo">502</span>          subProcIds[i] = subprocs[i].getProcId();<a name="line.502"></a>
+<span class="sourceLineNo">503</span>        }<a name="line.503"></a>
+<span class="sourceLineNo">504</span>      } else {<a name="line.504"></a>
+<span class="sourceLineNo">505</span>        assert !proc.hasParent();<a name="line.505"></a>
+<span class="sourceLineNo">506</span>        ProcedureWALFormat.writeInsert(slot, proc);<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      }<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      // Push the transaction data and wait until it is persisted<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      pushData(PushType.INSERT, slot, proc.getProcId(), subProcIds);<a name="line.510"></a>
+<span class="sourceLineNo">511</span>    } catch (IOException e) {<a name="line.511"></a>
+<span class="sourceLineNo">512</span>      // We are not able to serialize the procedure.<a name="line.512"></a>
+<span class="sourceLineNo">513</span>      // this is a code error, and we are not able to go on.<a name="line.513"></a>
+<span class="sourceLineNo">514</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: proc=" +<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          proc + ", subprocs=" + Arrays.toString(subprocs), e);<a name="line.515"></a>
+<span class="sourceLineNo">516</span>      throw new RuntimeException(e);<a name="line.516"></a>
+<span class="sourceLineNo">517</span>    } finally {<a name="line.517"></a>
+<span class="sourceLineNo">518</span>      releaseSlot(slot);<a name="line.518"></a>
 <span class="sourceLineNo">519</span>    }<a name="line.519"></a>
-<span class="sourceLineNo">520</span><a name="line.520"></a>
-<span class="sourceLineNo">521</span>    ByteSlot slot = acquireSlot();<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    try {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      // Serialize the insert<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      long[] procIds = new long[procs.length];<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      for (int i = 0; i &lt; procs.length; ++i) {<a name="line.525"></a>
-<span class="sourceLineNo">526</span>        assert !procs[i].hasParent();<a name="line.526"></a>
-<span class="sourceLineNo">527</span>        procIds[i] = procs[i].getProcId();<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        ProcedureWALFormat.writeInsert(slot, procs[i]);<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      }<a name="line.529"></a>
-<span class="sourceLineNo">530</span><a name="line.530"></a>
-<span class="sourceLineNo">531</span>      // Push the transaction data and wait until it is persisted<a name="line.531"></a>
-<span class="sourceLineNo">532</span>      pushData(PushType.INSERT, slot, Procedure.NO_PROC_ID, procIds);<a name="line.532"></a>
-<span class="sourceLineNo">533</span>    } catch (IOException e) {<a name="line.533"></a>
-<span class="sourceLineNo">534</span>      // We are not able to serialize the procedure.<a name="line.534"></a>
-<span class="sourceLineNo">535</span>      // this is a code error, and we are not able to go on.<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: " +<a name="line.536"></a>
-<span class="sourceLineNo">537</span>          Arrays.toString(procs), e);<a name="line.537"></a>
-<span class="sourceLineNo">538</span>      throw new RuntimeException(e);<a name="line.538"></a>
-<span class="sourceLineNo">539</span>    } finally {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      releaseSlot(slot);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>  public void update(final Procedure proc) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>    if (LOG.isTraceEnabled()) {<a name="line.546"></a>
-<span class="sourceLineNo">547</span>      LOG.trace("Update " + proc);<a name="line.547"></a>
+<span class="sourceLineNo">520</span>  }<a name="line.520"></a>
+<span class="sourceLineNo">521</span><a name="line.521"></a>
+<span class="sourceLineNo">522</span>  @Override<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  public void insert(final Procedure[] procs) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    if (LOG.isTraceEnabled()) {<a name="line.524"></a>
+<span class="sourceLineNo">525</span>      LOG.trace("Insert " + Arrays.toString(procs));<a name="line.525"></a>
+<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span>    ByteSlot slot = acquireSlot();<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    try {<a name="line.529"></a>
+<span class="sourceLineNo">530</span>      // Serialize the insert<a name="line.530"></a>
+<span class="sourceLineNo">531</span>      long[] procIds = new long[procs.length];<a name="line.531"></a>
+<span class="sourceLineNo">532</span>      for (int i = 0; i &lt; procs.length; ++i) {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>        assert !procs[i].hasParent();<a name="line.533"></a>
+<span class="sourceLineNo">534</span>        procIds[i] = procs[i].getProcId();<a name="line.534"></a>
+<span class="sourceLineNo">535</span>        ProcedureWALFormat.writeInsert(slot, procs[i]);<a name="line.535"></a>
+<span class="sourceLineNo">536</span>      }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>      // Push the transaction data and wait until it is persisted<a name="line.538"></a>
+<span class="sourceLineNo">539</span>      pushData(PushType.INSERT, slot, Procedure.NO_PROC_ID, procIds);<a name="line.539"></a>
+<span class="sourceLineNo">540</span>    } catch (IOException e) {<a name="line.540"></a>
+<span class="sourceLineNo">541</span>      // We are not able to serialize the procedure.<a name="line.541"></a>
+<span class="sourceLineNo">542</span>      // this is a code error, and we are not able to go on.<a name="line.542"></a>
+<span class="sourceLineNo">543</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: " +<a name="line.543"></a>
+<span class="sourceLineNo">544</span>          Arrays.toString(procs), e);<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      throw new RuntimeException(e);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    } finally {<a name="line.546"></a>
+<span class="sourceLineNo">547</span>      releaseSlot(slot);<a name="line.547"></a>
 <span class="sourceLineNo">548</span>    }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>    ByteSlot slot = acquireSlot();<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    try {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      // Serialize the update<a name="line.552"></a>
-<span class="sourceLineNo">553</span>      ProcedureWALFormat.writeUpdate(slot, proc);<a name="line.553"></a>
-<span class="sourceLineNo">554</span><a name="line.554"></a>
-<span class="sourceLineNo">555</span>      // Push the transaction data and wait until it is persisted<a name="line.555"></a>
-<span class="sourceLineNo">556</span>      pushData(PushType.UPDATE, slot, proc.getProcId(), null);<a name="line.556"></a>
-<span class="sourceLineNo">557</span>    } catch (IOException e) {<a name="line.557"></a>
-<span class="sourceLineNo">558</span>      // We are not able to serialize the procedure.<a name="line.558"></a>
-<span class="sourceLineNo">559</span>      // this is a code error, and we are not able to go on.<a name="line.559"></a>
-<span class="sourceLineNo">560</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.560"></a>
-<span class="sourceLineNo">561</span>      throw new RuntimeException(e);<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    } finally {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      releaseSlot(slot);<a name="line.563"></a>
-<span class="sourceLineNo">564</span>    }<a name="line.564"></a>
-<span class="sourceLineNo">565</span>  }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>  @Override<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  public void delete(final long procId) {<a name="line.568"></a>
-<span class="sourceLineNo">569</span>    if (LOG.isTraceEnabled()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>      LOG.trace("Delete " + procId);<a name="line.570"></a>
+<span class="sourceLineNo">549</span>  }<a name="line.549"></a>
+<span class="sourceLineNo">550</span><a name="line.550"></a>
+<span class="sourceLineNo">551</span>  @Override<a name="line.551"></a>
+<span class="sourceLineNo">552</span>  public void update(final Procedure proc) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    if (LOG.isTraceEnabled()) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      LOG.trace("Update " + proc);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span><a name="line.556"></a>
+<span class="sourceLineNo">557</span>    ByteSlot slot = acquireSlot();<a name="line.557"></a>
+<span class="sourceLineNo">558</span>    try {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>      // Serialize the update<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      ProcedureWALFormat.writeUpdate(slot, proc);<a name="line.560"></a>
+<span class="sourceLineNo">561</span><a name="line.561"></a>
+<span class="sourceLineNo">562</span>      // Push the transaction data and wait until it is persisted<a name="line.562"></a>
+<span class="sourceLineNo">563</span>      pushData(PushType.UPDATE, slot, proc.getProcId(), null);<a name="line.563"></a>
+<span class="sourceLineNo">564</span>    } catch (IOException e) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span>      // We are not able to serialize the procedure.<a name="line.565"></a>
+<span class="sourceLineNo">566</span>      // this is a code error, and we are not able to go on.<a name="line.566"></a>
+<span class="sourceLineNo">567</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.567"></a>
+<span class="sourceLineNo">568</span>      throw new RuntimeException(e);<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    } finally {<a name="line.569"></a>
+<span class="sourceLineNo">570</span>      releaseSlot(slot);<a name="line.570"></a>
 <span class="sourceLineNo">571</span>    }<a name="line.571"></a>
-<span class="sourceLineNo">572</span><a name="line.572"></a>
-<span class="sourceLineNo">573</span>    ByteSlot slot = acquireSlot();<a name="line.573"></a>
-<span class="sourceLineNo">574</span>    try {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>      // Serialize the delete<a name="line.575"></a>
-<span class="sourceLineNo">576</span>      ProcedureWALFormat.writeDelete(slot, procId);<a name="line.576"></a>
-<span class="sourceLineNo">577</span><a name="line.577"></a>
-<span class="sourceLineNo">578</span>      // Push the transaction data and wait until it is persisted<a name="line.578"></a>
-<span class="sourceLineNo">579</span>      pushData(PushType.DELETE, slot, procId, null);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    } catch (IOException e) {<a name="line.580"></a>
-<span class="sourceLineNo">581</span>      // We are not able to serialize the procedure.<a name="line.581"></a>
-<span class="sourceLineNo">582</span>      // this is a code error, and we are not able to go on.<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + procId, e);<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      throw new RuntimeException(e);<a name="line.584"></a>
-<span class="sourceLineNo">585</span>    } finally {<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      releaseSlot(slot);<a name="line.586"></a>
-<span class="sourceLineNo">587</span>    }<a name="line.587"></a>
-<span class="sourceLineNo">588</span>  }<a name="line.588"></a>
-<span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>  @Override<a name="line.590"></a>
-<span class="sourceLineNo">591</span>  public void delete(final Procedure proc, final long[] subProcIds) {<a name="line.591"></a>
-<span class="sourceLineNo">592</span>    assert proc != null : "expected a non-null procedure";<a name="line.592"></a>
-<span class="sourceLineNo">593</span>    assert subProcIds != null &amp;&amp; subProcIds.length &gt; 0 : "expected subProcIds";<a name="line.593"></a>
-<span class="sourceLineNo">594</span>    if (LOG.isTraceEnabled()) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>      LOG.trace("Update " + proc + " and Delete " + Arrays.toString(subProcIds));<a name="line.595"></a>
-<span class="sourceLineNo">596</span>    }<a name="line.596"></a>
-<span class="sourceLineNo">597</span><a name="line.597"></a>
-<span class="sourceLineNo">598</span>    ByteSlot slot = acquireSlot();<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    try {<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // Serialize the delete<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      ProcedureWALFormat.writeDelete(slot, proc, subProcIds);<a name="line.601"></a>
-<span class="sourceLineNo">602</span><a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // Push the transaction data and wait until it is persisted<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      pushData(PushType.DELETE, slot, proc.getProcId(), subProcIds);<a name="line.604"></a>
-<span class="sourceLineNo">605</span>    } catch (IOException e) {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      // We are not able to serialize the procedure.<a name="line.606"></a>
-<span class="sourceLineNo">607</span>      // this is a code error, and we are not able to go on.<a name="line.607"></a>
-<span class="sourceLineNo">608</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.608"></a>
-<span class="sourceLineNo">609</span>      throw new RuntimeException(e);<a name="line.609"></a>
-<span class="sourceLineNo">610</span>    } finally {<a name="line.610"></a>
-<span class="sourceLineNo">611</span>      releaseSlot(slot);<a name="line.611"></a>
-<span class="sourceLineNo">612</span>    }<a name="line.612"></a>
-<span class="sourceLineNo">613</span>  }<a name="line.613"></a>
-<span class="sourceLineNo">614</span><a name="line.614"></a>
-<span class="sourceLineNo">615</span>  @Override<a name="line.615"></a>
-<span class="sourceLineNo">616</span>  public void delete(final long[] procIds, final int offset, final int count) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>    if (count == 0) return;<a name="line.617"></a>
-<span class="sourceLineNo">618</span>    if (offset == 0 &amp;&amp; count == procIds.length) {<a name="line.618"></a>
-<span class="sourceLineNo">619</span>      delete(procIds);<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    } else if (count == 1) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      delete(procIds[offset]);<a name="line.621"></a>
-<span class="sourceLineNo">622</span>    } else {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>      delete(Arrays.copyOfRange(procIds, offset, offset + count));<a name="line.623"></a>
-<span class="sourceLineNo">624</span>    }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>  }<a name="line.625"></a>
-<span class="sourceLineNo">626</span><a name="line.626"></a>
-<span class="sourceLineNo">627</span>  private void delete(final long[] procIds) {<a name="line.627"></a>
-<span class="sourceLineNo">628</span>    if (LOG.isTraceEnabled()) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>      LOG.trace("Delete " + Arrays.toString(procIds));<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    }<a name="line.630"></a>
-<span class="sourceLineNo">631</span><a name="line.631"></a>
-<span class="sourceLineNo">632</span>    final ByteSlot slot = acquireSlot();<a name="line.632"></a>
-<span class="sourceLineNo">633</span>    try {<a name="line.633"></a>
-<span class="sourceLineNo">634</span>      // Serialize the delete<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.635"></a>
-<span class="sourceLineNo">636</span>        ProcedureWALFormat.writeDelete(slot, procIds[i]);<a name="line.636"></a>
-<span class="sourceLineNo">637</span>      }<a name="line.637"></a>
+<span class="sourceLineNo">572</span>  }<a name="line.572"></a>
+<span class="sourceLineNo">573</span><a name="line.573"></a>
+<span class="sourceLineNo">574</span>  @Override<a name="line.574"></a>
+<span class="sourceLineNo">575</span>  public void delete(final long procId) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span>    if (LOG.isTraceEnabled()) {<a name="line.576"></a>
+<span class="sourceLineNo">577</span>      LOG.trace("Delete " + procId);<a name="line.577"></a>
+<span class="sourceLineNo">578</span>    }<a name="line.578"></a>
+<span class="sourceLineNo">579</span><a name="line.579"></a>
+<span class="sourceLineNo">580</span>    ByteSlot slot = acquireSlot();<a name="line.580"></a>
+<span class="sourceLineNo">581</span>    try {<a name="line.581"></a>
+<span class="sourceLineNo">582</span>      // Serialize the delete<a name="line.582"></a>
+<span class="sourceLineNo">583</span>      ProcedureWALFormat.writeDelete(slot, procId);<a name="line.583"></a>
+<span class="sourceLineNo">584</span><a name="line.584"></a>
+<span class="sourceLineNo">585</span>      // Push the transaction data and wait until it is persisted<a name="line.585"></a>
+<span class="sourceLineNo">586</span>      pushData(PushType.DELETE, slot, procId, null);<a name="line.586"></a>
+<span class="sourceLineNo">587</span>    } catch (IOException e) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>      // We are not able to serialize the procedure.<a name="line.588"></a>
+<span class="sourceLineNo">589</span>      // this is a code error, and we are not able to go on.<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + procId, e);<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      throw new RuntimeException(e);<a name="line.591"></a>
+<span class="sourceLineNo">592</span>    } finally {<a name="line.592"></a>
+<span class="sourceLineNo">593</span>      releaseSlot(slot);<a name="line.593"></a>
+<span class="sourceLineNo">594</span>    }<a name="line.594"></a>
+<span class="sourceLineNo">595</span>  }<a name="line.595"></a>
+<span class="sourceLineNo">596</span><a name="line.596"></a>
+<span class="sourceLineNo">597</span>  @Override<a name="line.597"></a>
+<span class="sourceLineNo">598</span>  public void delete(final Procedure proc, final long[] subProcIds) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>    assert proc != null : "expected a non-null procedure";<a name="line.599"></a>
+<span class="sourceLineNo">600</span>    assert subProcIds != null &amp;&amp; subProcIds.length &gt; 0 : "expected subProcIds";<a name="line.600"></a>
+<span class="sourceLineNo">601</span>    if (LOG.isTraceEnabled()) {<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      LOG.trace("Update " + proc + " and Delete " + Arrays.toString(subProcIds));<a name="line.602"></a>
+<span class="sourceLineNo">603</span>    }<a name="line.603"></a>
+<span class="sourceLineNo">604</span><a name="line.604"></a>
+<span class="sourceLineNo">605</span>    ByteSlot slot = acquireSlot();<a name="line.605"></a>
+<span class="sourceLineNo">606</span>    try {<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // Serialize the delete<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      ProcedureWALFormat.writeDelete(slot, proc, subProcIds);<a name="line.608"></a>
+<span class="sourceLineNo">609</span><a name="line.609"></a>
+<span class="sourceLineNo">610</span>      // Push the transaction data and wait until it is persisted<a name="line.610"></a>
+<span class="sourceLineNo">611</span>      pushData(PushType.DELETE, slot, proc.getProcId(), subProcIds);<a name="line.611"></a>
+<span class="sourceLineNo">612</span>    } catch (IOException e) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>      // We are not able to serialize the procedure.<a name="line.613"></a>
+<span class="sourceLineNo">614</span>      // this is a code error, and we are not able to go on.<a name="line.614"></a>
+<span class="sourceLineNo">615</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.615"></a>
+<span class="sourceLineNo">616</span>      throw new RuntimeException(e);<a name="line.616"></a>
+<span class="sourceLineNo">617</span>    } finally {<a name="line.617"></a>
+<span class="sourceLineNo">618</span>      releaseSlot(slot);<a name="line.618"></a>
+<span class="sourceLineNo">619</span>    }<a name="line.619"></a>
+<span class="sourceLineNo">620</span>  }<a name="line.620"></a>
+<span class="sourceLineNo">621</span><a name="line.621"></a>
+<span class="sourceLineNo">622</span>  @Override<a name="line.622"></a>
+<span class="sourceLineNo">623</span>  public void delete(final long[] procIds, final int offset, final int count) {<a name="line.623"></a>
+<span class="sourceLineNo">624</span>    if (count == 0) return;<a name="line.624"></a>
+<span class="sourceLineNo">625</span>    if (offset == 0 &amp;&amp; count == procIds.length) {<a name="line.625"></a>
+<span class="sourceLineNo">626</span>      delete(procIds);<a name="line.626"></a>
+<span class="sourceLineNo">627</span>    } else if (count == 1) {<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      delete(procIds[offset]);<a name="line.628"></a>
+<span class="sourceLineNo">629</span>    } else {<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      delete(Arrays.copyOfRange(procIds, offset, offset + count));<a name="line.630"></a>
+<span class="sourceLineNo">631</span>    }<a name="line.631"></a>
+<span class="sourceLineNo">632</span>  }<a name="line.632"></a>
+<span class="sourceLineNo">633</span><a name="line.633"></a>
+<span class="sourceLineNo">634</span>  private void delete(final long[] procIds) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>    if (LOG.isTraceEnabled()) {<a name="line.635"></a>
+<span class="sourceLineNo">636</span>      LOG.trace("Delete " + Arrays.toString(procIds));<a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
 <span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>      // Push the transaction data and wait until it is persisted<a name="line.639"></a>
-<span class="sourceLineNo">640</span>      pushData(PushType.DELETE, slot, Procedure.NO_PROC_ID, procIds);<a name="line.640"></a>
-<span class="sourceLineNo">641</span>    } catch (IOException e) {<a name="line.641"></a>
-<span class="sourceLineNo">642</span>      // We are not able to serialize the procedure.<a name="line.642"></a>
-<span class="sourceLineNo">643</span>      // this is a code error, and we are not able to go on.<a name="line.643"></a>
-<span class="sourceLineNo">644</span>      LOG.error("Unable to serialize the procedures: " + Arrays.toString(procIds), e);<a name="line.644"></a>
-<span class="sourceLineNo">645</span>      throw new RuntimeException(e);<a name="line.645"></a>
-<span class="sourceLineNo">646</span>    } finally {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      releaseSlot(slot);<a name="line.647"></a>
-<span class="sourceLineNo">648</span>    }<a name="line.648"></a>
-<span class="sourceLineNo">649</span>  }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span>  private ByteSlot acquireSlot() {<a name="line.651"></a>
-<span class="sourceLineNo">652</span>    ByteSlot slot = slotsCache.poll();<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    return slot != null ? slot : new ByteSlot();<a name="line.653"></a>
-<span class="sourceLineNo">654</span>  }<a name="line.654"></a>
-<span class="sourceLineNo">655</span><a name="line.655"></a>
-<span class="sourceLineNo">656</span>  private void releaseSlot(final ByteSlot slot) {<a name="line.656"></a>
-<span class="sourceLineNo">657</span>    slot.reset();<a name="line.657"></a>
-<span class="sourceLineNo">658</span>    slotsCache.offer(slot);<a name="line.658"></a>
-<span class="sourceLineNo">659</span>  }<a name="line.659"></a>
-<span class="sourceLineNo">660</span><a name="line.660"></a>
-<span class="sourceLineNo">661</span>  private enum PushType { INSERT, UPDATE, DELETE }<a name="line.661"></a>
+<span class="sourceLineNo">639</span>    final ByteSlot slot = acquireSlot();<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    try {<a name="line.640"></a>
+<span class="sourceLineNo">641</span>      // Serialize the delete<a name="line.641"></a>
+<span class="sourceLineNo">642</span>      for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.642"></a>
+<span class="sourceLineNo">643</span>        ProcedureWALFormat.writeDelete(slot, procIds[i]);<a name="line.643"></a>
+<span class="sourceLineNo">644</span>      }<a name="line.644"></a>
+<span class="sourceLineNo">645</span><a name="line.645"></a>
+<span class="sourceLineNo">646</span>      // Push the transaction data and wait until it is persisted<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      pushData(PushType.DELETE, slot, Procedure.NO_PROC_ID, procIds);<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    } catch (IOException e) {<a name="line.648"></a>
+<span class="sourceLineNo">649</span>      // We are not able to serialize the procedure.<a name="line.649"></a>
+<span class="sourceLineNo">650</span>      // this is a code error, and we are not able to go on.<a name="line.650"></a>
+<span class="sourceLineNo">651</span>      LOG.error("Unable to serialize the procedures: " + Arrays.toString(procIds), e);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>      throw new RuntimeException(e);<a name="line.652"></a>
+<span class="sourceLineNo">653</span>    } finally {<a name="line.653"></a>
+<span class="sourceLineNo">654</span>      releaseSlot(slot);<a name="line.654"></a>
+<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
+<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
+<span class="sourceLineNo">657</span><a name="line.657"></a>
+<span class="sourceLineNo">658</span>  private ByteSlot acquireSlot() {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    ByteSlot slot = slotsCache.poll();<a name="line.659"></a>
+<span class="sourceLineNo">660</span>    return slot != null ? slot : new ByteSlot();<a name="line.660"></a>
+<span class="sourceLineNo">661</span>  }<a name="line.661"></a>
 <span class="sourceLineNo">662</span><a name="line.662"></a>
-<span class="sourceLineNo">663</span>  private long pushData(final PushType type, final ByteSlot slot,<a name="line.663"></a>
-<span class="sourceLineNo">664</span>      final long procId, final long[] subProcIds) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>    if (!isRunning()) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span>      throw new RuntimeException("the store must be running before inserting data");<a name="line.666"></a>
-<span class="sourceLineNo">667</span>    }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    if (logs.isEmpty()) {<a name="line.668"></a>
-<span class="sourceLineNo">669</span>      throw new RuntimeException("recoverLease() must be called before inserting data");<a name="line.669"></a>
-<span class="sourceLineNo">670</span>    }<a name="line.670"></a>
-<span class="sourceLineNo">671</span><a name="line.671"></a>
-<span class="sourceLineNo">672</span>    long logId = -1;<a name="line.672"></a>
-<span class="sourceLineNo">673</span>    lock.lock();<a name="line.673"></a>
-<span class="sourceLineNo">674</span>    try {<a name="line.674"></a>
-<span class="sourceLineNo">675</span>      // Wait for the sync to be completed<a name="line.675"></a>
-<span class="sourceLineNo">676</span>      while (true) {<a name="line.676"></a>
-<span class="sourceLineNo">677</span>        if (!isRunning()) {<a name="line.677"></a>
-<span class="sourceLineNo">678</span>          throw new RuntimeException("store no longer running");<a name="line.678"></a>
-<span class="sourceLineNo">679</span>        } else if (isSyncAborted()) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span>          throw new RuntimeException("sync aborted", syncException.get());<a name="line.680"></a>
-<span class="sourceLineNo">681</span>        } else if (inSync.get()) {<a name="line.681"></a>
-<span class="sourceLineNo">682</span>          syncCond.await();<a name="line.682"></a>
-<span class="sourceLineNo">683</span>        } else if (slotIndex &gt;= syncMaxSlot) {<a name="line.683"></a>
-<span class="sourceLineNo">684</span>          slotCond.signal();<a name="line.684"></a>
-<span class="sourceLineNo">685</span>          syncCond.await();<a name="line.685"></a>
-<span class="sourceLineNo">686</span>        } else {<a name="line.686"></a>
-<span class="sourceLineNo">687</span>          break;<a name="line.687"></a>
-<span class="sourceLineNo">688</span>        }<a name="line.688"></a>
-<span class="sourceLineNo">689</span>      }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>      final long pushSyncId = syncId.get();<a name="line.691"></a>
-<span class="sourceLineNo">692</span>      updateStoreTracker(type, procId, subProcIds);<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      slots[slotIndex++] = slot;<a name="line.693"></a>
-<span class="sourceLineNo">694</span>      logId = flushLogId;<a name="line.694"></a>
-<span class="sourceLineNo">695</span><a name="line.695"></a>
-<span class="sourceLineNo">696</span>      // Notify that there is new data<a name="line.696"></a>
-<span class="sourceLineNo">697</span>      if (slotIndex == 1) {<a name="line.697"></a>
-<span class="sourceLineNo">698</span>        waitCond.signal();<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      }<a name="line.699"></a>
-<span class="sourceLineNo">700</span><a name="line.700"></a>
-<span class="sourceLineNo">701</span>      // Notify that the slots are full<a name="line.701"></a>
-<span class="sourceLineNo">702</span>      if (slotIndex == syncMaxSlot) {<a name="line.702"></a>
-<span class="sourceLineNo">703</span>        waitCond.signal();<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        slotCond.signal();<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span><a name="line.706"></a>
-<span class="sourceLineNo">707</span>      while (pushSyncId == syncId.get() &amp;&amp; isRunning()) {<a name="line.707"></a>
-<span class="sourceLineNo">708</span>        syncCond.await();<a name="line.708"></a>
-<span class="sourceLineNo">709</span>      }<a name="line.709"></a>
-<span class="sourceLineNo">710</span>    } catch (InterruptedException e) {<a name="line.710"></a>
-<span class="sourceLineNo">711</span>      Thread.currentThread().interrupt();<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      sendAbortProcessSignal();<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      throw new RuntimeException(e);<a name="line.713"></a>
-<span class="sourceLineNo">714</span>    } finally {<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      lock.unlock();<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      if (isSyncAborted()) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span>        throw new RuntimeException("sync aborted", syncException.get());<a name="line.717"></a>
-<span class="sourceLineNo">718</span>      }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>    }<a name="line.719"></a>
-<span class="sourceLineNo">720</span>    return logId;<a name="line.720"></a>
-<span class="sourceLineNo">721</span>  }<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>  private void updateStoreTracker(final PushType type,<a name="line.723"></a>
-<span class="sourceLineNo">724</span>      final long procId, final long[] subProcIds) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span>    switch (type) {<a name="line.725"></a>
-<span class="sourceLineNo">726</span>      case INSERT:<a name="line.726"></a>
-<span class="sourceLineNo">727</span>        if (subProcIds == null) {<a name="line.727"></a>
-<span class="sourceLineNo">728</span>          storeTracker.insert(procId);<a name="line.728"></a>
-<span class="sourceLineNo">729</span>        } else if (procId == Procedure.NO_PROC_ID) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span>          storeTracker.insert(subProcIds);<a name="line.730"></a>
-<span class="sourceLineNo">731</span>        } else {<a name="line.731"></a>
-<span class="sourceLineNo">732</span>          storeTracker.insert(procId, subProcIds);<a name="line.732"></a>
-<span class="sourceLineNo">733</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.733"></a>
-<span class="sourceLineNo">734</span>        }<a name="line.734"></a>
-<span class="sourceLineNo">735</span>        break;<a name="line.735"></a>
-<span class="sourceLineNo">736</span>      case UPDATE:<a name="line.736"></a>
-<span class="sourceLineNo">737</span>        storeTracker.update(procId);<a name="line.737"></a>
-<span class="sourceLineNo">738</span>        holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.738"></a>
-<span class="sourceLineNo">739</span>        break;<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      case DELETE:<a name="line.740"></a>
-<span class="sourceLineNo">741</span>        if (subProcIds != null &amp;&amp; subProcIds.length &gt; 0) {<a name="line.741"></a>
-<span class="sourceLineNo">742</span>          storeTracker.delete(subProcIds);<a name="line.742"></a>
-<span class="sourceLineNo">743</span>          holdingCleanupTracker.setDeletedIfSet(subProcIds);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>        } else {<a name="line.744"></a>
-<span class="sourceLineNo">745</span>          storeTracker.delete(procId);<a name="line.745"></a>
-<span class="sourceLineNo">746</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.746"></a>
-<span class="sourceLineNo">747</span>        }<a name="line.747"></a>
-<span class="sourceLineNo">748</span>        break;<a name="line.748"></a>
-<span class="sourceLineNo">749</span>      default:<a name="line.749"></a>
-<span class="sourceLineNo">750</span>        throw new RuntimeException("invalid push type " + type);<a name="line.750"></a>
-<span class="sourceLineNo">751</span>    }<a name="line.751"></a>
-<span class="sourceLineNo">752</span>  }<a name="line.752"></a>
-<span class="sourceLineNo">753</span><a name="line.753"></a>
-<span class="sourceLineNo">754</span>  private boolean isSyncAborted() {<a name="line.754"></a>
-<span class="sourceLineNo">755</span>    return syncException.get() != null;<a name="line.755"></a>
-<span class="sourceLineNo">756</span>  }<a name="line.756"></a>
-<span class="sourceLineNo">757</span><a name="line.757"></a>
-<span class="sourceLineNo">758</span>  private void syncLoop() throws Throwable {<a name="line.758"></a>
-<span class="sourceLineNo">759</span>    long totalSyncedToStore = 0;<a name="line.759"></a>
-<span class="sourceLineNo">760</span>    inSync.set(false);<a name="line.760"></a>
-<span class="sourceLineNo">761</span>    lock.lock();<a name="line.761"></a>
-<span class="sourceLineNo">762</span>    try {<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (isRunning()) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        try {<a name="line.764"></a>
-<span class="sourceLineNo">765</span>          // Wait until new data is available<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          if (slotIndex == 0) {<a name="line.766"></a>
-<span class="sourceLineNo">767</span>            if (!loading.get()) {<a name="line.767"></a>
-<span class="sourceLineNo">768</span>              periodicRoll();<a name="line.768"></a>
-<span class="sourceLineNo">769</span>            }<a name="line.769"></a>
-<span class="sourceLineNo">770</span><a name="line.770"></a>
-<span class="sourceLineNo">771</span>            if (LOG.isTraceEnabled()) {<a name="line.771"></a>
-<span class="sourceLineNo">772</span>              float rollTsSec = getMillisFromLastRoll() / 1000.0f;<a name="line.772"></a>
-<span class="sourceLineNo">773</span>              LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)",<a name="line.773"></a>
-<span class="sourceLineNo">774</span>                        StringUtils.humanSize(totalSynced.get()),<a name="line.774"></a>
-<span class="sourceLineNo">775</span>                        StringUtils.humanSize(totalSynced.get() / rollTsSec)));<a name="line.775"></a>
+<span class="sourceLineNo">663</span>  private void releaseSlot(final ByteSlot slot) {<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    slot.reset();<a name="line.664"></a>
+<span class="sourceLineNo">665</span>    slotsCache.offer(slot);<a name="line.665"></a>
+<span class="sourceLineNo">666</span>  }<a name="line.666"></a>
+<span class="sourceLineNo">667</span><a name="line.667"></a>
+<span class="sourceLineNo">668</span>  private enum PushType { INSERT, UPDATE, DELETE }<a name="line.668"></a>
+<span class="sourceLineNo">669</span><a name="line.669"></a>
+<span class="sourceLineNo">670</span>  private long pushData(final PushType type, final ByteSlot slot,<a name="line.670"></a>
+<span class="sourceLineNo">671</span>      final long procId, final long[] subProcIds) {<a name="line.671"></a>
+<span class="sourceLineNo">672</span>    if (!isRunning()) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>      throw new RuntimeException("the store must be running before inserting data");<a name="line.673"></a>
+<span class="sourceLineNo">674</span>    }<a name="line.674"></a>
+<span class="sourceLineNo">675</span>    if (logs.isEmpty()) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span>      throw new RuntimeException("recoverLease() must be called before inserting data");<a name="line.676"></a>
+<span class="sourceLineNo">677</span>    }<a name="line.677"></a>
+<span class="sourceLineNo">678</span><a name="line.678"></a>
+<span class="sourceLineNo">679</span>    long logId = -1;<a name="line.679"></a>
+<span class="sourceLineNo">680</span>    lock.lock();<a name="line.680"></a>
+<span class="sourceLineNo">681</span>    try {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>      // Wait for the sync to be completed<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      while (true) {<a name="line.683"></a>
+<span class="sourceLineNo">684</span>        if (!isRunning()) {<a name="line.684"></a>
+<span class="sourceLineNo">685</span>          throw new RuntimeException("store no longer running");<a name="line.685"></a>
+<span class="sourceLineNo">686</span>        } else if (isSyncAborted()) {<a name="line.686"></a>
+<span class="sourceLineNo">687</span>          throw new RuntimeException("sync aborted", syncException.get());<a name="line.687"></a>
+<span class="sourceLineNo">688</span>        } else if (inSync.get()) {<a name="line.688"></a>
+<span class="sourceLineNo">689</span>          syncCond.await();<a name="line.689"></a>
+<span class="sourceLineNo">690</span>        } else if (slotIndex &gt;= syncMaxSlot) {<a name="line.690"></a>
+<span class="sourceLineNo">691</span>          slotCond.signal();<a name="line.691"></a>
+<span class="sourceLineNo">692</span>          syncCond.await();<a name="line.692"></a>
+<span class="sourceLineNo">693</span>        } else {<a name="line.693"></a>
+<span class="sourceLineNo">694</span>          break;<a name="line.694"></a>
+<span class="sourceLineNo">695</span>        }<a name="line.695"></a>
+<span class="sourceLineNo">696</span>      }<a name="line.696"></a>
+<span class="sourceLineNo">697</span><a name="line.697"></a>
+<span class="sourceLineNo">698</span>      final long pushSyncId = syncId.get();<a name="line.698"></a>
+<span class="sourceLineNo">699</span>      updateStoreTracker(type, procId, subProcIds);<a name="line.699"></a>
+<span class="sourceLineNo">700</span>      slots[slotIndex++] = slot;<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      logId = flushLogId;<a name="line.701"></a>
+<span class="sourceLineNo">702</span><a name="line.702"></a>
+<span class="sourceLineNo">703</span>      // Notify that there is new data<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (slotIndex == 1) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        waitCond.signal();<a name="line.705"></a>
+<span class="sourceLineNo">706</span>      }<a name="line.706"></a>
+<span class="sourceLineNo">707</span><a name="line.707"></a>
+<span class="sourceLineNo">708</span>      // Notify that the slots are full<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      if (slotIndex == syncMaxSlot) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        waitCond.signal();<a name="line.710"></a>
+<span class="sourceLineNo">711</span>        slotCond.signal();<a name="line.711"></a>
+<span class="sourceLineNo">712</span>      }<a name="line.712"></a>
+<span class="sourceLineNo">713</span><a name="line.713"></a>
+<span class="sourceLineNo">714</span>      while (pushSyncId == syncId.get() &amp;&amp; isRunning()) {<a name="line.714"></a>
+<span class="sourceLineNo">715</span>        syncCond.await();<a name="line.715"></a>
+<span class="sourceLineNo">716</span>      }<a name="line.716"></a>
+<span class="sourceLineNo">717</span>    } catch (InterruptedException e) {<a name="line.717"></a>
+<span class="sourceLineNo">718</span>      Thread.currentThread().interrupt();<a name="line.718"></a>
+<span class="sourceLineNo">719</span>      sendAbortProcessSignal();<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      throw new RuntimeException(e);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>    } finally {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>      lock.unlock();<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      if (isSyncAborted()) {<a name="line.723"></a>
+<span class="sourceLineNo">724</span>        throw new RuntimeException("sync aborted", syncException.get());<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      }<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>    return logId;<a name="line.727"></a>
+<span class="sourceLineNo">728</span>  }<a name="line.728"></a>
+<span class="sourceLineNo">729</span><a name="line.729"></a>
+<span class="sourceLineNo">730</span>  private void updateStoreTracker(final PushType type,<a name="line.730"></a>
+<span class="sourceLineNo">731</span>      final long procId, final long[] subProcIds) {<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    switch (type) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span>      case INSERT:<a name="line.733"></a>
+<span class="sourceLineNo">734</span>        if (subProcIds == null) {<a name="line.734"></a>
+<span class="sourceLineNo">735</span>          storeTracker.insert(procId);<a name="line.735"></a>
+<span class="sourceLineNo">736</span>        } else if (procId == Procedure.NO_PROC_ID) {<a name="line.736"></a>
+<span class="sourceLineNo">737</span>          storeTracker.insert(subProcIds);<a name="line.737"></a>
+<span class="sourceLineNo">738</span>        } else {<a name="line.738"></a>
+<span class="sourceLineNo">739</span>          storeTracker.insert(procId, subProcIds);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.740"></a>
+<span class="sourceLineNo">741</span>        }<a name="line.741"></a>
+<span class="sourceLineNo">742</span>        break;<a name="line.742"></a>
+<span class="sourceLineNo">743</span>      case UPDATE:<a name="line.743"></a>
+<span class="sourceLineNo">744</span>        storeTracker.update(procId);<a name="line.744"></a>
+<span class="sourceLineNo">745</span>        holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.745"></a>
+<span class="sourceLineNo">746</span>        break;<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      case DELETE:<a name="line.747"></a>
+<span class="sourceLineNo">748</span>        if (subProcIds != null &amp;&amp; subProcIds.length &gt; 0) {<a name="line.748"></a>
+<span class="sourceLineNo">749</span>          storeTracker.delete(subProcIds);<a name="line.749"></a>
+<span class="sourceLineNo">750</span>          holdingCleanupTracker.setDeletedIfSet(subProcIds);<a name="line.750"></a>
+<span class="sourceLineNo">751</span>        } else {<a name="line.751"></a>
+<span class="sourceLineNo">752</span>          storeTracker.delete(procId);<a name="line.752"></a>
+<span class="sourceLineNo">753</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.753"></a>
+<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
+<span class="sourceLineNo">755</span>        break;<a name="line.755"></a>
+<span class="sourceLineNo">756</span>      default:<a name="line.756"></a>
+<span class="sourceLineNo">757</span>        throw new RuntimeException("invalid push type " + type);<a name="line.757"></a>
+<span class="sourceLineNo">758</span>    }<a name="line.758"></a>
+<span class="sourceLineNo">759</span>  }<a name="line.759"></a>
+<span class="sourceLineNo">760</span><a name="line.760"></a>
+<span class="sourceLineNo">761</span>  private boolean isSyncAborted() {<a name="line.761"></a>
+<span class="sourceLineNo">762</span>    return syncException.get() != null;<a name="line.762"></a>
+<span class="sourceLineNo">763</span>  }<a name="line.763"></a>
+<span class="sourceLineNo">764</span><a name="line.764"></a>
+<span class="sourceLineNo">765</span>  private void syncLoop() throws Throwable {<a name="line.765"></a>
+<span class="sourceLineNo">766</span>    long totalSyncedToStore = 0;<a name="line.766"></a>
+<span class="sourceLineNo">767</span>    inSync.set(false);<a name="line.767"></a>
+<span class="sourceLineNo">768</span>    lock.lock();<a name="line.768"></a>
+<span class="sourceLineNo">769</span>    try {<a name="line.769"></a>
+<span class="sourceLineNo">770</span>      while (isRunning()) {<a name="line.770"></a>
+<span class="sourceLineNo">771</span>        try {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>          // Wait until new data is available<a name="line.772"></a>
+<span class="sourceLineNo">773</span>          if (slotIndex == 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>            if (!loading.get()) {<a name="line.774"></a>
+<span class="sourceLineNo">775</span>              periodicRoll();<a name="line.775"></a>
 <span class="sourceLineNo">776</span>            }<a name="line.776"></a>
 <span class="sourceLineNo">777</span><a name="line.777"></a>
-<span class="sourceLineNo">778</span>            waitCond.await(getMillisToNextPeriodicRoll(), TimeUnit.MILLISECONDS);<a name="line.778"></a>
-<span class="sourceLineNo">779</span>            if (slotIndex == 0) {<a name="line.779"></a>
-<span class="sourceLineNo">780</span>              // no data.. probably a stop() or a periodic roll<a name="line.780"></a>
-<span class="sourceLineNo">781</span>              continue;<a name="line.781"></a>
-<span class="sourceLineNo">782</span>            }<a name="line.782"></a>
-<span class="sourceLineNo">783</span>          }<a name="line.783"></a>
-<span class="sourceLineNo">784</span>          // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing<a name="line.784"></a>
-<span class="sourceLineNo">785</span>          syncMaxSlot = runningProcCount;<a name="line.785"></a>
-<span class="sourceLineNo">786</span>          assert syncMaxSlot &gt; 0 : "unexpected syncMaxSlot=" + syncMaxSlot;<a name="line.786"></a>
-<span class="sourceLineNo">787</span>          final long syncWaitSt = System.currentTimeMillis();<a name="line.787"></a>
-<span class="sourceLineNo">788</span>          if (slotIndex != syncMaxSlot) {<a name="line.788"></a>
-<span class="sourceLineNo">789</span>            slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);<a name="line.789"></a>
+<span class="sourceLineNo">778</span>            if (LOG.isTraceEnabled()) {<a name="line.778"></a>
+<span class="sourceLineNo">779</span>              float rollTsSec = getMillisFromLastRoll() / 1000.0f;<a name="line.779"></a>
+<span class="sourceLineNo">780</span>              LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)",<a name="line.780"></a>
+<span class="sourceLineNo">781</span>                        StringUtils.humanSize(totalSynced.get()),<a name="line.781"></a>
+<span class="sourceLineNo">782</span>                        StringUtils.humanSize(totalSynced.get() / rollTsSec)));<a name="line.782"></a>
+<span class="sourceLineNo">783</span>            }<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>            waitCond.await(getMillisToNextPeriodicRoll(), TimeUnit.MILLISECONDS);<a name="line.785"></a>
+<span class="sourceLineNo">786</span>            if (slotIndex == 0) {<a name="line.786"></a>
+<span class="sourceLineNo">787</span>              // no data.. probably a stop() or a periodic roll<a name="line.787"></a>
+<span class="sourceLineNo">788</span>              continue;<a name="line.788"></a>
+<span class="sourceLineNo">789</span>            }<a name="line.789"></a>
 <span class="sourceLineNo">790</span>          }<a name="line.790"></a>
-<span class="sourceLineNo">791</span><a name="line.791"></a>
-<span class="sourceLineNo">792</span>          final long currentTs = System.currentTimeMillis();<a name="line.792"></a>
-<span class="sourceLineNo">793</span>          final long syncWaitMs = currentTs - syncWaitSt;<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          final float rollSec = getMillisFromLastRoll() / 1000.0f;<a name="line.794"></a>
-<span class="sourceLineNo">795</span>          final float syncedPerSec = totalSyncedToStore / rollSec;<a name="line.795"></a>
-<span class="sourceLineNo">796</span>          if (LOG.isTraceEnabled() &amp;&amp; (syncWaitMs &gt; 10 || slotIndex &lt; syncMaxSlot)) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span>            LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)",<a name="line.797"></a>
-<span class="sourceLineNo">798</span>                      StringUtils.humanTimeDiff(syncWaitMs), slotIndex,<a name="line.798"></a>
-<span class="sourceLineNo">799</span>                      StringUtils.humanSize(totalSyncedToStore),<a name="line.799"></a>
-<span class="sourceLineNo">800</span>                      StringUtils.humanSize(syncedPerSec)));<a name="line.800"></a>
-<span class="sourceLineNo">801</span>          }<a name="line.801"></a>
-<span class="sourceLineNo">802</span><a name="line.802"></a>
-<span class="sourceLineNo">803</span>          // update webui circular buffers (TODO: get rid of allocations)<a name="line.803"></a>
-<span class="sourceLineNo">804</span>          final SyncMetrics syncMetrics = new SyncMetrics();<a name="line.804"></a>
-<span class="sourceLineNo">805</span>          syncMetrics.timestamp = currentTs;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>          syncMetrics.syncWaitMs = syncWaitMs;<a name="line.806"></a>
-<span class="sourceLineNo">807</span>          syncMetrics.syncedEntries = slotIndex;<a name="line.807"></a>
-<span class="sourceLineNo">808</span>          syncMetrics.totalSyncedBytes = totalSyncedToStore;<a name="line.808"></a>
-<span class="sourceLineNo">809</span>          syncMetrics.syncedPerSec = syncedPerSec;<a name="line.809"></a>
-<span class="sourceLineNo">810</span>          syncMetricsQueue.add(syncMetrics);<a name="line.810"></a>
-<span class="sourceLineNo">811</span><a name="line.811"></a>
-<span class="sourceLineNo">812</span>          // sync<a name="line.812"></a>
-<span class="sourceLineNo">813</span>          inSync.set(true);<a name="line.813"></a>
-<span class="sourceLineNo">814</span>          long slotSize = syncSlots();<a name="line.814"></a>
-<span class="sourceLineNo">815</span>          logs.getLast().addToSize(slotSize);<a name="line.815"></a>
-<span class="sourceLineNo">816</span>          totalSyncedToStore = totalSynced.addAndGet(slotSize);<a name="line.816"></a>
-<span class="sourceLineNo">817</span>          slotIndex = 0;<a name="line.817"></a>
-<span class="sourceLineNo">818</span>          inSync.set(false);<a name="line.818"></a>
-<span class="sourceLineNo">819</span>          syncId.incrementAndGet();<a name="line.819"></a>
-<span class="sourceLineNo">820</span>        } catch (InterruptedException e) {<a name="line.820"></a>
-<span class="sourceLineNo">821</span>          Thread.currentThread().interrupt();<a name="line.821"></a>
-<span class="sourceLineNo">822</span>          syncException.compareAndSet(null, e);<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          sendAbortProcessSignal();<a name="line.823"></a>
-<span class="sourceLineNo">824</span>          throw e;<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        } catch (Throwable t) {<a name="line.825"></a>
-<span class="sourceLineNo">826</span>          syncException.compareAndSet(null, t);<a name="line.826"></a>
-<span class="sourceLineNo">827</span>          sendAbortProcessSignal();<a name="line.827"></a>
-<span class="sourceLineNo">828</span>          throw t;<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        } finally {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          syncCond.signalAll();<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span>    } finally {<a name="line.833"></a>
-<span class="sourceLineNo">834</span>      lock.unlock();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span>  }<a name="line.836"></a>
-<span class="sourceLineNo">837</span><a name="line.837"></a>
-<span class="sourceLineNo">838</span>  public ArrayList&lt;SyncMetrics&gt; getSyncMetrics() {<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    lock.lock();<a name="line.839"></a>
-<span class="sourceLineNo">840</span>    try {<a name="line.840"></a>
-<span class="sourceLineNo">841</span>      return new ArrayList&lt;&gt;(syncMetricsQueue);<a name="line.841"></a>
-<span class="sourceLineNo">842</span>    } finally {<a name="line.842"></a>
-<span class="sourceLineNo">843</span>      lock.unlock();<a name="line.843"></a>
-<span class="sourceLineNo">844</span>    }<a name="line.844"></a>
-<span class="sourceLineNo">845</span>  }<a name="line.845"></a>
-<span class="sourceLineNo">846</span><a name="line.846"></a>
-<span class="sourceLineNo">847</span>  private long syncSlots() throws Throwable {<a name="line.847"></a>
-<span class="sourceLineNo">848</span>    int retry = 0;<a name="line.848"></a>
-<span class="sourceLineNo">849</span>    int logRolled = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span>    long totalSynced = 0;<a name="line.850"></a>
-<span class="sourceLineNo">851</span>    do {<a name="line.851"></a>
-<span class="sourceLineNo">852</span>      try {<a name="line.852"></a>
-<span class="sourceLineNo">853</span>        totalSynced = syncSlots(stream, slots, 0, slotIndex);<a name="line.853"></a>
-<span class="sourceLineNo">854</span>        break;<a name="line.854"></a>
-<span class="sourceLineNo">855</span>      } catch (Throwable e) {<a name="line.855"></a>
-<span class="sourceLineNo">856</span>        LOG.warn("unable to sync slots, retry=" + retry);<a name="line.856"></a>
-<span class="sourceLineNo">857</span>        if (++retry &gt;= maxRetriesBeforeRoll) {<a name="line.857"></a>
-<span class="sourceLineNo">858</span>          if (logRolled &gt;= maxSyncFailureRoll &amp;&amp; isRunning()) {<a name="line.858"></a>
-<span class="sourceLineNo">859</span>            LOG.error("Sync slots after log roll failed, abort.", e);<a name="line.859"></a>
-<span class="sourceLineNo">860</span>            throw e;<a name="line.860"></a>
-<span class="sourceLineNo">861</span>          }<a name="line.861"></a>
-<span class="sourceLineNo">862</span><a name="line.862"></a>
-<span class="sourceLineNo">863</span>          if (!rollWriterWithRetries()) {<a name="line.863"></a>
-<span class="sourceLineNo">864</span>            throw e;<a name="line.864"></a>
-<span class="sourceLineNo">865</span>          }<a name="line.865"></a>
-<span class="sourceLineNo">866</span><a name="line.866"></a>
-<span class="sourceLineNo">867</span>          logRolled++;<a name="line.867"></a>
-<span class="sourceLineNo">868</span>          retry = 0;<a name="line.868"></a>
-<span class="sourceLineNo">869</span>        }<a name="line.869"></a>
-<span class="sourceLineNo">870</span>      }<a name="line.870"></a>
-<span class="sourceLineNo">871</span>    } while (isRunning());<a name="line.871"></a>
-<span class="sourceLineNo">872</span>    return totalSynced;<a name="line.872"></a>
-<span class="sourceLineNo">873</span>  }<a name="line.873"></a>
-<span class="sourceLineNo">874</span><a name="line.874"></a>
-<span class="sourceLineNo">875</span>  protected long syncSlots(final FSDataOutputStream stream, final ByteSlot[] slots,<a name="line.875"></a>
-<span class="sourceLineNo">876</span>      final int offset, final int count) throws IOException {<a name="line.876"></a>
-<span class="sourceLineNo">877</span>    long totalSynced = 0;<a name="line.877"></a>
-<span class="sourceLineNo">878</span>    for (int i = 0; i &lt; count; ++i) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span>      final ByteSlot data = slots[offset + i];<a name="line.879"></a>
-<span class="sourceLineNo">880</span>      data.writeTo(stream);<a name="line.880"></a>
-<span class="sourceLineNo">881</span>      totalSynced += data.size();<a name="line.881"></a>
-<span class="sourceLineNo">882</span>    }<a name="line.882"></a>
-<span class="sourceLineNo">883</span><a name="line.883"></a>
-<span class="sourceLineNo">884</span>    syncStream(stream);<a name="line.884"></a>
-<span class="sourceLineNo">885</span>    sendPostSyncSignal();<a name="line.885"></a>
-<span class="sourceLineNo">886</span><a name="line.886"></a>
-<span class="sourceLineNo">887</span>    if (LOG.isTraceEnabled()) {<a name="line.887"></a>
-<span class="sourceLineNo">888</span>      LOG.trace("Sync slots=" + count + '/' + syncMaxSlot +<a name="line.888"></a>
-<span class="sourceLineNo">889</span>                ", flushed=" + StringUtils.humanSize(totalSynced));<a name="line.889"></a>
-<span class="sourceLineNo">890</span>    }<a name="line.890"></a>
-<span class="sourceLineNo">891</span>    return totalSynced;<a name="line.891"></a>
-<span class="sourceLineNo">892</span>  }<a name="line.892"></a>
+<span class="sourceLineNo">791</span>          // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing<a name="line.791"></a>
+<span class="sourceLineNo">792</span>          syncMaxSlot = runningProcCount;<a name="line.792"></a>
+<span class="sourceLineNo">793</span>          assert syncMaxSlot &gt; 0 : "unexpected syncMaxSlot=" + syncMaxSlot;<a name="line.793"></a>
+<span class="sourceLineNo">794</span>          final long syncWaitSt = System.currentTimeMillis();<a name="line.794"></a>
+<span class="sourceLineNo">795</span>          if (slotIndex != syncMaxSlot) {<a name="line.795"></a>
+<span class="sourceLineNo">796</span>            slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);<a name="line.796"></a>
+<span class="sourceLineNo">797</span>          }<a name="line.797"></a>
+<span class="sourceLineNo">798</span><a name="line.798"></a>
+<span class="sourceLineNo">799</span>          final long currentTs = System.currentTimeMillis();<a name="line.799"></a>
+<span class="sourceLineNo">800</span>          final long syncWaitMs = currentTs - syncWaitSt;<a name="line.800"></a>
+<span class="sourceLineNo">801</span>          final float rollSec = getMillisFromLastRoll() / 1000.0f;<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          final float syncedPerSec = totalSyncedToStore / rollSec;<a name="line.802"></a>
+<span class="sourceLineNo">803</span>          if (LOG.isTraceEnabled() &amp;&amp; (syncWaitMs &gt; 10 || slotIndex &lt; syncMaxSlot)) {<a name="line.803"></a>
+<span class="sourceLineNo">804</span>            LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)",<a name="line.804"></a>
+<span class="sourceLineNo">805</span>                      StringUtils.humanTimeDiff(syncWaitMs), slotIndex,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>                      StringUtils.humanSize(totalSyncedToStore),<a name="line.806"></a>
+<span class="sourceLineNo">807</span>                      StringUtils.humanSize(syncedPerSec)));<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          }<a name="line.808"></a>
+<span class="sourceLineNo">809</span><a name="line.809"></a>
+<span class="sourceLineNo">810</span>          // update webui circular buffers (TODO: get rid of allocations)<a name="line.810"></a>
+<span class="sourceLineNo">811</span>          final SyncMetrics syncMetrics = new SyncMetrics();<a name="line.811"></a>
+<span class="sourceLineNo">812</span>          syncMetrics.timestamp = currentTs;<a name="line.812"></a>
+<span class="sourceLineNo">813</span>          syncMetrics.syncWaitMs = syncWaitMs;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>          syncMetrics.syncedEntries = slotIndex;<a name="line.814"></a>
+<span class="sourceLineNo">815</span>          syncMetrics.totalSyncedBytes = totalSyncedToStore;<a name="line.815"></a>
+<span class="sourceLineNo">816</span>          syncMetrics.syncedPerSec = syncedPerSec;<a name="line.816"></a>
+<span class="sourceLineNo">817</span>          syncMetricsQueue.add(syncMetrics);<a name="line.817"></a>
+<span class="sourceLineNo">818</span><a name="line.818"></a>
+<span class="sourceLineNo">819</span>          // sync<a name="line.819"></a>
+<span class="sourceLineNo">820</span>          inSync.set(true);<a name="line.820"></a>
+<span class="sourceLineNo">821</span>          long slotSize = syncSlots();<a name="line.821"></a>
+<span class="sourceLineNo">822</span>          logs.getLast().addToSize(slotSize);<a name="line.822"></a>
+<span class="sourceLineNo">823</span>          totalSyncedToStore = totalSynced.addAndGet(slotSize);<a name="line.823"></a>
+<span class="sourceLineNo">824</span>          slotIndex = 0;<a name="line.824"></a>
+<span class="sourceLineNo">825</span>          inSync.set(false);<a name="line.825"></a>
+<span class="sourceLineNo">826</span>          syncId.incrementAndGet();<a name="line.826"></a>
+<span class="sourceLineNo">827</span>        } catch (InterruptedException e) {<a name="line.827"></a>
+<span class="sourceLineNo">828</span>          Thread.currentThread().interrupt();<a name="line.828"></a>
+<span class="sourceLineNo">829</span>          syncException.compareAndSet(null, e);<a name="line.829"></a>
+<span class="sourceLineNo">830</span>          sendAbortProcessSignal();<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          throw e;<a name="line.831"></a>
+<span class="sourceLineNo">832</span>        } catch (Throwable t) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>          syncException.compareAndSet(null, t);<a name="line.833"></a>
+<span class="sourceLineNo">834</span>          sendAbortProcessSignal();<a name="line.834"></a>
+<span class="sourceLineNo">835</span>          throw t;<a name="line.835"></a>
+<span class="sourceLineNo">836</span>        } finally {<a name="line.836"></a>
+<span class="sourceLineNo">837</span>          syncCond.signalAll();<a name="line.837"></a>
+<span class="sourceLineNo">838</span>        }<a name="line.838"></a>
+<span class="sourceLineNo">839</span>      }<a name="line.839"></a>
+<span class="sourceLineNo">840</span>    } finally {<a name="line.840"></a>
+<span class="sourceLineNo">841</span>      lock.unlock();<a name="line.841"></a>
+<span class="sourceLineNo">842</span>    }<a name="line.842"></a>
+<span class="sourceLineNo">843</span>  }<a name="line.843"></a>
+<span class="sourceLineNo">844</span><a name="line.844"></a>
+<span class="sourceLineNo">845</span>  public ArrayList&lt;SyncMetrics&gt; getSyncMetrics() {<a name="line.845"></a>
+<span class="sourceLineNo">846</span>    lock.lock();<a name="line.846"></a>
+<span class="sourceLineNo">847</span>    try {<a name="line.847"></a>
+<span class="sourceLineNo">848</span>      return new ArrayList&lt;&gt;(syncMetricsQueue);<a name="line.848"></a>
+<span class="sourceLineNo">849</span>    } finally {<a name="line.849"></a>
+<span class="sourceLineNo">850</span>      lock.unlock();<a name="line.850"></a>
+<span class="sourceLineNo">851</span>    }<a name="line.851"></a>
+<span class="sourceLineNo">852</span>  }<a name="line.852"></a>
+<span class="sourceLineNo">853</span><a name="line.853"></a>
+<span class="sourceLineNo">854</span>  private long syncSlots() throws Throwable {<a name="line.854"></a>
+<span class="sourceLineNo">855</span>    int retry = 0;<a name="line.855"></a>
+<span class="sourceLineNo">856</span>    int logRolled = 0;<a name="line.856"></a>
+<span class="sourceLineNo">857</span>    long totalSynced = 0;<a name="line.857"></a>
+<span class="sourceLineNo">858</span>    do {<a name="line.858"></a>
+<span class="sourceLineNo">859</span>      try {<a name="line.859"></a>
+<span class="sourceLineNo">860</span>        totalSynced = syncSlots(stream, slots, 0, slotIndex);<a name="line.860"></a>
+<span class="sourceLineNo">861</span>        break;<a name="line.861"></a>
+<span class="sourceLineNo">862</span>      } catch (Throwable e) {<a name="line.862"></a>
+<span class="sourceLineNo">863</span>        LOG.warn("unable to sync slots, retry=" + retry);<a name="line.863"></a>
+<span class="sourceLineNo">864</span>        if (++retry &gt;= maxRetriesBeforeRoll) {<a name="line.864"></a>
+<span class="sourceLineNo">865</span>          if (logRolled &gt;= maxSyncFailureRoll &amp;&amp; isRunning()) {<a name="line.865"></a>
+<span class="sourceLineNo">866</span>            LOG.error("Sync slots after log roll failed, abort.", e);<a name="line.866"></a>
+<span class="sourceLineNo">867</span>            throw e;<a name="line.867"></a>
+<span class="sourceLineNo">868</span>          }<a name="line.868"></a>
+<span class="sourceLineNo">869</span><a name="line.869"></a>
+<span class="sourceLineNo">870</span>          if (!rollWriterWithRetries()) {<a name="line.870"></a>
+<span class="sourceLineNo">871</span>            throw e;<a name="line.871"></a>
+<span class="sourceLineNo">872</span>          }<a name="line.872"></a>
+<span class="sourceLineNo">873</span><a name="line.873"></a>
+<span class="sourceLineNo">874</span>          logRolled++;<a name="line.874"></a>
+<span class="sourceLineNo">875</span>          retry = 0;<a name="line.875"></a>
+<span class="sourceLineNo">876</span>        }<a name="line.876"></a>
+<span class="sourceLineNo">877</span>      }<a name="line.877"></a>
+<span class="sourceLineNo">878</span>    } while (isRunning());<a name="line.878"></a>
+<span class="sourceLineNo">879</span>    return totalSynced;<a name="line.879"></a>
+<span class="sourceLineNo">880</span>  }<a name="line.880"></a>
+<span class="sourceLineNo">881</span><a name="line.881"></a>
+<span class="sourceLineNo">882</span>  protected long syncSlots(final FSDataOutputStream stream, final ByteSlot[] slots,<a name="line.882"></a>
+<span class="sourceLineNo">883</span>      final int offset, final int count) throws IOException {<a name="line.883"></a>
+<span class="sourceLineNo">884</span>    long totalSynced = 0;<a name="line.884"></a>
+<span class="sourceLineNo">885</span>    for (int i = 0; i &lt; count; ++i) {<a name="line.885"></a>
+<span class="sourceLineNo">886</span>      final ByteSlot data = slots[offset + i];<a name="line.886"></a>
+<span class="sourceLineNo">887</span>      data.writeTo(stream);<a name="line.887"></a>
+<span class="sourceLineNo">888</span>      totalSynced += data.size();<a name="line.888"></a>
+<span class="sourceLineNo">889</span>    }<a name="line.889"></a>
+<span class="sourceLineNo">890</span><a name="line.890"></a>
+<span class="sourceLineNo">891</span>    syncStream(stream);<a name="line.891"></a>
+<span class="sourceLineNo">892</span>    sendPostSyncSignal();<a name="line.892"></a>
 <span class="sourceLineNo">893</span><a name="line.893"></a>
-<span class="sourceLineNo">894</span>  protected void syncStream(final FSDataOutputStream stream) throws IOException {<a name="line.894"></a>
-<span class="sourceLineNo">895</span>    if (useHsync) {<a name="line.895"></a>
-<span class="sourceLineNo">896</span>      stream.hsync();<a name="line.896"></a>
-<span class="sourceLineNo">897</span>    } else {<a name="line.897"></a>
-<span class="sourceLineNo">898</span>      stream.hflush();<a name="line.898"></a>
-<span class="sourceLineNo">899</span>    }<a name="line.899"></a>
-<span class="sourceLineNo">900</span>  }<a name="line.900"></a>
-<span class="sourceLineNo">901</span><a name="line.901"></a>
-<span class="sourceLineNo">902</span>  private boolean rollWriterWithRetries() {<a name="line.902"></a>
-<span class="sourceLineNo">903</span>    for (int i = 0; i &lt; rollRetries &amp;&amp; isRunning(); ++i) {<a name="line.903"></a>
-<span class="sourceLineNo">904</span>      if (i &gt; 0) Threads.sleepWithoutInterrupt(waitBeforeRoll * i);<a name="line.904"></a>
-<span class="sourceLineNo">905</span><a name="line.905"></a>
-<span class="sourceLineNo">906</span>      try {<a name="line.906"></a>
-<span class="sourceLineNo">907</span>        if (rollWriter()) {<a name="line.907"></a>
-<span class="sourceLineNo">908</span>          return true;<a name="line.908"></a>
-<span class="sourceLineNo">909</span>        }<a name="line.909"></a>
-<span class="sourceLineNo">910</span>      } catch (IOException e) {<a name="line.910"></a>
-<span class="sourceLineNo">911</span>        LOG.warn("Unable to roll the log, attempt=" + (i + 1), e);<a name="line.911"></a>
-<span class="sourceLineNo">912</span>      }<a name="line.912"></a>
-<span class="sourceLineNo">913</span>    }<a name="line.913"></a>
-<span class="sourceLineNo">914</span>    LOG.error(HBaseMarkers.FATAL, "Unable to roll the log");<a name="line.914"></a>
-<span class="sourceLineNo">915</span>    return false;<a name="line.915"></a>
-<span class="sourceLineNo">916</span>  }<a name="line.916"></a>
-<span class="sourceLineNo">917</span><a name="line.917"></a>
-<span class="sourceLineNo">918</span>  private boolean tryRollWriter() {<a name="line.918"></a>
-<span class="sourceLineNo">919</span>    try {<a name="line.919"></a>
-<span class="sourceLineNo">920</span>      return rollWriter();<a name="line.920"></a>
-<span class="sourceLineNo">921</span>    } catch (IOException e) {<a name="line.921"></a>
-<span class="sourceLineNo">922</span>      LOG.warn("Unable to roll the log", e);<a name="line.922"></a>
-<span class="sourceLineNo">923</span>      return false;<a name="line.923"></a>
-<span class="sour

<TRUNCATED>

[04/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
index c63e65a..21df2b9 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
@@ -432,900 +432,907 @@
 <span class="sourceLineNo">424</span>      it.next(); // Skip the current log<a name="line.424"></a>
 <span class="sourceLineNo">425</span><a name="line.425"></a>
 <span class="sourceLineNo">426</span>      ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {<a name="line.426"></a>
-<span class="sourceLineNo">427</span>        @Override<a name="line.427"></a>
-<span class="sourceLineNo">428</span>        public void setMaxProcId(long maxProcId) {<a name="line.428"></a>
-<span class="sourceLineNo">429</span>          loader.setMaxProcId(maxProcId);<a name="line.429"></a>
-<span class="sourceLineNo">430</span>        }<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>        @Override<a name="line.432"></a>
-<span class="sourceLineNo">433</span>        public void load(ProcedureIterator procIter) throws IOException {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>          loader.load(procIter);<a name="line.434"></a>
-<span class="sourceLineNo">435</span>        }<a name="line.435"></a>
-<span class="sourceLineNo">436</span><a name="line.436"></a>
-<span class="sourceLineNo">437</span>        @Override<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.438"></a>
-<span class="sourceLineNo">439</span>          loader.handleCorrupted(procIter);<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>        @Override<a name="line.442"></a>
-<span class="sourceLineNo">443</span>        public void markCorruptedWAL(ProcedureWALFile log, IOException e) {<a name="line.443"></a>
-<span class="sourceLineNo">444</span>          if (corruptedLogs == null) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span>            corruptedLogs = new HashSet&lt;&gt;();<a name="line.445"></a>
-<span class="sourceLineNo">446</span>          }<a name="line.446"></a>
-<span class="sourceLineNo">447</span>          corruptedLogs.add(log);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>          // TODO: sideline corrupted log<a name="line.448"></a>
-<span class="sourceLineNo">449</span>        }<a name="line.449"></a>
-<span class="sourceLineNo">450</span>      });<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    } finally {<a name="line.451"></a>
-<span class="sourceLineNo">452</span>      try {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>        // try to cleanup inactive wals and complete the operation<a name="line.453"></a>
-<span class="sourceLineNo">454</span>        buildHoldingCleanupTracker();<a name="line.454"></a>
-<span class="sourceLineNo">455</span>        tryCleanupLogsOnLoad();<a name="line.455"></a>
-<span class="sourceLineNo">456</span>        loading.set(false);<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      } finally {<a name="line.457"></a>
-<span class="sourceLineNo">458</span>        lock.unlock();<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      }<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>  }<a name="line.461"></a>
-<span class="sourceLineNo">462</span><a name="line.462"></a>
-<span class="sourceLineNo">463</span>  private void tryCleanupLogsOnLoad() {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>    // nothing to cleanup.<a name="line.464"></a>
-<span class="sourceLineNo">465</span>    if (logs.size() &lt;= 1) return;<a name="line.465"></a>
-<span class="sourceLineNo">466</span><a name="line.466"></a>
-<span class="sourceLineNo">467</span>    // the config says to not cleanup wals on load.<a name="line.467"></a>
-<span class="sourceLineNo">468</span>    if (!conf.getBoolean(EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>      DEFAULT_EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY)) {<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      LOG.debug("WALs cleanup on load is not enabled: " + getActiveLogs());<a name="line.470"></a>
-<span class="sourceLineNo">471</span>      return;<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    }<a name="line.472"></a>
+<span class="sourceLineNo">427</span>        long count = 0;<a name="line.427"></a>
+<span class="sourceLineNo">428</span><a name="line.428"></a>
+<span class="sourceLineNo">429</span>        @Override<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        public void setMaxProcId(long maxProcId) {<a name="line.430"></a>
+<span class="sourceLineNo">431</span>          loader.setMaxProcId(maxProcId);<a name="line.431"></a>
+<span class="sourceLineNo">432</span>        }<a name="line.432"></a>
+<span class="sourceLineNo">433</span><a name="line.433"></a>
+<span class="sourceLineNo">434</span>        @Override<a name="line.434"></a>
+<span class="sourceLineNo">435</span>        public void load(ProcedureIterator procIter) throws IOException {<a name="line.435"></a>
+<span class="sourceLineNo">436</span>          if ((++count % 1000) == 0) {<a name="line.436"></a>
+<span class="sourceLineNo">437</span>            // Log every 1000 procedures otherwise it looks like Master is dead if loads of WALs<a name="line.437"></a>
+<span class="sourceLineNo">438</span>            // and procedures to load.<a name="line.438"></a>
+<span class="sourceLineNo">439</span>            LOG.debug("Loaded {} procedures", this.count);<a name="line.439"></a>
+<span class="sourceLineNo">440</span>          }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>          loader.load(procIter);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>        }<a name="line.442"></a>
+<span class="sourceLineNo">443</span><a name="line.443"></a>
+<span class="sourceLineNo">444</span>        @Override<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.445"></a>
+<span class="sourceLineNo">446</span>          loader.handleCorrupted(procIter);<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        }<a name="line.447"></a>
+<span class="sourceLineNo">448</span><a name="line.448"></a>
+<span class="sourceLineNo">449</span>        @Override<a name="line.449"></a>
+<span class="sourceLineNo">450</span>        public void markCorruptedWAL(ProcedureWALFile log, IOException e) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>          if (corruptedLogs == null) {<a name="line.451"></a>
+<span class="sourceLineNo">452</span>            corruptedLogs = new HashSet&lt;&gt;();<a name="line.452"></a>
+<span class="sourceLineNo">453</span>          }<a name="line.453"></a>
+<span class="sourceLineNo">454</span>          corruptedLogs.add(log);<a name="line.454"></a>
+<span class="sourceLineNo">455</span>          // TODO: sideline corrupted log<a name="line.455"></a>
+<span class="sourceLineNo">456</span>        }<a name="line.456"></a>
+<span class="sourceLineNo">457</span>      });<a name="line.457"></a>
+<span class="sourceLineNo">458</span>    } finally {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>      try {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>        // try to cleanup inactive wals and complete the operation<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        buildHoldingCleanupTracker();<a name="line.461"></a>
+<span class="sourceLineNo">462</span>        tryCleanupLogsOnLoad();<a name="line.462"></a>
+<span class="sourceLineNo">463</span>        loading.set(false);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>      } finally {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>        lock.unlock();<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      }<a name="line.466"></a>
+<span class="sourceLineNo">467</span>    }<a name="line.467"></a>
+<span class="sourceLineNo">468</span>  }<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>  private void tryCleanupLogsOnLoad() {<a name="line.470"></a>
+<span class="sourceLineNo">471</span>    // nothing to cleanup.<a name="line.471"></a>
+<span class="sourceLineNo">472</span>    if (logs.size() &lt;= 1) return;<a name="line.472"></a>
 <span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>    try {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>      periodicRoll();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    } catch (IOException e) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      LOG.warn("Unable to cleanup logs on load: " + e.getMessage(), e);<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    }<a name="line.478"></a>
-<span class="sourceLineNo">479</span>  }<a name="line.479"></a>
+<span class="sourceLineNo">474</span>    // the config says to not cleanup wals on load.<a name="line.474"></a>
+<span class="sourceLineNo">475</span>    if (!conf.getBoolean(EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY,<a name="line.475"></a>
+<span class="sourceLineNo">476</span>      DEFAULT_EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY)) {<a name="line.476"></a>
+<span class="sourceLineNo">477</span>      LOG.debug("WALs cleanup on load is not enabled: " + getActiveLogs());<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      return;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
 <span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span>  @Override<a name="line.481"></a>
-<span class="sourceLineNo">482</span>  public void insert(final Procedure proc, final Procedure[] subprocs) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    if (LOG.isTraceEnabled()) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span>      LOG.trace("Insert " + proc + ", subproc=" + Arrays.toString(subprocs));<a name="line.484"></a>
+<span class="sourceLineNo">481</span>    try {<a name="line.481"></a>
+<span class="sourceLineNo">482</span>      periodicRoll();<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    } catch (IOException e) {<a name="line.483"></a>
+<span class="sourceLineNo">484</span>      LOG.warn("Unable to cleanup logs on load: " + e.getMessage(), e);<a name="line.484"></a>
 <span class="sourceLineNo">485</span>    }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>    ByteSlot slot = acquireSlot();<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    try {<a name="line.488"></a>
-<span class="sourceLineNo">489</span>      // Serialize the insert<a name="line.489"></a>
-<span class="sourceLineNo">490</span>      long[] subProcIds = null;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>      if (subprocs != null) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span>        ProcedureWALFormat.writeInsert(slot, proc, subprocs);<a name="line.492"></a>
-<span class="sourceLineNo">493</span>        subProcIds = new long[subprocs.length];<a name="line.493"></a>
-<span class="sourceLineNo">494</span>        for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>          subProcIds[i] = subprocs[i].getProcId();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>        }<a name="line.496"></a>
-<span class="sourceLineNo">497</span>      } else {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>        assert !proc.hasParent();<a name="line.498"></a>
-<span class="sourceLineNo">499</span>        ProcedureWALFormat.writeInsert(slot, proc);<a name="line.499"></a>
-<span class="sourceLineNo">500</span>      }<a name="line.500"></a>
-<span class="sourceLineNo">501</span><a name="line.501"></a>
-<span class="sourceLineNo">502</span>      // Push the transaction data and wait until it is persisted<a name="line.502"></a>
-<span class="sourceLineNo">503</span>      pushData(PushType.INSERT, slot, proc.getProcId(), subProcIds);<a name="line.503"></a>
-<span class="sourceLineNo">504</span>    } catch (IOException e) {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      // We are not able to serialize the procedure.<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      // this is a code error, and we are not able to go on.<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: proc=" +<a name="line.507"></a>
-<span class="sourceLineNo">508</span>          proc + ", subprocs=" + Arrays.toString(subprocs), e);<a name="line.508"></a>
-<span class="sourceLineNo">509</span>      throw new RuntimeException(e);<a name="line.509"></a>
-<span class="sourceLineNo">510</span>    } finally {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      releaseSlot(slot);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>    }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>  }<a name="line.513"></a>
-<span class="sourceLineNo">514</span><a name="line.514"></a>
-<span class="sourceLineNo">515</span>  @Override<a name="line.515"></a>
-<span class="sourceLineNo">516</span>  public void insert(final Procedure[] procs) {<a name="line.516"></a>
-<span class="sourceLineNo">517</span>    if (LOG.isTraceEnabled()) {<a name="line.517"></a>
-<span class="sourceLineNo">518</span>      LOG.trace("Insert " + Arrays.toString(procs));<a name="line.518"></a>
+<span class="sourceLineNo">486</span>  }<a name="line.486"></a>
+<span class="sourceLineNo">487</span><a name="line.487"></a>
+<span class="sourceLineNo">488</span>  @Override<a name="line.488"></a>
+<span class="sourceLineNo">489</span>  public void insert(final Procedure proc, final Procedure[] subprocs) {<a name="line.489"></a>
+<span class="sourceLineNo">490</span>    if (LOG.isTraceEnabled()) {<a name="line.490"></a>
+<span class="sourceLineNo">491</span>      LOG.trace("Insert " + proc + ", subproc=" + Arrays.toString(subprocs));<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    }<a name="line.492"></a>
+<span class="sourceLineNo">493</span><a name="line.493"></a>
+<span class="sourceLineNo">494</span>    ByteSlot slot = acquireSlot();<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    try {<a name="line.495"></a>
+<span class="sourceLineNo">496</span>      // Serialize the insert<a name="line.496"></a>
+<span class="sourceLineNo">497</span>      long[] subProcIds = null;<a name="line.497"></a>
+<span class="sourceLineNo">498</span>      if (subprocs != null) {<a name="line.498"></a>
+<span class="sourceLineNo">499</span>        ProcedureWALFormat.writeInsert(slot, proc, subprocs);<a name="line.499"></a>
+<span class="sourceLineNo">500</span>        subProcIds = new long[subprocs.length];<a name="line.500"></a>
+<span class="sourceLineNo">501</span>        for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.501"></a>
+<span class="sourceLineNo">502</span>          subProcIds[i] = subprocs[i].getProcId();<a name="line.502"></a>
+<span class="sourceLineNo">503</span>        }<a name="line.503"></a>
+<span class="sourceLineNo">504</span>      } else {<a name="line.504"></a>
+<span class="sourceLineNo">505</span>        assert !proc.hasParent();<a name="line.505"></a>
+<span class="sourceLineNo">506</span>        ProcedureWALFormat.writeInsert(slot, proc);<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      }<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      // Push the transaction data and wait until it is persisted<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      pushData(PushType.INSERT, slot, proc.getProcId(), subProcIds);<a name="line.510"></a>
+<span class="sourceLineNo">511</span>    } catch (IOException e) {<a name="line.511"></a>
+<span class="sourceLineNo">512</span>      // We are not able to serialize the procedure.<a name="line.512"></a>
+<span class="sourceLineNo">513</span>      // this is a code error, and we are not able to go on.<a name="line.513"></a>
+<span class="sourceLineNo">514</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: proc=" +<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          proc + ", subprocs=" + Arrays.toString(subprocs), e);<a name="line.515"></a>
+<span class="sourceLineNo">516</span>      throw new RuntimeException(e);<a name="line.516"></a>
+<span class="sourceLineNo">517</span>    } finally {<a name="line.517"></a>
+<span class="sourceLineNo">518</span>      releaseSlot(slot);<a name="line.518"></a>
 <span class="sourceLineNo">519</span>    }<a name="line.519"></a>
-<span class="sourceLineNo">520</span><a name="line.520"></a>
-<span class="sourceLineNo">521</span>    ByteSlot slot = acquireSlot();<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    try {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      // Serialize the insert<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      long[] procIds = new long[procs.length];<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      for (int i = 0; i &lt; procs.length; ++i) {<a name="line.525"></a>
-<span class="sourceLineNo">526</span>        assert !procs[i].hasParent();<a name="line.526"></a>
-<span class="sourceLineNo">527</span>        procIds[i] = procs[i].getProcId();<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        ProcedureWALFormat.writeInsert(slot, procs[i]);<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      }<a name="line.529"></a>
-<span class="sourceLineNo">530</span><a name="line.530"></a>
-<span class="sourceLineNo">531</span>      // Push the transaction data and wait until it is persisted<a name="line.531"></a>
-<span class="sourceLineNo">532</span>      pushData(PushType.INSERT, slot, Procedure.NO_PROC_ID, procIds);<a name="line.532"></a>
-<span class="sourceLineNo">533</span>    } catch (IOException e) {<a name="line.533"></a>
-<span class="sourceLineNo">534</span>      // We are not able to serialize the procedure.<a name="line.534"></a>
-<span class="sourceLineNo">535</span>      // this is a code error, and we are not able to go on.<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: " +<a name="line.536"></a>
-<span class="sourceLineNo">537</span>          Arrays.toString(procs), e);<a name="line.537"></a>
-<span class="sourceLineNo">538</span>      throw new RuntimeException(e);<a name="line.538"></a>
-<span class="sourceLineNo">539</span>    } finally {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      releaseSlot(slot);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>  public void update(final Procedure proc) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>    if (LOG.isTraceEnabled()) {<a name="line.546"></a>
-<span class="sourceLineNo">547</span>      LOG.trace("Update " + proc);<a name="line.547"></a>
+<span class="sourceLineNo">520</span>  }<a name="line.520"></a>
+<span class="sourceLineNo">521</span><a name="line.521"></a>
+<span class="sourceLineNo">522</span>  @Override<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  public void insert(final Procedure[] procs) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    if (LOG.isTraceEnabled()) {<a name="line.524"></a>
+<span class="sourceLineNo">525</span>      LOG.trace("Insert " + Arrays.toString(procs));<a name="line.525"></a>
+<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span>    ByteSlot slot = acquireSlot();<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    try {<a name="line.529"></a>
+<span class="sourceLineNo">530</span>      // Serialize the insert<a name="line.530"></a>
+<span class="sourceLineNo">531</span>      long[] procIds = new long[procs.length];<a name="line.531"></a>
+<span class="sourceLineNo">532</span>      for (int i = 0; i &lt; procs.length; ++i) {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>        assert !procs[i].hasParent();<a name="line.533"></a>
+<span class="sourceLineNo">534</span>        procIds[i] = procs[i].getProcId();<a name="line.534"></a>
+<span class="sourceLineNo">535</span>        ProcedureWALFormat.writeInsert(slot, procs[i]);<a name="line.535"></a>
+<span class="sourceLineNo">536</span>      }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>      // Push the transaction data and wait until it is persisted<a name="line.538"></a>
+<span class="sourceLineNo">539</span>      pushData(PushType.INSERT, slot, Procedure.NO_PROC_ID, procIds);<a name="line.539"></a>
+<span class="sourceLineNo">540</span>    } catch (IOException e) {<a name="line.540"></a>
+<span class="sourceLineNo">541</span>      // We are not able to serialize the procedure.<a name="line.541"></a>
+<span class="sourceLineNo">542</span>      // this is a code error, and we are not able to go on.<a name="line.542"></a>
+<span class="sourceLineNo">543</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: " +<a name="line.543"></a>
+<span class="sourceLineNo">544</span>          Arrays.toString(procs), e);<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      throw new RuntimeException(e);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    } finally {<a name="line.546"></a>
+<span class="sourceLineNo">547</span>      releaseSlot(slot);<a name="line.547"></a>
 <span class="sourceLineNo">548</span>    }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>    ByteSlot slot = acquireSlot();<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    try {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      // Serialize the update<a name="line.552"></a>
-<span class="sourceLineNo">553</span>      ProcedureWALFormat.writeUpdate(slot, proc);<a name="line.553"></a>
-<span class="sourceLineNo">554</span><a name="line.554"></a>
-<span class="sourceLineNo">555</span>      // Push the transaction data and wait until it is persisted<a name="line.555"></a>
-<span class="sourceLineNo">556</span>      pushData(PushType.UPDATE, slot, proc.getProcId(), null);<a name="line.556"></a>
-<span class="sourceLineNo">557</span>    } catch (IOException e) {<a name="line.557"></a>
-<span class="sourceLineNo">558</span>      // We are not able to serialize the procedure.<a name="line.558"></a>
-<span class="sourceLineNo">559</span>      // this is a code error, and we are not able to go on.<a name="line.559"></a>
-<span class="sourceLineNo">560</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.560"></a>
-<span class="sourceLineNo">561</span>      throw new RuntimeException(e);<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    } finally {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      releaseSlot(slot);<a name="line.563"></a>
-<span class="sourceLineNo">564</span>    }<a name="line.564"></a>
-<span class="sourceLineNo">565</span>  }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>  @Override<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  public void delete(final long procId) {<a name="line.568"></a>
-<span class="sourceLineNo">569</span>    if (LOG.isTraceEnabled()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>      LOG.trace("Delete " + procId);<a name="line.570"></a>
+<span class="sourceLineNo">549</span>  }<a name="line.549"></a>
+<span class="sourceLineNo">550</span><a name="line.550"></a>
+<span class="sourceLineNo">551</span>  @Override<a name="line.551"></a>
+<span class="sourceLineNo">552</span>  public void update(final Procedure proc) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    if (LOG.isTraceEnabled()) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      LOG.trace("Update " + proc);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span><a name="line.556"></a>
+<span class="sourceLineNo">557</span>    ByteSlot slot = acquireSlot();<a name="line.557"></a>
+<span class="sourceLineNo">558</span>    try {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>      // Serialize the update<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      ProcedureWALFormat.writeUpdate(slot, proc);<a name="line.560"></a>
+<span class="sourceLineNo">561</span><a name="line.561"></a>
+<span class="sourceLineNo">562</span>      // Push the transaction data and wait until it is persisted<a name="line.562"></a>
+<span class="sourceLineNo">563</span>      pushData(PushType.UPDATE, slot, proc.getProcId(), null);<a name="line.563"></a>
+<span class="sourceLineNo">564</span>    } catch (IOException e) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span>      // We are not able to serialize the procedure.<a name="line.565"></a>
+<span class="sourceLineNo">566</span>      // this is a code error, and we are not able to go on.<a name="line.566"></a>
+<span class="sourceLineNo">567</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.567"></a>
+<span class="sourceLineNo">568</span>      throw new RuntimeException(e);<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    } finally {<a name="line.569"></a>
+<span class="sourceLineNo">570</span>      releaseSlot(slot);<a name="line.570"></a>
 <span class="sourceLineNo">571</span>    }<a name="line.571"></a>
-<span class="sourceLineNo">572</span><a name="line.572"></a>
-<span class="sourceLineNo">573</span>    ByteSlot slot = acquireSlot();<a name="line.573"></a>
-<span class="sourceLineNo">574</span>    try {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>      // Serialize the delete<a name="line.575"></a>
-<span class="sourceLineNo">576</span>      ProcedureWALFormat.writeDelete(slot, procId);<a name="line.576"></a>
-<span class="sourceLineNo">577</span><a name="line.577"></a>
-<span class="sourceLineNo">578</span>      // Push the transaction data and wait until it is persisted<a name="line.578"></a>
-<span class="sourceLineNo">579</span>      pushData(PushType.DELETE, slot, procId, null);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    } catch (IOException e) {<a name="line.580"></a>
-<span class="sourceLineNo">581</span>      // We are not able to serialize the procedure.<a name="line.581"></a>
-<span class="sourceLineNo">582</span>      // this is a code error, and we are not able to go on.<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + procId, e);<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      throw new RuntimeException(e);<a name="line.584"></a>
-<span class="sourceLineNo">585</span>    } finally {<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      releaseSlot(slot);<a name="line.586"></a>
-<span class="sourceLineNo">587</span>    }<a name="line.587"></a>
-<span class="sourceLineNo">588</span>  }<a name="line.588"></a>
-<span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>  @Override<a name="line.590"></a>
-<span class="sourceLineNo">591</span>  public void delete(final Procedure proc, final long[] subProcIds) {<a name="line.591"></a>
-<span class="sourceLineNo">592</span>    assert proc != null : "expected a non-null procedure";<a name="line.592"></a>
-<span class="sourceLineNo">593</span>    assert subProcIds != null &amp;&amp; subProcIds.length &gt; 0 : "expected subProcIds";<a name="line.593"></a>
-<span class="sourceLineNo">594</span>    if (LOG.isTraceEnabled()) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>      LOG.trace("Update " + proc + " and Delete " + Arrays.toString(subProcIds));<a name="line.595"></a>
-<span class="sourceLineNo">596</span>    }<a name="line.596"></a>
-<span class="sourceLineNo">597</span><a name="line.597"></a>
-<span class="sourceLineNo">598</span>    ByteSlot slot = acquireSlot();<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    try {<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // Serialize the delete<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      ProcedureWALFormat.writeDelete(slot, proc, subProcIds);<a name="line.601"></a>
-<span class="sourceLineNo">602</span><a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // Push the transaction data and wait until it is persisted<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      pushData(PushType.DELETE, slot, proc.getProcId(), subProcIds);<a name="line.604"></a>
-<span class="sourceLineNo">605</span>    } catch (IOException e) {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      // We are not able to serialize the procedure.<a name="line.606"></a>
-<span class="sourceLineNo">607</span>      // this is a code error, and we are not able to go on.<a name="line.607"></a>
-<span class="sourceLineNo">608</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.608"></a>
-<span class="sourceLineNo">609</span>      throw new RuntimeException(e);<a name="line.609"></a>
-<span class="sourceLineNo">610</span>    } finally {<a name="line.610"></a>
-<span class="sourceLineNo">611</span>      releaseSlot(slot);<a name="line.611"></a>
-<span class="sourceLineNo">612</span>    }<a name="line.612"></a>
-<span class="sourceLineNo">613</span>  }<a name="line.613"></a>
-<span class="sourceLineNo">614</span><a name="line.614"></a>
-<span class="sourceLineNo">615</span>  @Override<a name="line.615"></a>
-<span class="sourceLineNo">616</span>  public void delete(final long[] procIds, final int offset, final int count) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>    if (count == 0) return;<a name="line.617"></a>
-<span class="sourceLineNo">618</span>    if (offset == 0 &amp;&amp; count == procIds.length) {<a name="line.618"></a>
-<span class="sourceLineNo">619</span>      delete(procIds);<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    } else if (count == 1) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      delete(procIds[offset]);<a name="line.621"></a>
-<span class="sourceLineNo">622</span>    } else {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>      delete(Arrays.copyOfRange(procIds, offset, offset + count));<a name="line.623"></a>
-<span class="sourceLineNo">624</span>    }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>  }<a name="line.625"></a>
-<span class="sourceLineNo">626</span><a name="line.626"></a>
-<span class="sourceLineNo">627</span>  private void delete(final long[] procIds) {<a name="line.627"></a>
-<span class="sourceLineNo">628</span>    if (LOG.isTraceEnabled()) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>      LOG.trace("Delete " + Arrays.toString(procIds));<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    }<a name="line.630"></a>
-<span class="sourceLineNo">631</span><a name="line.631"></a>
-<span class="sourceLineNo">632</span>    final ByteSlot slot = acquireSlot();<a name="line.632"></a>
-<span class="sourceLineNo">633</span>    try {<a name="line.633"></a>
-<span class="sourceLineNo">634</span>      // Serialize the delete<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.635"></a>
-<span class="sourceLineNo">636</span>        ProcedureWALFormat.writeDelete(slot, procIds[i]);<a name="line.636"></a>
-<span class="sourceLineNo">637</span>      }<a name="line.637"></a>
+<span class="sourceLineNo">572</span>  }<a name="line.572"></a>
+<span class="sourceLineNo">573</span><a name="line.573"></a>
+<span class="sourceLineNo">574</span>  @Override<a name="line.574"></a>
+<span class="sourceLineNo">575</span>  public void delete(final long procId) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span>    if (LOG.isTraceEnabled()) {<a name="line.576"></a>
+<span class="sourceLineNo">577</span>      LOG.trace("Delete " + procId);<a name="line.577"></a>
+<span class="sourceLineNo">578</span>    }<a name="line.578"></a>
+<span class="sourceLineNo">579</span><a name="line.579"></a>
+<span class="sourceLineNo">580</span>    ByteSlot slot = acquireSlot();<a name="line.580"></a>
+<span class="sourceLineNo">581</span>    try {<a name="line.581"></a>
+<span class="sourceLineNo">582</span>      // Serialize the delete<a name="line.582"></a>
+<span class="sourceLineNo">583</span>      ProcedureWALFormat.writeDelete(slot, procId);<a name="line.583"></a>
+<span class="sourceLineNo">584</span><a name="line.584"></a>
+<span class="sourceLineNo">585</span>      // Push the transaction data and wait until it is persisted<a name="line.585"></a>
+<span class="sourceLineNo">586</span>      pushData(PushType.DELETE, slot, procId, null);<a name="line.586"></a>
+<span class="sourceLineNo">587</span>    } catch (IOException e) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>      // We are not able to serialize the procedure.<a name="line.588"></a>
+<span class="sourceLineNo">589</span>      // this is a code error, and we are not able to go on.<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + procId, e);<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      throw new RuntimeException(e);<a name="line.591"></a>
+<span class="sourceLineNo">592</span>    } finally {<a name="line.592"></a>
+<span class="sourceLineNo">593</span>      releaseSlot(slot);<a name="line.593"></a>
+<span class="sourceLineNo">594</span>    }<a name="line.594"></a>
+<span class="sourceLineNo">595</span>  }<a name="line.595"></a>
+<span class="sourceLineNo">596</span><a name="line.596"></a>
+<span class="sourceLineNo">597</span>  @Override<a name="line.597"></a>
+<span class="sourceLineNo">598</span>  public void delete(final Procedure proc, final long[] subProcIds) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>    assert proc != null : "expected a non-null procedure";<a name="line.599"></a>
+<span class="sourceLineNo">600</span>    assert subProcIds != null &amp;&amp; subProcIds.length &gt; 0 : "expected subProcIds";<a name="line.600"></a>
+<span class="sourceLineNo">601</span>    if (LOG.isTraceEnabled()) {<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      LOG.trace("Update " + proc + " and Delete " + Arrays.toString(subProcIds));<a name="line.602"></a>
+<span class="sourceLineNo">603</span>    }<a name="line.603"></a>
+<span class="sourceLineNo">604</span><a name="line.604"></a>
+<span class="sourceLineNo">605</span>    ByteSlot slot = acquireSlot();<a name="line.605"></a>
+<span class="sourceLineNo">606</span>    try {<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // Serialize the delete<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      ProcedureWALFormat.writeDelete(slot, proc, subProcIds);<a name="line.608"></a>
+<span class="sourceLineNo">609</span><a name="line.609"></a>
+<span class="sourceLineNo">610</span>      // Push the transaction data and wait until it is persisted<a name="line.610"></a>
+<span class="sourceLineNo">611</span>      pushData(PushType.DELETE, slot, proc.getProcId(), subProcIds);<a name="line.611"></a>
+<span class="sourceLineNo">612</span>    } catch (IOException e) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>      // We are not able to serialize the procedure.<a name="line.613"></a>
+<span class="sourceLineNo">614</span>      // this is a code error, and we are not able to go on.<a name="line.614"></a>
+<span class="sourceLineNo">615</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.615"></a>
+<span class="sourceLineNo">616</span>      throw new RuntimeException(e);<a name="line.616"></a>
+<span class="sourceLineNo">617</span>    } finally {<a name="line.617"></a>
+<span class="sourceLineNo">618</span>      releaseSlot(slot);<a name="line.618"></a>
+<span class="sourceLineNo">619</span>    }<a name="line.619"></a>
+<span class="sourceLineNo">620</span>  }<a name="line.620"></a>
+<span class="sourceLineNo">621</span><a name="line.621"></a>
+<span class="sourceLineNo">622</span>  @Override<a name="line.622"></a>
+<span class="sourceLineNo">623</span>  public void delete(final long[] procIds, final int offset, final int count) {<a name="line.623"></a>
+<span class="sourceLineNo">624</span>    if (count == 0) return;<a name="line.624"></a>
+<span class="sourceLineNo">625</span>    if (offset == 0 &amp;&amp; count == procIds.length) {<a name="line.625"></a>
+<span class="sourceLineNo">626</span>      delete(procIds);<a name="line.626"></a>
+<span class="sourceLineNo">627</span>    } else if (count == 1) {<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      delete(procIds[offset]);<a name="line.628"></a>
+<span class="sourceLineNo">629</span>    } else {<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      delete(Arrays.copyOfRange(procIds, offset, offset + count));<a name="line.630"></a>
+<span class="sourceLineNo">631</span>    }<a name="line.631"></a>
+<span class="sourceLineNo">632</span>  }<a name="line.632"></a>
+<span class="sourceLineNo">633</span><a name="line.633"></a>
+<span class="sourceLineNo">634</span>  private void delete(final long[] procIds) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>    if (LOG.isTraceEnabled()) {<a name="line.635"></a>
+<span class="sourceLineNo">636</span>      LOG.trace("Delete " + Arrays.toString(procIds));<a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
 <span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>      // Push the transaction data and wait until it is persisted<a name="line.639"></a>
-<span class="sourceLineNo">640</span>      pushData(PushType.DELETE, slot, Procedure.NO_PROC_ID, procIds);<a name="line.640"></a>
-<span class="sourceLineNo">641</span>    } catch (IOException e) {<a name="line.641"></a>
-<span class="sourceLineNo">642</span>      // We are not able to serialize the procedure.<a name="line.642"></a>
-<span class="sourceLineNo">643</span>      // this is a code error, and we are not able to go on.<a name="line.643"></a>
-<span class="sourceLineNo">644</span>      LOG.error("Unable to serialize the procedures: " + Arrays.toString(procIds), e);<a name="line.644"></a>
-<span class="sourceLineNo">645</span>      throw new RuntimeException(e);<a name="line.645"></a>
-<span class="sourceLineNo">646</span>    } finally {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      releaseSlot(slot);<a name="line.647"></a>
-<span class="sourceLineNo">648</span>    }<a name="line.648"></a>
-<span class="sourceLineNo">649</span>  }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span>  private ByteSlot acquireSlot() {<a name="line.651"></a>
-<span class="sourceLineNo">652</span>    ByteSlot slot = slotsCache.poll();<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    return slot != null ? slot : new ByteSlot();<a name="line.653"></a>
-<span class="sourceLineNo">654</span>  }<a name="line.654"></a>
-<span class="sourceLineNo">655</span><a name="line.655"></a>
-<span class="sourceLineNo">656</span>  private void releaseSlot(final ByteSlot slot) {<a name="line.656"></a>
-<span class="sourceLineNo">657</span>    slot.reset();<a name="line.657"></a>
-<span class="sourceLineNo">658</span>    slotsCache.offer(slot);<a name="line.658"></a>
-<span class="sourceLineNo">659</span>  }<a name="line.659"></a>
-<span class="sourceLineNo">660</span><a name="line.660"></a>
-<span class="sourceLineNo">661</span>  private enum PushType { INSERT, UPDATE, DELETE }<a name="line.661"></a>
+<span class="sourceLineNo">639</span>    final ByteSlot slot = acquireSlot();<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    try {<a name="line.640"></a>
+<span class="sourceLineNo">641</span>      // Serialize the delete<a name="line.641"></a>
+<span class="sourceLineNo">642</span>      for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.642"></a>
+<span class="sourceLineNo">643</span>        ProcedureWALFormat.writeDelete(slot, procIds[i]);<a name="line.643"></a>
+<span class="sourceLineNo">644</span>      }<a name="line.644"></a>
+<span class="sourceLineNo">645</span><a name="line.645"></a>
+<span class="sourceLineNo">646</span>      // Push the transaction data and wait until it is persisted<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      pushData(PushType.DELETE, slot, Procedure.NO_PROC_ID, procIds);<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    } catch (IOException e) {<a name="line.648"></a>
+<span class="sourceLineNo">649</span>      // We are not able to serialize the procedure.<a name="line.649"></a>
+<span class="sourceLineNo">650</span>      // this is a code error, and we are not able to go on.<a name="line.650"></a>
+<span class="sourceLineNo">651</span>      LOG.error("Unable to serialize the procedures: " + Arrays.toString(procIds), e);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>      throw new RuntimeException(e);<a name="line.652"></a>
+<span class="sourceLineNo">653</span>    } finally {<a name="line.653"></a>
+<span class="sourceLineNo">654</span>      releaseSlot(slot);<a name="line.654"></a>
+<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
+<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
+<span class="sourceLineNo">657</span><a name="line.657"></a>
+<span class="sourceLineNo">658</span>  private ByteSlot acquireSlot() {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    ByteSlot slot = slotsCache.poll();<a name="line.659"></a>
+<span class="sourceLineNo">660</span>    return slot != null ? slot : new ByteSlot();<a name="line.660"></a>
+<span class="sourceLineNo">661</span>  }<a name="line.661"></a>
 <span class="sourceLineNo">662</span><a name="line.662"></a>
-<span class="sourceLineNo">663</span>  private long pushData(final PushType type, final ByteSlot slot,<a name="line.663"></a>
-<span class="sourceLineNo">664</span>      final long procId, final long[] subProcIds) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>    if (!isRunning()) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span>      throw new RuntimeException("the store must be running before inserting data");<a name="line.666"></a>
-<span class="sourceLineNo">667</span>    }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    if (logs.isEmpty()) {<a name="line.668"></a>
-<span class="sourceLineNo">669</span>      throw new RuntimeException("recoverLease() must be called before inserting data");<a name="line.669"></a>
-<span class="sourceLineNo">670</span>    }<a name="line.670"></a>
-<span class="sourceLineNo">671</span><a name="line.671"></a>
-<span class="sourceLineNo">672</span>    long logId = -1;<a name="line.672"></a>
-<span class="sourceLineNo">673</span>    lock.lock();<a name="line.673"></a>
-<span class="sourceLineNo">674</span>    try {<a name="line.674"></a>
-<span class="sourceLineNo">675</span>      // Wait for the sync to be completed<a name="line.675"></a>
-<span class="sourceLineNo">676</span>      while (true) {<a name="line.676"></a>
-<span class="sourceLineNo">677</span>        if (!isRunning()) {<a name="line.677"></a>
-<span class="sourceLineNo">678</span>          throw new RuntimeException("store no longer running");<a name="line.678"></a>
-<span class="sourceLineNo">679</span>        } else if (isSyncAborted()) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span>          throw new RuntimeException("sync aborted", syncException.get());<a name="line.680"></a>
-<span class="sourceLineNo">681</span>        } else if (inSync.get()) {<a name="line.681"></a>
-<span class="sourceLineNo">682</span>          syncCond.await();<a name="line.682"></a>
-<span class="sourceLineNo">683</span>        } else if (slotIndex &gt;= syncMaxSlot) {<a name="line.683"></a>
-<span class="sourceLineNo">684</span>          slotCond.signal();<a name="line.684"></a>
-<span class="sourceLineNo">685</span>          syncCond.await();<a name="line.685"></a>
-<span class="sourceLineNo">686</span>        } else {<a name="line.686"></a>
-<span class="sourceLineNo">687</span>          break;<a name="line.687"></a>
-<span class="sourceLineNo">688</span>        }<a name="line.688"></a>
-<span class="sourceLineNo">689</span>      }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>      final long pushSyncId = syncId.get();<a name="line.691"></a>
-<span class="sourceLineNo">692</span>      updateStoreTracker(type, procId, subProcIds);<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      slots[slotIndex++] = slot;<a name="line.693"></a>
-<span class="sourceLineNo">694</span>      logId = flushLogId;<a name="line.694"></a>
-<span class="sourceLineNo">695</span><a name="line.695"></a>
-<span class="sourceLineNo">696</span>      // Notify that there is new data<a name="line.696"></a>
-<span class="sourceLineNo">697</span>      if (slotIndex == 1) {<a name="line.697"></a>
-<span class="sourceLineNo">698</span>        waitCond.signal();<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      }<a name="line.699"></a>
-<span class="sourceLineNo">700</span><a name="line.700"></a>
-<span class="sourceLineNo">701</span>      // Notify that the slots are full<a name="line.701"></a>
-<span class="sourceLineNo">702</span>      if (slotIndex == syncMaxSlot) {<a name="line.702"></a>
-<span class="sourceLineNo">703</span>        waitCond.signal();<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        slotCond.signal();<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span><a name="line.706"></a>
-<span class="sourceLineNo">707</span>      while (pushSyncId == syncId.get() &amp;&amp; isRunning()) {<a name="line.707"></a>
-<span class="sourceLineNo">708</span>        syncCond.await();<a name="line.708"></a>
-<span class="sourceLineNo">709</span>      }<a name="line.709"></a>
-<span class="sourceLineNo">710</span>    } catch (InterruptedException e) {<a name="line.710"></a>
-<span class="sourceLineNo">711</span>      Thread.currentThread().interrupt();<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      sendAbortProcessSignal();<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      throw new RuntimeException(e);<a name="line.713"></a>
-<span class="sourceLineNo">714</span>    } finally {<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      lock.unlock();<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      if (isSyncAborted()) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span>        throw new RuntimeException("sync aborted", syncException.get());<a name="line.717"></a>
-<span class="sourceLineNo">718</span>      }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>    }<a name="line.719"></a>
-<span class="sourceLineNo">720</span>    return logId;<a name="line.720"></a>
-<span class="sourceLineNo">721</span>  }<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>  private void updateStoreTracker(final PushType type,<a name="line.723"></a>
-<span class="sourceLineNo">724</span>      final long procId, final long[] subProcIds) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span>    switch (type) {<a name="line.725"></a>
-<span class="sourceLineNo">726</span>      case INSERT:<a name="line.726"></a>
-<span class="sourceLineNo">727</span>        if (subProcIds == null) {<a name="line.727"></a>
-<span class="sourceLineNo">728</span>          storeTracker.insert(procId);<a name="line.728"></a>
-<span class="sourceLineNo">729</span>        } else if (procId == Procedure.NO_PROC_ID) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span>          storeTracker.insert(subProcIds);<a name="line.730"></a>
-<span class="sourceLineNo">731</span>        } else {<a name="line.731"></a>
-<span class="sourceLineNo">732</span>          storeTracker.insert(procId, subProcIds);<a name="line.732"></a>
-<span class="sourceLineNo">733</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.733"></a>
-<span class="sourceLineNo">734</span>        }<a name="line.734"></a>
-<span class="sourceLineNo">735</span>        break;<a name="line.735"></a>
-<span class="sourceLineNo">736</span>      case UPDATE:<a name="line.736"></a>
-<span class="sourceLineNo">737</span>        storeTracker.update(procId);<a name="line.737"></a>
-<span class="sourceLineNo">738</span>        holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.738"></a>
-<span class="sourceLineNo">739</span>        break;<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      case DELETE:<a name="line.740"></a>
-<span class="sourceLineNo">741</span>        if (subProcIds != null &amp;&amp; subProcIds.length &gt; 0) {<a name="line.741"></a>
-<span class="sourceLineNo">742</span>          storeTracker.delete(subProcIds);<a name="line.742"></a>
-<span class="sourceLineNo">743</span>          holdingCleanupTracker.setDeletedIfSet(subProcIds);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>        } else {<a name="line.744"></a>
-<span class="sourceLineNo">745</span>          storeTracker.delete(procId);<a name="line.745"></a>
-<span class="sourceLineNo">746</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.746"></a>
-<span class="sourceLineNo">747</span>        }<a name="line.747"></a>
-<span class="sourceLineNo">748</span>        break;<a name="line.748"></a>
-<span class="sourceLineNo">749</span>      default:<a name="line.749"></a>
-<span class="sourceLineNo">750</span>        throw new RuntimeException("invalid push type " + type);<a name="line.750"></a>
-<span class="sourceLineNo">751</span>    }<a name="line.751"></a>
-<span class="sourceLineNo">752</span>  }<a name="line.752"></a>
-<span class="sourceLineNo">753</span><a name="line.753"></a>
-<span class="sourceLineNo">754</span>  private boolean isSyncAborted() {<a name="line.754"></a>
-<span class="sourceLineNo">755</span>    return syncException.get() != null;<a name="line.755"></a>
-<span class="sourceLineNo">756</span>  }<a name="line.756"></a>
-<span class="sourceLineNo">757</span><a name="line.757"></a>
-<span class="sourceLineNo">758</span>  private void syncLoop() throws Throwable {<a name="line.758"></a>
-<span class="sourceLineNo">759</span>    long totalSyncedToStore = 0;<a name="line.759"></a>
-<span class="sourceLineNo">760</span>    inSync.set(false);<a name="line.760"></a>
-<span class="sourceLineNo">761</span>    lock.lock();<a name="line.761"></a>
-<span class="sourceLineNo">762</span>    try {<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (isRunning()) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        try {<a name="line.764"></a>
-<span class="sourceLineNo">765</span>          // Wait until new data is available<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          if (slotIndex == 0) {<a name="line.766"></a>
-<span class="sourceLineNo">767</span>            if (!loading.get()) {<a name="line.767"></a>
-<span class="sourceLineNo">768</span>              periodicRoll();<a name="line.768"></a>
-<span class="sourceLineNo">769</span>            }<a name="line.769"></a>
-<span class="sourceLineNo">770</span><a name="line.770"></a>
-<span class="sourceLineNo">771</span>            if (LOG.isTraceEnabled()) {<a name="line.771"></a>
-<span class="sourceLineNo">772</span>              float rollTsSec = getMillisFromLastRoll() / 1000.0f;<a name="line.772"></a>
-<span class="sourceLineNo">773</span>              LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)",<a name="line.773"></a>
-<span class="sourceLineNo">774</span>                        StringUtils.humanSize(totalSynced.get()),<a name="line.774"></a>
-<span class="sourceLineNo">775</span>                        StringUtils.humanSize(totalSynced.get() / rollTsSec)));<a name="line.775"></a>
+<span class="sourceLineNo">663</span>  private void releaseSlot(final ByteSlot slot) {<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    slot.reset();<a name="line.664"></a>
+<span class="sourceLineNo">665</span>    slotsCache.offer(slot);<a name="line.665"></a>
+<span class="sourceLineNo">666</span>  }<a name="line.666"></a>
+<span class="sourceLineNo">667</span><a name="line.667"></a>
+<span class="sourceLineNo">668</span>  private enum PushType { INSERT, UPDATE, DELETE }<a name="line.668"></a>
+<span class="sourceLineNo">669</span><a name="line.669"></a>
+<span class="sourceLineNo">670</span>  private long pushData(final PushType type, final ByteSlot slot,<a name="line.670"></a>
+<span class="sourceLineNo">671</span>      final long procId, final long[] subProcIds) {<a name="line.671"></a>
+<span class="sourceLineNo">672</span>    if (!isRunning()) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>      throw new RuntimeException("the store must be running before inserting data");<a name="line.673"></a>
+<span class="sourceLineNo">674</span>    }<a name="line.674"></a>
+<span class="sourceLineNo">675</span>    if (logs.isEmpty()) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span>      throw new RuntimeException("recoverLease() must be called before inserting data");<a name="line.676"></a>
+<span class="sourceLineNo">677</span>    }<a name="line.677"></a>
+<span class="sourceLineNo">678</span><a name="line.678"></a>
+<span class="sourceLineNo">679</span>    long logId = -1;<a name="line.679"></a>
+<span class="sourceLineNo">680</span>    lock.lock();<a name="line.680"></a>
+<span class="sourceLineNo">681</span>    try {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>      // Wait for the sync to be completed<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      while (true) {<a name="line.683"></a>
+<span class="sourceLineNo">684</span>        if (!isRunning()) {<a name="line.684"></a>
+<span class="sourceLineNo">685</span>          throw new RuntimeException("store no longer running");<a name="line.685"></a>
+<span class="sourceLineNo">686</span>        } else if (isSyncAborted()) {<a name="line.686"></a>
+<span class="sourceLineNo">687</span>          throw new RuntimeException("sync aborted", syncException.get());<a name="line.687"></a>
+<span class="sourceLineNo">688</span>        } else if (inSync.get()) {<a name="line.688"></a>
+<span class="sourceLineNo">689</span>          syncCond.await();<a name="line.689"></a>
+<span class="sourceLineNo">690</span>        } else if (slotIndex &gt;= syncMaxSlot) {<a name="line.690"></a>
+<span class="sourceLineNo">691</span>          slotCond.signal();<a name="line.691"></a>
+<span class="sourceLineNo">692</span>          syncCond.await();<a name="line.692"></a>
+<span class="sourceLineNo">693</span>        } else {<a name="line.693"></a>
+<span class="sourceLineNo">694</span>          break;<a name="line.694"></a>
+<span class="sourceLineNo">695</span>        }<a name="line.695"></a>
+<span class="sourceLineNo">696</span>      }<a name="line.696"></a>
+<span class="sourceLineNo">697</span><a name="line.697"></a>
+<span class="sourceLineNo">698</span>      final long pushSyncId = syncId.get();<a name="line.698"></a>
+<span class="sourceLineNo">699</span>      updateStoreTracker(type, procId, subProcIds);<a name="line.699"></a>
+<span class="sourceLineNo">700</span>      slots[slotIndex++] = slot;<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      logId = flushLogId;<a name="line.701"></a>
+<span class="sourceLineNo">702</span><a name="line.702"></a>
+<span class="sourceLineNo">703</span>      // Notify that there is new data<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (slotIndex == 1) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        waitCond.signal();<a name="line.705"></a>
+<span class="sourceLineNo">706</span>      }<a name="line.706"></a>
+<span class="sourceLineNo">707</span><a name="line.707"></a>
+<span class="sourceLineNo">708</span>      // Notify that the slots are full<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      if (slotIndex == syncMaxSlot) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        waitCond.signal();<a name="line.710"></a>
+<span class="sourceLineNo">711</span>        slotCond.signal();<a name="line.711"></a>
+<span class="sourceLineNo">712</span>      }<a name="line.712"></a>
+<span class="sourceLineNo">713</span><a name="line.713"></a>
+<span class="sourceLineNo">714</span>      while (pushSyncId == syncId.get() &amp;&amp; isRunning()) {<a name="line.714"></a>
+<span class="sourceLineNo">715</span>        syncCond.await();<a name="line.715"></a>
+<span class="sourceLineNo">716</span>      }<a name="line.716"></a>
+<span class="sourceLineNo">717</span>    } catch (InterruptedException e) {<a name="line.717"></a>
+<span class="sourceLineNo">718</span>      Thread.currentThread().interrupt();<a name="line.718"></a>
+<span class="sourceLineNo">719</span>      sendAbortProcessSignal();<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      throw new RuntimeException(e);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>    } finally {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>      lock.unlock();<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      if (isSyncAborted()) {<a name="line.723"></a>
+<span class="sourceLineNo">724</span>        throw new RuntimeException("sync aborted", syncException.get());<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      }<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>    return logId;<a name="line.727"></a>
+<span class="sourceLineNo">728</span>  }<a name="line.728"></a>
+<span class="sourceLineNo">729</span><a name="line.729"></a>
+<span class="sourceLineNo">730</span>  private void updateStoreTracker(final PushType type,<a name="line.730"></a>
+<span class="sourceLineNo">731</span>      final long procId, final long[] subProcIds) {<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    switch (type) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span>      case INSERT:<a name="line.733"></a>
+<span class="sourceLineNo">734</span>        if (subProcIds == null) {<a name="line.734"></a>
+<span class="sourceLineNo">735</span>          storeTracker.insert(procId);<a name="line.735"></a>
+<span class="sourceLineNo">736</span>        } else if (procId == Procedure.NO_PROC_ID) {<a name="line.736"></a>
+<span class="sourceLineNo">737</span>          storeTracker.insert(subProcIds);<a name="line.737"></a>
+<span class="sourceLineNo">738</span>        } else {<a name="line.738"></a>
+<span class="sourceLineNo">739</span>          storeTracker.insert(procId, subProcIds);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.740"></a>
+<span class="sourceLineNo">741</span>        }<a name="line.741"></a>
+<span class="sourceLineNo">742</span>        break;<a name="line.742"></a>
+<span class="sourceLineNo">743</span>      case UPDATE:<a name="line.743"></a>
+<span class="sourceLineNo">744</span>        storeTracker.update(procId);<a name="line.744"></a>
+<span class="sourceLineNo">745</span>        holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.745"></a>
+<span class="sourceLineNo">746</span>        break;<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      case DELETE:<a name="line.747"></a>
+<span class="sourceLineNo">748</span>        if (subProcIds != null &amp;&amp; subProcIds.length &gt; 0) {<a name="line.748"></a>
+<span class="sourceLineNo">749</span>          storeTracker.delete(subProcIds);<a name="line.749"></a>
+<span class="sourceLineNo">750</span>          holdingCleanupTracker.setDeletedIfSet(subProcIds);<a name="line.750"></a>
+<span class="sourceLineNo">751</span>        } else {<a name="line.751"></a>
+<span class="sourceLineNo">752</span>          storeTracker.delete(procId);<a name="line.752"></a>
+<span class="sourceLineNo">753</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.753"></a>
+<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
+<span class="sourceLineNo">755</span>        break;<a name="line.755"></a>
+<span class="sourceLineNo">756</span>      default:<a name="line.756"></a>
+<span class="sourceLineNo">757</span>        throw new RuntimeException("invalid push type " + type);<a name="line.757"></a>
+<span class="sourceLineNo">758</span>    }<a name="line.758"></a>
+<span class="sourceLineNo">759</span>  }<a name="line.759"></a>
+<span class="sourceLineNo">760</span><a name="line.760"></a>
+<span class="sourceLineNo">761</span>  private boolean isSyncAborted() {<a name="line.761"></a>
+<span class="sourceLineNo">762</span>    return syncException.get() != null;<a name="line.762"></a>
+<span class="sourceLineNo">763</span>  }<a name="line.763"></a>
+<span class="sourceLineNo">764</span><a name="line.764"></a>
+<span class="sourceLineNo">765</span>  private void syncLoop() throws Throwable {<a name="line.765"></a>
+<span class="sourceLineNo">766</span>    long totalSyncedToStore = 0;<a name="line.766"></a>
+<span class="sourceLineNo">767</span>    inSync.set(false);<a name="line.767"></a>
+<span class="sourceLineNo">768</span>    lock.lock();<a name="line.768"></a>
+<span class="sourceLineNo">769</span>    try {<a name="line.769"></a>
+<span class="sourceLineNo">770</span>      while (isRunning()) {<a name="line.770"></a>
+<span class="sourceLineNo">771</span>        try {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>          // Wait until new data is available<a name="line.772"></a>
+<span class="sourceLineNo">773</span>          if (slotIndex == 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>            if (!loading.get()) {<a name="line.774"></a>
+<span class="sourceLineNo">775</span>              periodicRoll();<a name="line.775"></a>
 <span class="sourceLineNo">776</span>            }<a name="line.776"></a>
 <span class="sourceLineNo">777</span><a name="line.777"></a>
-<span class="sourceLineNo">778</span>            waitCond.await(getMillisToNextPeriodicRoll(), TimeUnit.MILLISECONDS);<a name="line.778"></a>
-<span class="sourceLineNo">779</span>            if (slotIndex == 0) {<a name="line.779"></a>
-<span class="sourceLineNo">780</span>              // no data.. probably a stop() or a periodic roll<a name="line.780"></a>
-<span class="sourceLineNo">781</span>              continue;<a name="line.781"></a>
-<span class="sourceLineNo">782</span>            }<a name="line.782"></a>
-<span class="sourceLineNo">783</span>          }<a name="line.783"></a>
-<span class="sourceLineNo">784</span>          // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing<a name="line.784"></a>
-<span class="sourceLineNo">785</span>          syncMaxSlot = runningProcCount;<a name="line.785"></a>
-<span class="sourceLineNo">786</span>          assert syncMaxSlot &gt; 0 : "unexpected syncMaxSlot=" + syncMaxSlot;<a name="line.786"></a>
-<span class="sourceLineNo">787</span>          final long syncWaitSt = System.currentTimeMillis();<a name="line.787"></a>
-<span class="sourceLineNo">788</span>          if (slotIndex != syncMaxSlot) {<a name="line.788"></a>
-<span class="sourceLineNo">789</span>            slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);<a name="line.789"></a>
+<span class="sourceLineNo">778</span>            if (LOG.isTraceEnabled()) {<a name="line.778"></a>
+<span class="sourceLineNo">779</span>              float rollTsSec = getMillisFromLastRoll() / 1000.0f;<a name="line.779"></a>
+<span class="sourceLineNo">780</span>              LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)",<a name="line.780"></a>
+<span class="sourceLineNo">781</span>                        StringUtils.humanSize(totalSynced.get()),<a name="line.781"></a>
+<span class="sourceLineNo">782</span>                        StringUtils.humanSize(totalSynced.get() / rollTsSec)));<a name="line.782"></a>
+<span class="sourceLineNo">783</span>            }<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>            waitCond.await(getMillisToNextPeriodicRoll(), TimeUnit.MILLISECONDS);<a name="line.785"></a>
+<span class="sourceLineNo">786</span>            if (slotIndex == 0) {<a name="line.786"></a>
+<span class="sourceLineNo">787</span>              // no data.. probably a stop() or a periodic roll<a name="line.787"></a>
+<span class="sourceLineNo">788</span>              continue;<a name="line.788"></a>
+<span class="sourceLineNo">789</span>            }<a name="line.789"></a>
 <span class="sourceLineNo">790</span>          }<a name="line.790"></a>
-<span class="sourceLineNo">791</span><a name="line.791"></a>
-<span class="sourceLineNo">792</span>          final long currentTs = System.currentTimeMillis();<a name="line.792"></a>
-<span class="sourceLineNo">793</span>          final long syncWaitMs = currentTs - syncWaitSt;<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          final float rollSec = getMillisFromLastRoll() / 1000.0f;<a name="line.794"></a>
-<span class="sourceLineNo">795</span>          final float syncedPerSec = totalSyncedToStore / rollSec;<a name="line.795"></a>
-<span class="sourceLineNo">796</span>          if (LOG.isTraceEnabled() &amp;&amp; (syncWaitMs &gt; 10 || slotIndex &lt; syncMaxSlot)) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span>            LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)",<a name="line.797"></a>
-<span class="sourceLineNo">798</span>                      StringUtils.humanTimeDiff(syncWaitMs), slotIndex,<a name="line.798"></a>
-<span class="sourceLineNo">799</span>                      StringUtils.humanSize(totalSyncedToStore),<a name="line.799"></a>
-<span class="sourceLineNo">800</span>                      StringUtils.humanSize(syncedPerSec)));<a name="line.800"></a>
-<span class="sourceLineNo">801</span>          }<a name="line.801"></a>
-<span class="sourceLineNo">802</span><a name="line.802"></a>
-<span class="sourceLineNo">803</span>          // update webui circular buffers (TODO: get rid of allocations)<a name="line.803"></a>
-<span class="sourceLineNo">804</span>          final SyncMetrics syncMetrics = new SyncMetrics();<a name="line.804"></a>
-<span class="sourceLineNo">805</span>          syncMetrics.timestamp = currentTs;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>          syncMetrics.syncWaitMs = syncWaitMs;<a name="line.806"></a>
-<span class="sourceLineNo">807</span>          syncMetrics.syncedEntries = slotIndex;<a name="line.807"></a>
-<span class="sourceLineNo">808</span>          syncMetrics.totalSyncedBytes = totalSyncedToStore;<a name="line.808"></a>
-<span class="sourceLineNo">809</span>          syncMetrics.syncedPerSec = syncedPerSec;<a name="line.809"></a>
-<span class="sourceLineNo">810</span>          syncMetricsQueue.add(syncMetrics);<a name="line.810"></a>
-<span class="sourceLineNo">811</span><a name="line.811"></a>
-<span class="sourceLineNo">812</span>          // sync<a name="line.812"></a>
-<span class="sourceLineNo">813</span>          inSync.set(true);<a name="line.813"></a>
-<span class="sourceLineNo">814</span>          long slotSize = syncSlots();<a name="line.814"></a>
-<span class="sourceLineNo">815</span>          logs.getLast().addToSize(slotSize);<a name="line.815"></a>
-<span class="sourceLineNo">816</span>          totalSyncedToStore = totalSynced.addAndGet(slotSize);<a name="line.816"></a>
-<span class="sourceLineNo">817</span>          slotIndex = 0;<a name="line.817"></a>
-<span class="sourceLineNo">818</span>          inSync.set(false);<a name="line.818"></a>
-<span class="sourceLineNo">819</span>          syncId.incrementAndGet();<a name="line.819"></a>
-<span class="sourceLineNo">820</span>        } catch (InterruptedException e) {<a name="line.820"></a>
-<span class="sourceLineNo">821</span>          Thread.currentThread().interrupt();<a name="line.821"></a>
-<span class="sourceLineNo">822</span>          syncException.compareAndSet(null, e);<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          sendAbortProcessSignal();<a name="line.823"></a>
-<span class="sourceLineNo">824</span>          throw e;<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        } catch (Throwable t) {<a name="line.825"></a>
-<span class="sourceLineNo">826</span>          syncException.compareAndSet(null, t);<a name="line.826"></a>
-<span class="sourceLineNo">827</span>          sendAbortProcessSignal();<a name="line.827"></a>
-<span class="sourceLineNo">828</span>          throw t;<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        } finally {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          syncCond.signalAll();<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span>    } finally {<a name="line.833"></a>
-<span class="sourceLineNo">834</span>      lock.unlock();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span>  }<a name="line.836"></a>
-<span class="sourceLineNo">837</span><a name="line.837"></a>
-<span class="sourceLineNo">838</span>  public ArrayList&lt;SyncMetrics&gt; getSyncMetrics() {<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    lock.lock();<a name="line.839"></a>
-<span class="sourceLineNo">840</span>    try {<a name="line.840"></a>
-<span class="sourceLineNo">841</span>      return new ArrayList&lt;&gt;(syncMetricsQueue);<a name="line.841"></a>
-<span class="sourceLineNo">842</span>    } finally {<a name="line.842"></a>
-<span class="sourceLineNo">843</span>      lock.unlock();<a name="line.843"></a>
-<span class="sourceLineNo">844</span>    }<a name="line.844"></a>
-<span class="sourceLineNo">845</span>  }<a name="line.845"></a>
-<span class="sourceLineNo">846</span><a name="line.846"></a>
-<span class="sourceLineNo">847</span>  private long syncSlots() throws Throwable {<a name="line.847"></a>
-<span class="sourceLineNo">848</span>    int retry = 0;<a name="line.848"></a>
-<span class="sourceLineNo">849</span>    int logRolled = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span>    long totalSynced = 0;<a name="line.850"></a>
-<span class="sourceLineNo">851</span>    do {<a name="line.851"></a>
-<span class="sourceLineNo">852</span>      try {<a name="line.852"></a>
-<span class="sourceLineNo">853</span>        totalSynced = syncSlots(stream, slots, 0, slotIndex);<a name="line.853"></a>
-<span class="sourceLineNo">854</span>        break;<a name="line.854"></a>
-<span class="sourceLineNo">855</span>      } catch (Throwable e) {<a name="line.855"></a>
-<span class="sourceLineNo">856</span>        LOG.warn("unable to sync slots, retry=" + retry);<a name="line.856"></a>
-<span class="sourceLineNo">857</span>        if (++retry &gt;= maxRetriesBeforeRoll) {<a name="line.857"></a>
-<span class="sourceLineNo">858</span>          if (logRolled &gt;= maxSyncFailureRoll &amp;&amp; isRunning()) {<a name="line.858"></a>
-<span class="sourceLineNo">859</span>            LOG.error("Sync slots after log roll failed, abort.", e);<a name="line.859"></a>
-<span class="sourceLineNo">860</span>            throw e;<a name="line.860"></a>
-<span class="sourceLineNo">861</span>          }<a name="line.861"></a>
-<span class="sourceLineNo">862</span><a name="line.862"></a>
-<span class="sourceLineNo">863</span>          if (!rollWriterWithRetries()) {<a name="line.863"></a>
-<span class="sourceLineNo">864</span>            throw e;<a name="line.864"></a>
-<span class="sourceLineNo">865</span>          }<a name="line.865"></a>
-<span class="sourceLineNo">866</span><a name="line.866"></a>
-<span class="sourceLineNo">867</span>          logRolled++;<a name="line.867"></a>
-<span class="sourceLineNo">868</span>          retry = 0;<a name="line.868"></a>
-<span class="sourceLineNo">869</span>        }<a name="line.869"></a>
-<span class="sourceLineNo">870</span>      }<a name="line.870"></a>
-<span class="sourceLineNo">871</span>    } while (isRunning());<a name="line.871"></a>
-<span class="sourceLineNo">872</span>    return totalSynced;<a name="line.872"></a>
-<span class="sourceLineNo">873</span>  }<a name="line.873"></a>
-<span class="sourceLineNo">874</span><a name="line.874"></a>
-<span class="sourceLineNo">875</span>  protected long syncSlots(final FSDataOutputStream stream, final ByteSlot[] slots,<a name="line.875"></a>
-<span class="sourceLineNo">876</span>      final int offset, final int count) throws IOException {<a name="line.876"></a>
-<span class="sourceLineNo">877</span>    long totalSynced = 0;<a name="line.877"></a>
-<span class="sourceLineNo">878</span>    for (int i = 0; i &lt; count; ++i) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span>      final ByteSlot data = slots[offset + i];<a name="line.879"></a>
-<span class="sourceLineNo">880</span>      data.writeTo(stream);<a name="line.880"></a>
-<span class="sourceLineNo">881</span>      totalSynced += data.size();<a name="line.881"></a>
-<span class="sourceLineNo">882</span>    }<a name="line.882"></a>
-<span class="sourceLineNo">883</span><a name="line.883"></a>
-<span class="sourceLineNo">884</span>    syncStream(stream);<a name="line.884"></a>
-<span class="sourceLineNo">885</span>    sendPostSyncSignal();<a name="line.885"></a>
-<span class="sourceLineNo">886</span><a name="line.886"></a>
-<span class="sourceLineNo">887</span>    if (LOG.isTraceEnabled()) {<a name="line.887"></a>
-<span class="sourceLineNo">888</span>      LOG.trace("Sync slots=" + count + '/' + syncMaxSlot +<a name="line.888"></a>
-<span class="sourceLineNo">889</span>                ", flushed=" + StringUtils.humanSize(totalSynced));<a name="line.889"></a>
-<span class="sourceLineNo">890</span>    }<a name="line.890"></a>
-<span class="sourceLineNo">891</span>    return totalSynced;<a name="line.891"></a>
-<span class="sourceLineNo">892</span>  }<a name="line.892"></a>
+<span class="sourceLineNo">791</span>          // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing<a name="line.791"></a>
+<span class="sourceLineNo">792</span>          syncMaxSlot = runningProcCount;<a name="line.792"></a>
+<span class="sourceLineNo">793</span>          assert syncMaxSlot &gt; 0 : "unexpected syncMaxSlot=" + syncMaxSlot;<a name="line.793"></a>
+<span class="sourceLineNo">794</span>          final long syncWaitSt = System.currentTimeMillis();<a name="line.794"></a>
+<span class="sourceLineNo">795</span>          if (slotIndex != syncMaxSlot) {<a name="line.795"></a>
+<span class="sourceLineNo">796</span>            slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);<a name="line.796"></a>
+<span class="sourceLineNo">797</span>          }<a name="line.797"></a>
+<span class="sourceLineNo">798</span><a name="line.798"></a>
+<span class="sourceLineNo">799</span>          final long currentTs = System.currentTimeMillis();<a name="line.799"></a>
+<span class="sourceLineNo">800</span>          final long syncWaitMs = currentTs - syncWaitSt;<a name="line.800"></a>
+<span class="sourceLineNo">801</span>          final float rollSec = getMillisFromLastRoll() / 1000.0f;<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          final float syncedPerSec = totalSyncedToStore / rollSec;<a name="line.802"></a>
+<span class="sourceLineNo">803</span>          if (LOG.isTraceEnabled() &amp;&amp; (syncWaitMs &gt; 10 || slotIndex &lt; syncMaxSlot)) {<a name="line.803"></a>
+<span class="sourceLineNo">804</span>            LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)",<a name="line.804"></a>
+<span class="sourceLineNo">805</span>                      StringUtils.humanTimeDiff(syncWaitMs), slotIndex,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>                      StringUtils.humanSize(totalSyncedToStore),<a name="line.806"></a>
+<span class="sourceLineNo">807</span>                      StringUtils.humanSize(syncedPerSec)));<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          }<a name="line.808"></a>
+<span class="sourceLineNo">809</span><a name="line.809"></a>
+<span class="sourceLineNo">810</span>          // update webui circular buffers (TODO: get rid of allocations)<a name="line.810"></a>
+<span class="sourceLineNo">811</span>          final SyncMetrics syncMetrics = new SyncMetrics();<a name="line.811"></a>
+<span class="sourceLineNo">812</span>          syncMetrics.timestamp = currentTs;<a name="line.812"></a>
+<span class="sourceLineNo">813</span>          syncMetrics.syncWaitMs = syncWaitMs;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>          syncMetrics.syncedEntries = slotIndex;<a name="line.814"></a>
+<span class="sourceLineNo">815</span>          syncMetrics.totalSyncedBytes = totalSyncedToStore;<a name="line.815"></a>
+<span class="sourceLineNo">816</span>          syncMetrics.syncedPerSec = syncedPerSec;<a name="line.816"></a>
+<span class="sourceLineNo">817</span>          syncMetricsQueue.add(syncMetrics);<a name="line.817"></a>
+<span class="sourceLineNo">818</span><a name="line.818"></a>
+<span class="sourceLineNo">819</span>          // sync<a name="line.819"></a>
+<span class="sourceLineNo">820</span>          inSync.set(true);<a name="line.820"></a>
+<span class="sourceLineNo">821</span>          long slotSize = syncSlots();<a name="line.821"></a>
+<span class="sourceLineNo">822</span>          logs.getLast().addToSize(slotSize);<a name="line.822"></a>
+<span class="sourceLineNo">823</span>          totalSyncedToStore = totalSynced.addAndGet(slotSize);<a name="line.823"></a>
+<span class="sourceLineNo">824</span>          slotIndex = 0;<a name="line.824"></a>
+<span class="sourceLineNo">825</span>          inSync.set(false);<a name="line.825"></a>
+<span class="sourceLineNo">826</span>          syncId.incrementAndGet();<a name="line.826"></a>
+<span class="sourceLineNo">827</span>        } catch (InterruptedException e) {<a name="line.827"></a>
+<span class="sourceLineNo">828</span>          Thread.currentThread().interrupt();<a name="line.828"></a>
+<span class="sourceLineNo">829</span>          syncException.compareAndSet(null, e);<a name="line.829"></a>
+<span class="sourceLineNo">830</span>          sendAbortProcessSignal();<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          throw e;<a name="line.831"></a>
+<span class="sourceLineNo">832</span>        } catch (Throwable t) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>          syncException.compareAndSet(null, t);<a name="line.833"></a>
+<span class="sourceLineNo">834</span>          sendAbortProcessSignal();<a name="line.834"></a>
+<span class="sourceLineNo">835</span>          throw t;<a name="line.835"></a>
+<span class="sourceLineNo">836</span>        } finally {<a name="line.836"></a>
+<span class="sourceLineNo">837</span>          syncCond.signalAll();<a name="line.837"></a>
+<span class="sourceLineNo">838</span>        }<a name="line.838"></a>
+<span class="sourceLineNo">839</span>      }<a name="line.839"></a>
+<span class="sourceLineNo">840</span>    } finally {<a name="line.840"></a>
+<span class="sourceLineNo">841</span>      lock.unlock();<a name="line.841"></a>
+<span class="sourceLineNo">842</span>    }<a name="line.842"></a>
+<span class="sourceLineNo">843</span>  }<a name="line.843"></a>
+<span class="sourceLineNo">844</span><a name="line.844"></a>
+<span class="sourceLineNo">845</span>  public ArrayList&lt;SyncMetrics&gt; getSyncMetrics() {<a name="line.845"></a>
+<span class="sourceLineNo">846</span>    lock.lock();<a name="line.846"></a>
+<span class="sourceLineNo">847</span>    try {<a name="line.847"></a>
+<span class="sourceLineNo">848</span>      return new ArrayList&lt;&gt;(syncMetricsQueue);<a name="line.848"></a>
+<span class="sourceLineNo">849</span>    } finally {<a name="line.849"></a>
+<span class="sourceLineNo">850</span>      lock.unlock();<a name="line.850"></a>
+<span class="sourceLineNo">851</span>    }<a name="line.851"></a>
+<span class="sourceLineNo">852</span>  }<a name="line.852"></a>
+<span class="sourceLineNo">853</span><a name="line.853"></a>
+<span class="sourceLineNo">854</span>  private long syncSlots() throws Throwable {<a name="line.854"></a>
+<span class="sourceLineNo">855</span>    int retry = 0;<a name="line.855"></a>
+<span class="sourceLineNo">856</span>    int logRolled = 0;<a name="line.856"></a>
+<span class="sourceLineNo">857</span>    long totalSynced = 0;<a name="line.857"></a>
+<span class="sourceLineNo">858</span>    do {<a name="line.858"></a>
+<span class="sourceLineNo">859</span>      try {<a name="line.859"></a>
+<span class="sourceLineNo">860</span>        totalSynced = syncSlots(stream, slots, 0, slotIndex);<a name="line.860"></a>
+<span class="sourceLineNo">861</span>        break;<a name="line.861"></a>
+<span class="sourceLineNo">862</span>      } catch (Throwable e) {<a name="line.862"></a>
+<span class="sourceLineNo">863</span>        LOG.warn("unable to sync slots, retry=" + retry);<a name="line.863"></a>
+<span class="sourceLineNo">864</span>        if (++retry &gt;= maxRetriesBeforeRoll) {<a name="line.864"></a>
+<span class="sourceLineNo">865</span>          if (logRolled &gt;= maxSyncFailureRoll &amp;&amp; isRunning()) {<a name="line.865"></a>
+<span class="sourceLineNo">866</span>            LOG.error("Sync slots after log roll failed, abort.", e);<a name="line.866"></a>
+<span class="sourceLineNo">867</span>            throw e;<a name="line.867"></a>
+<span class="sourceLineNo">868</span>          }<a name="line.868"></a>
+<span class="sourceLineNo">869</span><a name="line.869"></a>
+<span class="sourceLineNo">870</span>          if (!rollWriterWithRetries()) {<a name="line.870"></a>
+<span class="sourceLineNo">871</span>            throw e;<a name="line.871"></a>
+<span class="sourceLineNo">872</span>          }<a name="line.872"></a>
+<span class="sourceLineNo">873</span><a name="line.873"></a>
+<span class="sourceLineNo">874</span>          logRolled++;<a name="line.874"></a>
+<span class="sourceLineNo">875</span>          retry = 0;<a name="line.875"></a>
+<span class="sourceLineNo">876</span>        }<a name="line.876"></a>
+<span class="sourceLineNo">877</span>      }<a name="line.877"></a>
+<span class="sourceLineNo">878</span>    } while (isRunning());<a name="line.878"></a>
+<span class="sourceLineNo">879</span>    return totalSynced;<a name="line.879"></a>
+<span class="sourceLineNo">880</span>  }<a name="line.880"></a>
+<span class="sourceLineNo">881</span><a name="line.881"></a>
+<span class="sourceLineNo">882</span>  protected long syncSlots(final FSDataOutputStream stream, final ByteSlot[] slots,<a name="line.882"></a>
+<span class="sourceLineNo">883</span>      final int offset, final int count) throws IOException {<a name="line.883"></a>
+<span class="sourceLineNo">884</span>    long totalSynced = 0;<a name="line.884"></a>
+<span class="sourceLineNo">885</span>    for (int i = 0; i &lt; count; ++i) {<a name="line.885"></a>
+<span class="sourceLineNo">886</span>      final ByteSlot data = slots[offset + i];<a name="line.886"></a>
+<span class="sourceLineNo">887</span>      data.writeTo(stream);<a name="line.887"></a>
+<span class="sourceLineNo">888</span>      totalSynced += data.size();<a name="line.888"></a>
+<span class="sourceLineNo">889</span>    }<a name="line.889"></a>
+<span class="sourceLineNo">890</span><a name="line.890"></a>
+<span class="sourceLineNo">891</span>    syncStream(stream);<a name="line.891"></a>
+<span class="sourceLineNo">892</span>    sendPostSyncSignal();<a name="line.892"></a>
 <span class="sourceLineNo">893</span><a name="line.893"></a>
-<span class="sourceLineNo">894</span>  protected void syncStream(final FSDataOutputStream stream) throws IOException {<a name="line.894"></a>
-<span class="sourceLineNo">895</span>    if (useHsync) {<a name="line.895"></a>
-<span class="sourceLineNo">896</span>      stream.hsync();<a name="line.896"></a>
-<span class="sourceLineNo">897</span>    } else {<a name="line.897"></a>
-<span class="sourceLineNo">898</span>      stream.hflush();<a name="line.898"></a>
-<span class="sourceLineNo">899</span>    }<a name="line.899"></a>
-<span class="sourceLineNo">900</span>  }<a name="line.900"></a>
-<span class="sourceLineNo">901</span><a name="line.901"></a>
-<span class="sourceLineNo">902</span>  private boolean rollWriterWithRetries() {<a name="line.902"></a>
-<span class="sourceLineNo">903</span>    for (int i = 0; i &lt; rollRetries &amp;&amp; isRunning(); ++i) {<a name="line.903"></a>
-<span class="sourceLineNo">904</span>      if (i &gt; 0) Threads.sleepWithoutInterrupt(waitBeforeRoll * i);<a name="line.904"></a>
-<span class="sourceLineNo">905</span><a name="line.905"></a>
-<span class="sourceLineNo">906</span>      try {<a name="line.906"></a>
-<span class="sourceLineNo">907</span>        if (rollWriter()) {<a name="line.907"></a>
-<span class="sourceLineNo">908</span>          return true;<a name="line.908"></a>
-<span class="sourceLineNo">909</span>        }<a name="line.909"></a>
-<span class="sourceLineNo">910</span>      } catch (IOException e) {<a name="line.910"></a>
-<span class="sourceLineNo">911</span>        LOG.warn("Unable to roll the log, attempt=" + (i + 1), e);<a name="line.911"></a>
-<span class="sourceLineNo">912</span>      }<a name="line.912"></a>
-<span class="sourceLineNo">913</span>    }<a name="line.913"></a>
-<span class="sourceLineNo">914</span>    LOG.error(HBaseMarkers.FATAL, "Unable to roll the log");<a name="line.914"></a>
-<span class="sourceLineNo">915</span>    return false;<a name="line.915"></a>
-<span class="sourceLineNo">916</span>  }<a name="line.916"></a>
-<span class="sourceLineNo">917</span><a name="line.917"></a>
-<span class="sourceLineNo">918</span>  private boolean tryRollWriter() {<a name="line.918"></a>
-<span class="sourceLineNo">919</span>    try {<a name="line.919"></a>
-<span class="sourceLineNo">920</span>      return rollWriter();<a name="line.920"></a>
-<span class="sourceLineNo">921</span>    } catch (IOException e) {<a name="line.921"></a>
-<span class="sourceLineNo">922</span>      LOG.warn("Unable to roll the log", e);<a name="line.922"></a>
-<span class="sourceLineNo">923</span>      return fal

<TRUNCATED>

[05/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html
index c63e65a..21df2b9 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html
@@ -432,900 +432,907 @@
 <span class="sourceLineNo">424</span>      it.next(); // Skip the current log<a name="line.424"></a>
 <span class="sourceLineNo">425</span><a name="line.425"></a>
 <span class="sourceLineNo">426</span>      ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {<a name="line.426"></a>
-<span class="sourceLineNo">427</span>        @Override<a name="line.427"></a>
-<span class="sourceLineNo">428</span>        public void setMaxProcId(long maxProcId) {<a name="line.428"></a>
-<span class="sourceLineNo">429</span>          loader.setMaxProcId(maxProcId);<a name="line.429"></a>
-<span class="sourceLineNo">430</span>        }<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>        @Override<a name="line.432"></a>
-<span class="sourceLineNo">433</span>        public void load(ProcedureIterator procIter) throws IOException {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>          loader.load(procIter);<a name="line.434"></a>
-<span class="sourceLineNo">435</span>        }<a name="line.435"></a>
-<span class="sourceLineNo">436</span><a name="line.436"></a>
-<span class="sourceLineNo">437</span>        @Override<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.438"></a>
-<span class="sourceLineNo">439</span>          loader.handleCorrupted(procIter);<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>        @Override<a name="line.442"></a>
-<span class="sourceLineNo">443</span>        public void markCorruptedWAL(ProcedureWALFile log, IOException e) {<a name="line.443"></a>
-<span class="sourceLineNo">444</span>          if (corruptedLogs == null) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span>            corruptedLogs = new HashSet&lt;&gt;();<a name="line.445"></a>
-<span class="sourceLineNo">446</span>          }<a name="line.446"></a>
-<span class="sourceLineNo">447</span>          corruptedLogs.add(log);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>          // TODO: sideline corrupted log<a name="line.448"></a>
-<span class="sourceLineNo">449</span>        }<a name="line.449"></a>
-<span class="sourceLineNo">450</span>      });<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    } finally {<a name="line.451"></a>
-<span class="sourceLineNo">452</span>      try {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>        // try to cleanup inactive wals and complete the operation<a name="line.453"></a>
-<span class="sourceLineNo">454</span>        buildHoldingCleanupTracker();<a name="line.454"></a>
-<span class="sourceLineNo">455</span>        tryCleanupLogsOnLoad();<a name="line.455"></a>
-<span class="sourceLineNo">456</span>        loading.set(false);<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      } finally {<a name="line.457"></a>
-<span class="sourceLineNo">458</span>        lock.unlock();<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      }<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>  }<a name="line.461"></a>
-<span class="sourceLineNo">462</span><a name="line.462"></a>
-<span class="sourceLineNo">463</span>  private void tryCleanupLogsOnLoad() {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>    // nothing to cleanup.<a name="line.464"></a>
-<span class="sourceLineNo">465</span>    if (logs.size() &lt;= 1) return;<a name="line.465"></a>
-<span class="sourceLineNo">466</span><a name="line.466"></a>
-<span class="sourceLineNo">467</span>    // the config says to not cleanup wals on load.<a name="line.467"></a>
-<span class="sourceLineNo">468</span>    if (!conf.getBoolean(EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>      DEFAULT_EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY)) {<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      LOG.debug("WALs cleanup on load is not enabled: " + getActiveLogs());<a name="line.470"></a>
-<span class="sourceLineNo">471</span>      return;<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    }<a name="line.472"></a>
+<span class="sourceLineNo">427</span>        long count = 0;<a name="line.427"></a>
+<span class="sourceLineNo">428</span><a name="line.428"></a>
+<span class="sourceLineNo">429</span>        @Override<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        public void setMaxProcId(long maxProcId) {<a name="line.430"></a>
+<span class="sourceLineNo">431</span>          loader.setMaxProcId(maxProcId);<a name="line.431"></a>
+<span class="sourceLineNo">432</span>        }<a name="line.432"></a>
+<span class="sourceLineNo">433</span><a name="line.433"></a>
+<span class="sourceLineNo">434</span>        @Override<a name="line.434"></a>
+<span class="sourceLineNo">435</span>        public void load(ProcedureIterator procIter) throws IOException {<a name="line.435"></a>
+<span class="sourceLineNo">436</span>          if ((++count % 1000) == 0) {<a name="line.436"></a>
+<span class="sourceLineNo">437</span>            // Log every 1000 procedures otherwise it looks like Master is dead if loads of WALs<a name="line.437"></a>
+<span class="sourceLineNo">438</span>            // and procedures to load.<a name="line.438"></a>
+<span class="sourceLineNo">439</span>            LOG.debug("Loaded {} procedures", this.count);<a name="line.439"></a>
+<span class="sourceLineNo">440</span>          }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>          loader.load(procIter);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>        }<a name="line.442"></a>
+<span class="sourceLineNo">443</span><a name="line.443"></a>
+<span class="sourceLineNo">444</span>        @Override<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.445"></a>
+<span class="sourceLineNo">446</span>          loader.handleCorrupted(procIter);<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        }<a name="line.447"></a>
+<span class="sourceLineNo">448</span><a name="line.448"></a>
+<span class="sourceLineNo">449</span>        @Override<a name="line.449"></a>
+<span class="sourceLineNo">450</span>        public void markCorruptedWAL(ProcedureWALFile log, IOException e) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>          if (corruptedLogs == null) {<a name="line.451"></a>
+<span class="sourceLineNo">452</span>            corruptedLogs = new HashSet&lt;&gt;();<a name="line.452"></a>
+<span class="sourceLineNo">453</span>          }<a name="line.453"></a>
+<span class="sourceLineNo">454</span>          corruptedLogs.add(log);<a name="line.454"></a>
+<span class="sourceLineNo">455</span>          // TODO: sideline corrupted log<a name="line.455"></a>
+<span class="sourceLineNo">456</span>        }<a name="line.456"></a>
+<span class="sourceLineNo">457</span>      });<a name="line.457"></a>
+<span class="sourceLineNo">458</span>    } finally {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>      try {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>        // try to cleanup inactive wals and complete the operation<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        buildHoldingCleanupTracker();<a name="line.461"></a>
+<span class="sourceLineNo">462</span>        tryCleanupLogsOnLoad();<a name="line.462"></a>
+<span class="sourceLineNo">463</span>        loading.set(false);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>      } finally {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>        lock.unlock();<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      }<a name="line.466"></a>
+<span class="sourceLineNo">467</span>    }<a name="line.467"></a>
+<span class="sourceLineNo">468</span>  }<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>  private void tryCleanupLogsOnLoad() {<a name="line.470"></a>
+<span class="sourceLineNo">471</span>    // nothing to cleanup.<a name="line.471"></a>
+<span class="sourceLineNo">472</span>    if (logs.size() &lt;= 1) return;<a name="line.472"></a>
 <span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>    try {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>      periodicRoll();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    } catch (IOException e) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      LOG.warn("Unable to cleanup logs on load: " + e.getMessage(), e);<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    }<a name="line.478"></a>
-<span class="sourceLineNo">479</span>  }<a name="line.479"></a>
+<span class="sourceLineNo">474</span>    // the config says to not cleanup wals on load.<a name="line.474"></a>
+<span class="sourceLineNo">475</span>    if (!conf.getBoolean(EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY,<a name="line.475"></a>
+<span class="sourceLineNo">476</span>      DEFAULT_EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY)) {<a name="line.476"></a>
+<span class="sourceLineNo">477</span>      LOG.debug("WALs cleanup on load is not enabled: " + getActiveLogs());<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      return;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
 <span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span>  @Override<a name="line.481"></a>
-<span class="sourceLineNo">482</span>  public void insert(final Procedure proc, final Procedure[] subprocs) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    if (LOG.isTraceEnabled()) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span>      LOG.trace("Insert " + proc + ", subproc=" + Arrays.toString(subprocs));<a name="line.484"></a>
+<span class="sourceLineNo">481</span>    try {<a name="line.481"></a>
+<span class="sourceLineNo">482</span>      periodicRoll();<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    } catch (IOException e) {<a name="line.483"></a>
+<span class="sourceLineNo">484</span>      LOG.warn("Unable to cleanup logs on load: " + e.getMessage(), e);<a name="line.484"></a>
 <span class="sourceLineNo">485</span>    }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>    ByteSlot slot = acquireSlot();<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    try {<a name="line.488"></a>
-<span class="sourceLineNo">489</span>      // Serialize the insert<a name="line.489"></a>
-<span class="sourceLineNo">490</span>      long[] subProcIds = null;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>      if (subprocs != null) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span>        ProcedureWALFormat.writeInsert(slot, proc, subprocs);<a name="line.492"></a>
-<span class="sourceLineNo">493</span>        subProcIds = new long[subprocs.length];<a name="line.493"></a>
-<span class="sourceLineNo">494</span>        for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>          subProcIds[i] = subprocs[i].getProcId();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>        }<a name="line.496"></a>
-<span class="sourceLineNo">497</span>      } else {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>        assert !proc.hasParent();<a name="line.498"></a>
-<span class="sourceLineNo">499</span>        ProcedureWALFormat.writeInsert(slot, proc);<a name="line.499"></a>
-<span class="sourceLineNo">500</span>      }<a name="line.500"></a>
-<span class="sourceLineNo">501</span><a name="line.501"></a>
-<span class="sourceLineNo">502</span>      // Push the transaction data and wait until it is persisted<a name="line.502"></a>
-<span class="sourceLineNo">503</span>      pushData(PushType.INSERT, slot, proc.getProcId(), subProcIds);<a name="line.503"></a>
-<span class="sourceLineNo">504</span>    } catch (IOException e) {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      // We are not able to serialize the procedure.<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      // this is a code error, and we are not able to go on.<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: proc=" +<a name="line.507"></a>
-<span class="sourceLineNo">508</span>          proc + ", subprocs=" + Arrays.toString(subprocs), e);<a name="line.508"></a>
-<span class="sourceLineNo">509</span>      throw new RuntimeException(e);<a name="line.509"></a>
-<span class="sourceLineNo">510</span>    } finally {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      releaseSlot(slot);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>    }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>  }<a name="line.513"></a>
-<span class="sourceLineNo">514</span><a name="line.514"></a>
-<span class="sourceLineNo">515</span>  @Override<a name="line.515"></a>
-<span class="sourceLineNo">516</span>  public void insert(final Procedure[] procs) {<a name="line.516"></a>
-<span class="sourceLineNo">517</span>    if (LOG.isTraceEnabled()) {<a name="line.517"></a>
-<span class="sourceLineNo">518</span>      LOG.trace("Insert " + Arrays.toString(procs));<a name="line.518"></a>
+<span class="sourceLineNo">486</span>  }<a name="line.486"></a>
+<span class="sourceLineNo">487</span><a name="line.487"></a>
+<span class="sourceLineNo">488</span>  @Override<a name="line.488"></a>
+<span class="sourceLineNo">489</span>  public void insert(final Procedure proc, final Procedure[] subprocs) {<a name="line.489"></a>
+<span class="sourceLineNo">490</span>    if (LOG.isTraceEnabled()) {<a name="line.490"></a>
+<span class="sourceLineNo">491</span>      LOG.trace("Insert " + proc + ", subproc=" + Arrays.toString(subprocs));<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    }<a name="line.492"></a>
+<span class="sourceLineNo">493</span><a name="line.493"></a>
+<span class="sourceLineNo">494</span>    ByteSlot slot = acquireSlot();<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    try {<a name="line.495"></a>
+<span class="sourceLineNo">496</span>      // Serialize the insert<a name="line.496"></a>
+<span class="sourceLineNo">497</span>      long[] subProcIds = null;<a name="line.497"></a>
+<span class="sourceLineNo">498</span>      if (subprocs != null) {<a name="line.498"></a>
+<span class="sourceLineNo">499</span>        ProcedureWALFormat.writeInsert(slot, proc, subprocs);<a name="line.499"></a>
+<span class="sourceLineNo">500</span>        subProcIds = new long[subprocs.length];<a name="line.500"></a>
+<span class="sourceLineNo">501</span>        for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.501"></a>
+<span class="sourceLineNo">502</span>          subProcIds[i] = subprocs[i].getProcId();<a name="line.502"></a>
+<span class="sourceLineNo">503</span>        }<a name="line.503"></a>
+<span class="sourceLineNo">504</span>      } else {<a name="line.504"></a>
+<span class="sourceLineNo">505</span>        assert !proc.hasParent();<a name="line.505"></a>
+<span class="sourceLineNo">506</span>        ProcedureWALFormat.writeInsert(slot, proc);<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      }<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      // Push the transaction data and wait until it is persisted<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      pushData(PushType.INSERT, slot, proc.getProcId(), subProcIds);<a name="line.510"></a>
+<span class="sourceLineNo">511</span>    } catch (IOException e) {<a name="line.511"></a>
+<span class="sourceLineNo">512</span>      // We are not able to serialize the procedure.<a name="line.512"></a>
+<span class="sourceLineNo">513</span>      // this is a code error, and we are not able to go on.<a name="line.513"></a>
+<span class="sourceLineNo">514</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: proc=" +<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          proc + ", subprocs=" + Arrays.toString(subprocs), e);<a name="line.515"></a>
+<span class="sourceLineNo">516</span>      throw new RuntimeException(e);<a name="line.516"></a>
+<span class="sourceLineNo">517</span>    } finally {<a name="line.517"></a>
+<span class="sourceLineNo">518</span>      releaseSlot(slot);<a name="line.518"></a>
 <span class="sourceLineNo">519</span>    }<a name="line.519"></a>
-<span class="sourceLineNo">520</span><a name="line.520"></a>
-<span class="sourceLineNo">521</span>    ByteSlot slot = acquireSlot();<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    try {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      // Serialize the insert<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      long[] procIds = new long[procs.length];<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      for (int i = 0; i &lt; procs.length; ++i) {<a name="line.525"></a>
-<span class="sourceLineNo">526</span>        assert !procs[i].hasParent();<a name="line.526"></a>
-<span class="sourceLineNo">527</span>        procIds[i] = procs[i].getProcId();<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        ProcedureWALFormat.writeInsert(slot, procs[i]);<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      }<a name="line.529"></a>
-<span class="sourceLineNo">530</span><a name="line.530"></a>
-<span class="sourceLineNo">531</span>      // Push the transaction data and wait until it is persisted<a name="line.531"></a>
-<span class="sourceLineNo">532</span>      pushData(PushType.INSERT, slot, Procedure.NO_PROC_ID, procIds);<a name="line.532"></a>
-<span class="sourceLineNo">533</span>    } catch (IOException e) {<a name="line.533"></a>
-<span class="sourceLineNo">534</span>      // We are not able to serialize the procedure.<a name="line.534"></a>
-<span class="sourceLineNo">535</span>      // this is a code error, and we are not able to go on.<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: " +<a name="line.536"></a>
-<span class="sourceLineNo">537</span>          Arrays.toString(procs), e);<a name="line.537"></a>
-<span class="sourceLineNo">538</span>      throw new RuntimeException(e);<a name="line.538"></a>
-<span class="sourceLineNo">539</span>    } finally {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      releaseSlot(slot);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>  public void update(final Procedure proc) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>    if (LOG.isTraceEnabled()) {<a name="line.546"></a>
-<span class="sourceLineNo">547</span>      LOG.trace("Update " + proc);<a name="line.547"></a>
+<span class="sourceLineNo">520</span>  }<a name="line.520"></a>
+<span class="sourceLineNo">521</span><a name="line.521"></a>
+<span class="sourceLineNo">522</span>  @Override<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  public void insert(final Procedure[] procs) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    if (LOG.isTraceEnabled()) {<a name="line.524"></a>
+<span class="sourceLineNo">525</span>      LOG.trace("Insert " + Arrays.toString(procs));<a name="line.525"></a>
+<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span>    ByteSlot slot = acquireSlot();<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    try {<a name="line.529"></a>
+<span class="sourceLineNo">530</span>      // Serialize the insert<a name="line.530"></a>
+<span class="sourceLineNo">531</span>      long[] procIds = new long[procs.length];<a name="line.531"></a>
+<span class="sourceLineNo">532</span>      for (int i = 0; i &lt; procs.length; ++i) {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>        assert !procs[i].hasParent();<a name="line.533"></a>
+<span class="sourceLineNo">534</span>        procIds[i] = procs[i].getProcId();<a name="line.534"></a>
+<span class="sourceLineNo">535</span>        ProcedureWALFormat.writeInsert(slot, procs[i]);<a name="line.535"></a>
+<span class="sourceLineNo">536</span>      }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>      // Push the transaction data and wait until it is persisted<a name="line.538"></a>
+<span class="sourceLineNo">539</span>      pushData(PushType.INSERT, slot, Procedure.NO_PROC_ID, procIds);<a name="line.539"></a>
+<span class="sourceLineNo">540</span>    } catch (IOException e) {<a name="line.540"></a>
+<span class="sourceLineNo">541</span>      // We are not able to serialize the procedure.<a name="line.541"></a>
+<span class="sourceLineNo">542</span>      // this is a code error, and we are not able to go on.<a name="line.542"></a>
+<span class="sourceLineNo">543</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: " +<a name="line.543"></a>
+<span class="sourceLineNo">544</span>          Arrays.toString(procs), e);<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      throw new RuntimeException(e);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    } finally {<a name="line.546"></a>
+<span class="sourceLineNo">547</span>      releaseSlot(slot);<a name="line.547"></a>
 <span class="sourceLineNo">548</span>    }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>    ByteSlot slot = acquireSlot();<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    try {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      // Serialize the update<a name="line.552"></a>
-<span class="sourceLineNo">553</span>      ProcedureWALFormat.writeUpdate(slot, proc);<a name="line.553"></a>
-<span class="sourceLineNo">554</span><a name="line.554"></a>
-<span class="sourceLineNo">555</span>      // Push the transaction data and wait until it is persisted<a name="line.555"></a>
-<span class="sourceLineNo">556</span>      pushData(PushType.UPDATE, slot, proc.getProcId(), null);<a name="line.556"></a>
-<span class="sourceLineNo">557</span>    } catch (IOException e) {<a name="line.557"></a>
-<span class="sourceLineNo">558</span>      // We are not able to serialize the procedure.<a name="line.558"></a>
-<span class="sourceLineNo">559</span>      // this is a code error, and we are not able to go on.<a name="line.559"></a>
-<span class="sourceLineNo">560</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.560"></a>
-<span class="sourceLineNo">561</span>      throw new RuntimeException(e);<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    } finally {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      releaseSlot(slot);<a name="line.563"></a>
-<span class="sourceLineNo">564</span>    }<a name="line.564"></a>
-<span class="sourceLineNo">565</span>  }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>  @Override<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  public void delete(final long procId) {<a name="line.568"></a>
-<span class="sourceLineNo">569</span>    if (LOG.isTraceEnabled()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>      LOG.trace("Delete " + procId);<a name="line.570"></a>
+<span class="sourceLineNo">549</span>  }<a name="line.549"></a>
+<span class="sourceLineNo">550</span><a name="line.550"></a>
+<span class="sourceLineNo">551</span>  @Override<a name="line.551"></a>
+<span class="sourceLineNo">552</span>  public void update(final Procedure proc) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    if (LOG.isTraceEnabled()) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      LOG.trace("Update " + proc);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span><a name="line.556"></a>
+<span class="sourceLineNo">557</span>    ByteSlot slot = acquireSlot();<a name="line.557"></a>
+<span class="sourceLineNo">558</span>    try {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>      // Serialize the update<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      ProcedureWALFormat.writeUpdate(slot, proc);<a name="line.560"></a>
+<span class="sourceLineNo">561</span><a name="line.561"></a>
+<span class="sourceLineNo">562</span>      // Push the transaction data and wait until it is persisted<a name="line.562"></a>
+<span class="sourceLineNo">563</span>      pushData(PushType.UPDATE, slot, proc.getProcId(), null);<a name="line.563"></a>
+<span class="sourceLineNo">564</span>    } catch (IOException e) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span>      // We are not able to serialize the procedure.<a name="line.565"></a>
+<span class="sourceLineNo">566</span>      // this is a code error, and we are not able to go on.<a name="line.566"></a>
+<span class="sourceLineNo">567</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.567"></a>
+<span class="sourceLineNo">568</span>      throw new RuntimeException(e);<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    } finally {<a name="line.569"></a>
+<span class="sourceLineNo">570</span>      releaseSlot(slot);<a name="line.570"></a>
 <span class="sourceLineNo">571</span>    }<a name="line.571"></a>
-<span class="sourceLineNo">572</span><a name="line.572"></a>
-<span class="sourceLineNo">573</span>    ByteSlot slot = acquireSlot();<a name="line.573"></a>
-<span class="sourceLineNo">574</span>    try {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>      // Serialize the delete<a name="line.575"></a>
-<span class="sourceLineNo">576</span>      ProcedureWALFormat.writeDelete(slot, procId);<a name="line.576"></a>
-<span class="sourceLineNo">577</span><a name="line.577"></a>
-<span class="sourceLineNo">578</span>      // Push the transaction data and wait until it is persisted<a name="line.578"></a>
-<span class="sourceLineNo">579</span>      pushData(PushType.DELETE, slot, procId, null);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    } catch (IOException e) {<a name="line.580"></a>
-<span class="sourceLineNo">581</span>      // We are not able to serialize the procedure.<a name="line.581"></a>
-<span class="sourceLineNo">582</span>      // this is a code error, and we are not able to go on.<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + procId, e);<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      throw new RuntimeException(e);<a name="line.584"></a>
-<span class="sourceLineNo">585</span>    } finally {<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      releaseSlot(slot);<a name="line.586"></a>
-<span class="sourceLineNo">587</span>    }<a name="line.587"></a>
-<span class="sourceLineNo">588</span>  }<a name="line.588"></a>
-<span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>  @Override<a name="line.590"></a>
-<span class="sourceLineNo">591</span>  public void delete(final Procedure proc, final long[] subProcIds) {<a name="line.591"></a>
-<span class="sourceLineNo">592</span>    assert proc != null : "expected a non-null procedure";<a name="line.592"></a>
-<span class="sourceLineNo">593</span>    assert subProcIds != null &amp;&amp; subProcIds.length &gt; 0 : "expected subProcIds";<a name="line.593"></a>
-<span class="sourceLineNo">594</span>    if (LOG.isTraceEnabled()) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>      LOG.trace("Update " + proc + " and Delete " + Arrays.toString(subProcIds));<a name="line.595"></a>
-<span class="sourceLineNo">596</span>    }<a name="line.596"></a>
-<span class="sourceLineNo">597</span><a name="line.597"></a>
-<span class="sourceLineNo">598</span>    ByteSlot slot = acquireSlot();<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    try {<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // Serialize the delete<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      ProcedureWALFormat.writeDelete(slot, proc, subProcIds);<a name="line.601"></a>
-<span class="sourceLineNo">602</span><a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // Push the transaction data and wait until it is persisted<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      pushData(PushType.DELETE, slot, proc.getProcId(), subProcIds);<a name="line.604"></a>
-<span class="sourceLineNo">605</span>    } catch (IOException e) {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      // We are not able to serialize the procedure.<a name="line.606"></a>
-<span class="sourceLineNo">607</span>      // this is a code error, and we are not able to go on.<a name="line.607"></a>
-<span class="sourceLineNo">608</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.608"></a>
-<span class="sourceLineNo">609</span>      throw new RuntimeException(e);<a name="line.609"></a>
-<span class="sourceLineNo">610</span>    } finally {<a name="line.610"></a>
-<span class="sourceLineNo">611</span>      releaseSlot(slot);<a name="line.611"></a>
-<span class="sourceLineNo">612</span>    }<a name="line.612"></a>
-<span class="sourceLineNo">613</span>  }<a name="line.613"></a>
-<span class="sourceLineNo">614</span><a name="line.614"></a>
-<span class="sourceLineNo">615</span>  @Override<a name="line.615"></a>
-<span class="sourceLineNo">616</span>  public void delete(final long[] procIds, final int offset, final int count) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>    if (count == 0) return;<a name="line.617"></a>
-<span class="sourceLineNo">618</span>    if (offset == 0 &amp;&amp; count == procIds.length) {<a name="line.618"></a>
-<span class="sourceLineNo">619</span>      delete(procIds);<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    } else if (count == 1) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      delete(procIds[offset]);<a name="line.621"></a>
-<span class="sourceLineNo">622</span>    } else {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>      delete(Arrays.copyOfRange(procIds, offset, offset + count));<a name="line.623"></a>
-<span class="sourceLineNo">624</span>    }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>  }<a name="line.625"></a>
-<span class="sourceLineNo">626</span><a name="line.626"></a>
-<span class="sourceLineNo">627</span>  private void delete(final long[] procIds) {<a name="line.627"></a>
-<span class="sourceLineNo">628</span>    if (LOG.isTraceEnabled()) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>      LOG.trace("Delete " + Arrays.toString(procIds));<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    }<a name="line.630"></a>
-<span class="sourceLineNo">631</span><a name="line.631"></a>
-<span class="sourceLineNo">632</span>    final ByteSlot slot = acquireSlot();<a name="line.632"></a>
-<span class="sourceLineNo">633</span>    try {<a name="line.633"></a>
-<span class="sourceLineNo">634</span>      // Serialize the delete<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.635"></a>
-<span class="sourceLineNo">636</span>        ProcedureWALFormat.writeDelete(slot, procIds[i]);<a name="line.636"></a>
-<span class="sourceLineNo">637</span>      }<a name="line.637"></a>
+<span class="sourceLineNo">572</span>  }<a name="line.572"></a>
+<span class="sourceLineNo">573</span><a name="line.573"></a>
+<span class="sourceLineNo">574</span>  @Override<a name="line.574"></a>
+<span class="sourceLineNo">575</span>  public void delete(final long procId) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span>    if (LOG.isTraceEnabled()) {<a name="line.576"></a>
+<span class="sourceLineNo">577</span>      LOG.trace("Delete " + procId);<a name="line.577"></a>
+<span class="sourceLineNo">578</span>    }<a name="line.578"></a>
+<span class="sourceLineNo">579</span><a name="line.579"></a>
+<span class="sourceLineNo">580</span>    ByteSlot slot = acquireSlot();<a name="line.580"></a>
+<span class="sourceLineNo">581</span>    try {<a name="line.581"></a>
+<span class="sourceLineNo">582</span>      // Serialize the delete<a name="line.582"></a>
+<span class="sourceLineNo">583</span>      ProcedureWALFormat.writeDelete(slot, procId);<a name="line.583"></a>
+<span class="sourceLineNo">584</span><a name="line.584"></a>
+<span class="sourceLineNo">585</span>      // Push the transaction data and wait until it is persisted<a name="line.585"></a>
+<span class="sourceLineNo">586</span>      pushData(PushType.DELETE, slot, procId, null);<a name="line.586"></a>
+<span class="sourceLineNo">587</span>    } catch (IOException e) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>      // We are not able to serialize the procedure.<a name="line.588"></a>
+<span class="sourceLineNo">589</span>      // this is a code error, and we are not able to go on.<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + procId, e);<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      throw new RuntimeException(e);<a name="line.591"></a>
+<span class="sourceLineNo">592</span>    } finally {<a name="line.592"></a>
+<span class="sourceLineNo">593</span>      releaseSlot(slot);<a name="line.593"></a>
+<span class="sourceLineNo">594</span>    }<a name="line.594"></a>
+<span class="sourceLineNo">595</span>  }<a name="line.595"></a>
+<span class="sourceLineNo">596</span><a name="line.596"></a>
+<span class="sourceLineNo">597</span>  @Override<a name="line.597"></a>
+<span class="sourceLineNo">598</span>  public void delete(final Procedure proc, final long[] subProcIds) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>    assert proc != null : "expected a non-null procedure";<a name="line.599"></a>
+<span class="sourceLineNo">600</span>    assert subProcIds != null &amp;&amp; subProcIds.length &gt; 0 : "expected subProcIds";<a name="line.600"></a>
+<span class="sourceLineNo">601</span>    if (LOG.isTraceEnabled()) {<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      LOG.trace("Update " + proc + " and Delete " + Arrays.toString(subProcIds));<a name="line.602"></a>
+<span class="sourceLineNo">603</span>    }<a name="line.603"></a>
+<span class="sourceLineNo">604</span><a name="line.604"></a>
+<span class="sourceLineNo">605</span>    ByteSlot slot = acquireSlot();<a name="line.605"></a>
+<span class="sourceLineNo">606</span>    try {<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // Serialize the delete<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      ProcedureWALFormat.writeDelete(slot, proc, subProcIds);<a name="line.608"></a>
+<span class="sourceLineNo">609</span><a name="line.609"></a>
+<span class="sourceLineNo">610</span>      // Push the transaction data and wait until it is persisted<a name="line.610"></a>
+<span class="sourceLineNo">611</span>      pushData(PushType.DELETE, slot, proc.getProcId(), subProcIds);<a name="line.611"></a>
+<span class="sourceLineNo">612</span>    } catch (IOException e) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>      // We are not able to serialize the procedure.<a name="line.613"></a>
+<span class="sourceLineNo">614</span>      // this is a code error, and we are not able to go on.<a name="line.614"></a>
+<span class="sourceLineNo">615</span>      LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e);<a name="line.615"></a>
+<span class="sourceLineNo">616</span>      throw new RuntimeException(e);<a name="line.616"></a>
+<span class="sourceLineNo">617</span>    } finally {<a name="line.617"></a>
+<span class="sourceLineNo">618</span>      releaseSlot(slot);<a name="line.618"></a>
+<span class="sourceLineNo">619</span>    }<a name="line.619"></a>
+<span class="sourceLineNo">620</span>  }<a name="line.620"></a>
+<span class="sourceLineNo">621</span><a name="line.621"></a>
+<span class="sourceLineNo">622</span>  @Override<a name="line.622"></a>
+<span class="sourceLineNo">623</span>  public void delete(final long[] procIds, final int offset, final int count) {<a name="line.623"></a>
+<span class="sourceLineNo">624</span>    if (count == 0) return;<a name="line.624"></a>
+<span class="sourceLineNo">625</span>    if (offset == 0 &amp;&amp; count == procIds.length) {<a name="line.625"></a>
+<span class="sourceLineNo">626</span>      delete(procIds);<a name="line.626"></a>
+<span class="sourceLineNo">627</span>    } else if (count == 1) {<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      delete(procIds[offset]);<a name="line.628"></a>
+<span class="sourceLineNo">629</span>    } else {<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      delete(Arrays.copyOfRange(procIds, offset, offset + count));<a name="line.630"></a>
+<span class="sourceLineNo">631</span>    }<a name="line.631"></a>
+<span class="sourceLineNo">632</span>  }<a name="line.632"></a>
+<span class="sourceLineNo">633</span><a name="line.633"></a>
+<span class="sourceLineNo">634</span>  private void delete(final long[] procIds) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>    if (LOG.isTraceEnabled()) {<a name="line.635"></a>
+<span class="sourceLineNo">636</span>      LOG.trace("Delete " + Arrays.toString(procIds));<a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
 <span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>      // Push the transaction data and wait until it is persisted<a name="line.639"></a>
-<span class="sourceLineNo">640</span>      pushData(PushType.DELETE, slot, Procedure.NO_PROC_ID, procIds);<a name="line.640"></a>
-<span class="sourceLineNo">641</span>    } catch (IOException e) {<a name="line.641"></a>
-<span class="sourceLineNo">642</span>      // We are not able to serialize the procedure.<a name="line.642"></a>
-<span class="sourceLineNo">643</span>      // this is a code error, and we are not able to go on.<a name="line.643"></a>
-<span class="sourceLineNo">644</span>      LOG.error("Unable to serialize the procedures: " + Arrays.toString(procIds), e);<a name="line.644"></a>
-<span class="sourceLineNo">645</span>      throw new RuntimeException(e);<a name="line.645"></a>
-<span class="sourceLineNo">646</span>    } finally {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      releaseSlot(slot);<a name="line.647"></a>
-<span class="sourceLineNo">648</span>    }<a name="line.648"></a>
-<span class="sourceLineNo">649</span>  }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span>  private ByteSlot acquireSlot() {<a name="line.651"></a>
-<span class="sourceLineNo">652</span>    ByteSlot slot = slotsCache.poll();<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    return slot != null ? slot : new ByteSlot();<a name="line.653"></a>
-<span class="sourceLineNo">654</span>  }<a name="line.654"></a>
-<span class="sourceLineNo">655</span><a name="line.655"></a>
-<span class="sourceLineNo">656</span>  private void releaseSlot(final ByteSlot slot) {<a name="line.656"></a>
-<span class="sourceLineNo">657</span>    slot.reset();<a name="line.657"></a>
-<span class="sourceLineNo">658</span>    slotsCache.offer(slot);<a name="line.658"></a>
-<span class="sourceLineNo">659</span>  }<a name="line.659"></a>
-<span class="sourceLineNo">660</span><a name="line.660"></a>
-<span class="sourceLineNo">661</span>  private enum PushType { INSERT, UPDATE, DELETE }<a name="line.661"></a>
+<span class="sourceLineNo">639</span>    final ByteSlot slot = acquireSlot();<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    try {<a name="line.640"></a>
+<span class="sourceLineNo">641</span>      // Serialize the delete<a name="line.641"></a>
+<span class="sourceLineNo">642</span>      for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.642"></a>
+<span class="sourceLineNo">643</span>        ProcedureWALFormat.writeDelete(slot, procIds[i]);<a name="line.643"></a>
+<span class="sourceLineNo">644</span>      }<a name="line.644"></a>
+<span class="sourceLineNo">645</span><a name="line.645"></a>
+<span class="sourceLineNo">646</span>      // Push the transaction data and wait until it is persisted<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      pushData(PushType.DELETE, slot, Procedure.NO_PROC_ID, procIds);<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    } catch (IOException e) {<a name="line.648"></a>
+<span class="sourceLineNo">649</span>      // We are not able to serialize the procedure.<a name="line.649"></a>
+<span class="sourceLineNo">650</span>      // this is a code error, and we are not able to go on.<a name="line.650"></a>
+<span class="sourceLineNo">651</span>      LOG.error("Unable to serialize the procedures: " + Arrays.toString(procIds), e);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>      throw new RuntimeException(e);<a name="line.652"></a>
+<span class="sourceLineNo">653</span>    } finally {<a name="line.653"></a>
+<span class="sourceLineNo">654</span>      releaseSlot(slot);<a name="line.654"></a>
+<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
+<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
+<span class="sourceLineNo">657</span><a name="line.657"></a>
+<span class="sourceLineNo">658</span>  private ByteSlot acquireSlot() {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    ByteSlot slot = slotsCache.poll();<a name="line.659"></a>
+<span class="sourceLineNo">660</span>    return slot != null ? slot : new ByteSlot();<a name="line.660"></a>
+<span class="sourceLineNo">661</span>  }<a name="line.661"></a>
 <span class="sourceLineNo">662</span><a name="line.662"></a>
-<span class="sourceLineNo">663</span>  private long pushData(final PushType type, final ByteSlot slot,<a name="line.663"></a>
-<span class="sourceLineNo">664</span>      final long procId, final long[] subProcIds) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>    if (!isRunning()) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span>      throw new RuntimeException("the store must be running before inserting data");<a name="line.666"></a>
-<span class="sourceLineNo">667</span>    }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    if (logs.isEmpty()) {<a name="line.668"></a>
-<span class="sourceLineNo">669</span>      throw new RuntimeException("recoverLease() must be called before inserting data");<a name="line.669"></a>
-<span class="sourceLineNo">670</span>    }<a name="line.670"></a>
-<span class="sourceLineNo">671</span><a name="line.671"></a>
-<span class="sourceLineNo">672</span>    long logId = -1;<a name="line.672"></a>
-<span class="sourceLineNo">673</span>    lock.lock();<a name="line.673"></a>
-<span class="sourceLineNo">674</span>    try {<a name="line.674"></a>
-<span class="sourceLineNo">675</span>      // Wait for the sync to be completed<a name="line.675"></a>
-<span class="sourceLineNo">676</span>      while (true) {<a name="line.676"></a>
-<span class="sourceLineNo">677</span>        if (!isRunning()) {<a name="line.677"></a>
-<span class="sourceLineNo">678</span>          throw new RuntimeException("store no longer running");<a name="line.678"></a>
-<span class="sourceLineNo">679</span>        } else if (isSyncAborted()) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span>          throw new RuntimeException("sync aborted", syncException.get());<a name="line.680"></a>
-<span class="sourceLineNo">681</span>        } else if (inSync.get()) {<a name="line.681"></a>
-<span class="sourceLineNo">682</span>          syncCond.await();<a name="line.682"></a>
-<span class="sourceLineNo">683</span>        } else if (slotIndex &gt;= syncMaxSlot) {<a name="line.683"></a>
-<span class="sourceLineNo">684</span>          slotCond.signal();<a name="line.684"></a>
-<span class="sourceLineNo">685</span>          syncCond.await();<a name="line.685"></a>
-<span class="sourceLineNo">686</span>        } else {<a name="line.686"></a>
-<span class="sourceLineNo">687</span>          break;<a name="line.687"></a>
-<span class="sourceLineNo">688</span>        }<a name="line.688"></a>
-<span class="sourceLineNo">689</span>      }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>      final long pushSyncId = syncId.get();<a name="line.691"></a>
-<span class="sourceLineNo">692</span>      updateStoreTracker(type, procId, subProcIds);<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      slots[slotIndex++] = slot;<a name="line.693"></a>
-<span class="sourceLineNo">694</span>      logId = flushLogId;<a name="line.694"></a>
-<span class="sourceLineNo">695</span><a name="line.695"></a>
-<span class="sourceLineNo">696</span>      // Notify that there is new data<a name="line.696"></a>
-<span class="sourceLineNo">697</span>      if (slotIndex == 1) {<a name="line.697"></a>
-<span class="sourceLineNo">698</span>        waitCond.signal();<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      }<a name="line.699"></a>
-<span class="sourceLineNo">700</span><a name="line.700"></a>
-<span class="sourceLineNo">701</span>      // Notify that the slots are full<a name="line.701"></a>
-<span class="sourceLineNo">702</span>      if (slotIndex == syncMaxSlot) {<a name="line.702"></a>
-<span class="sourceLineNo">703</span>        waitCond.signal();<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        slotCond.signal();<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span><a name="line.706"></a>
-<span class="sourceLineNo">707</span>      while (pushSyncId == syncId.get() &amp;&amp; isRunning()) {<a name="line.707"></a>
-<span class="sourceLineNo">708</span>        syncCond.await();<a name="line.708"></a>
-<span class="sourceLineNo">709</span>      }<a name="line.709"></a>
-<span class="sourceLineNo">710</span>    } catch (InterruptedException e) {<a name="line.710"></a>
-<span class="sourceLineNo">711</span>      Thread.currentThread().interrupt();<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      sendAbortProcessSignal();<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      throw new RuntimeException(e);<a name="line.713"></a>
-<span class="sourceLineNo">714</span>    } finally {<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      lock.unlock();<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      if (isSyncAborted()) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span>        throw new RuntimeException("sync aborted", syncException.get());<a name="line.717"></a>
-<span class="sourceLineNo">718</span>      }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>    }<a name="line.719"></a>
-<span class="sourceLineNo">720</span>    return logId;<a name="line.720"></a>
-<span class="sourceLineNo">721</span>  }<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>  private void updateStoreTracker(final PushType type,<a name="line.723"></a>
-<span class="sourceLineNo">724</span>      final long procId, final long[] subProcIds) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span>    switch (type) {<a name="line.725"></a>
-<span class="sourceLineNo">726</span>      case INSERT:<a name="line.726"></a>
-<span class="sourceLineNo">727</span>        if (subProcIds == null) {<a name="line.727"></a>
-<span class="sourceLineNo">728</span>          storeTracker.insert(procId);<a name="line.728"></a>
-<span class="sourceLineNo">729</span>        } else if (procId == Procedure.NO_PROC_ID) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span>          storeTracker.insert(subProcIds);<a name="line.730"></a>
-<span class="sourceLineNo">731</span>        } else {<a name="line.731"></a>
-<span class="sourceLineNo">732</span>          storeTracker.insert(procId, subProcIds);<a name="line.732"></a>
-<span class="sourceLineNo">733</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.733"></a>
-<span class="sourceLineNo">734</span>        }<a name="line.734"></a>
-<span class="sourceLineNo">735</span>        break;<a name="line.735"></a>
-<span class="sourceLineNo">736</span>      case UPDATE:<a name="line.736"></a>
-<span class="sourceLineNo">737</span>        storeTracker.update(procId);<a name="line.737"></a>
-<span class="sourceLineNo">738</span>        holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.738"></a>
-<span class="sourceLineNo">739</span>        break;<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      case DELETE:<a name="line.740"></a>
-<span class="sourceLineNo">741</span>        if (subProcIds != null &amp;&amp; subProcIds.length &gt; 0) {<a name="line.741"></a>
-<span class="sourceLineNo">742</span>          storeTracker.delete(subProcIds);<a name="line.742"></a>
-<span class="sourceLineNo">743</span>          holdingCleanupTracker.setDeletedIfSet(subProcIds);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>        } else {<a name="line.744"></a>
-<span class="sourceLineNo">745</span>          storeTracker.delete(procId);<a name="line.745"></a>
-<span class="sourceLineNo">746</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.746"></a>
-<span class="sourceLineNo">747</span>        }<a name="line.747"></a>
-<span class="sourceLineNo">748</span>        break;<a name="line.748"></a>
-<span class="sourceLineNo">749</span>      default:<a name="line.749"></a>
-<span class="sourceLineNo">750</span>        throw new RuntimeException("invalid push type " + type);<a name="line.750"></a>
-<span class="sourceLineNo">751</span>    }<a name="line.751"></a>
-<span class="sourceLineNo">752</span>  }<a name="line.752"></a>
-<span class="sourceLineNo">753</span><a name="line.753"></a>
-<span class="sourceLineNo">754</span>  private boolean isSyncAborted() {<a name="line.754"></a>
-<span class="sourceLineNo">755</span>    return syncException.get() != null;<a name="line.755"></a>
-<span class="sourceLineNo">756</span>  }<a name="line.756"></a>
-<span class="sourceLineNo">757</span><a name="line.757"></a>
-<span class="sourceLineNo">758</span>  private void syncLoop() throws Throwable {<a name="line.758"></a>
-<span class="sourceLineNo">759</span>    long totalSyncedToStore = 0;<a name="line.759"></a>
-<span class="sourceLineNo">760</span>    inSync.set(false);<a name="line.760"></a>
-<span class="sourceLineNo">761</span>    lock.lock();<a name="line.761"></a>
-<span class="sourceLineNo">762</span>    try {<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (isRunning()) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        try {<a name="line.764"></a>
-<span class="sourceLineNo">765</span>          // Wait until new data is available<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          if (slotIndex == 0) {<a name="line.766"></a>
-<span class="sourceLineNo">767</span>            if (!loading.get()) {<a name="line.767"></a>
-<span class="sourceLineNo">768</span>              periodicRoll();<a name="line.768"></a>
-<span class="sourceLineNo">769</span>            }<a name="line.769"></a>
-<span class="sourceLineNo">770</span><a name="line.770"></a>
-<span class="sourceLineNo">771</span>            if (LOG.isTraceEnabled()) {<a name="line.771"></a>
-<span class="sourceLineNo">772</span>              float rollTsSec = getMillisFromLastRoll() / 1000.0f;<a name="line.772"></a>
-<span class="sourceLineNo">773</span>              LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)",<a name="line.773"></a>
-<span class="sourceLineNo">774</span>                        StringUtils.humanSize(totalSynced.get()),<a name="line.774"></a>
-<span class="sourceLineNo">775</span>                        StringUtils.humanSize(totalSynced.get() / rollTsSec)));<a name="line.775"></a>
+<span class="sourceLineNo">663</span>  private void releaseSlot(final ByteSlot slot) {<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    slot.reset();<a name="line.664"></a>
+<span class="sourceLineNo">665</span>    slotsCache.offer(slot);<a name="line.665"></a>
+<span class="sourceLineNo">666</span>  }<a name="line.666"></a>
+<span class="sourceLineNo">667</span><a name="line.667"></a>
+<span class="sourceLineNo">668</span>  private enum PushType { INSERT, UPDATE, DELETE }<a name="line.668"></a>
+<span class="sourceLineNo">669</span><a name="line.669"></a>
+<span class="sourceLineNo">670</span>  private long pushData(final PushType type, final ByteSlot slot,<a name="line.670"></a>
+<span class="sourceLineNo">671</span>      final long procId, final long[] subProcIds) {<a name="line.671"></a>
+<span class="sourceLineNo">672</span>    if (!isRunning()) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>      throw new RuntimeException("the store must be running before inserting data");<a name="line.673"></a>
+<span class="sourceLineNo">674</span>    }<a name="line.674"></a>
+<span class="sourceLineNo">675</span>    if (logs.isEmpty()) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span>      throw new RuntimeException("recoverLease() must be called before inserting data");<a name="line.676"></a>
+<span class="sourceLineNo">677</span>    }<a name="line.677"></a>
+<span class="sourceLineNo">678</span><a name="line.678"></a>
+<span class="sourceLineNo">679</span>    long logId = -1;<a name="line.679"></a>
+<span class="sourceLineNo">680</span>    lock.lock();<a name="line.680"></a>
+<span class="sourceLineNo">681</span>    try {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>      // Wait for the sync to be completed<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      while (true) {<a name="line.683"></a>
+<span class="sourceLineNo">684</span>        if (!isRunning()) {<a name="line.684"></a>
+<span class="sourceLineNo">685</span>          throw new RuntimeException("store no longer running");<a name="line.685"></a>
+<span class="sourceLineNo">686</span>        } else if (isSyncAborted()) {<a name="line.686"></a>
+<span class="sourceLineNo">687</span>          throw new RuntimeException("sync aborted", syncException.get());<a name="line.687"></a>
+<span class="sourceLineNo">688</span>        } else if (inSync.get()) {<a name="line.688"></a>
+<span class="sourceLineNo">689</span>          syncCond.await();<a name="line.689"></a>
+<span class="sourceLineNo">690</span>        } else if (slotIndex &gt;= syncMaxSlot) {<a name="line.690"></a>
+<span class="sourceLineNo">691</span>          slotCond.signal();<a name="line.691"></a>
+<span class="sourceLineNo">692</span>          syncCond.await();<a name="line.692"></a>
+<span class="sourceLineNo">693</span>        } else {<a name="line.693"></a>
+<span class="sourceLineNo">694</span>          break;<a name="line.694"></a>
+<span class="sourceLineNo">695</span>        }<a name="line.695"></a>
+<span class="sourceLineNo">696</span>      }<a name="line.696"></a>
+<span class="sourceLineNo">697</span><a name="line.697"></a>
+<span class="sourceLineNo">698</span>      final long pushSyncId = syncId.get();<a name="line.698"></a>
+<span class="sourceLineNo">699</span>      updateStoreTracker(type, procId, subProcIds);<a name="line.699"></a>
+<span class="sourceLineNo">700</span>      slots[slotIndex++] = slot;<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      logId = flushLogId;<a name="line.701"></a>
+<span class="sourceLineNo">702</span><a name="line.702"></a>
+<span class="sourceLineNo">703</span>      // Notify that there is new data<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (slotIndex == 1) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        waitCond.signal();<a name="line.705"></a>
+<span class="sourceLineNo">706</span>      }<a name="line.706"></a>
+<span class="sourceLineNo">707</span><a name="line.707"></a>
+<span class="sourceLineNo">708</span>      // Notify that the slots are full<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      if (slotIndex == syncMaxSlot) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        waitCond.signal();<a name="line.710"></a>
+<span class="sourceLineNo">711</span>        slotCond.signal();<a name="line.711"></a>
+<span class="sourceLineNo">712</span>      }<a name="line.712"></a>
+<span class="sourceLineNo">713</span><a name="line.713"></a>
+<span class="sourceLineNo">714</span>      while (pushSyncId == syncId.get() &amp;&amp; isRunning()) {<a name="line.714"></a>
+<span class="sourceLineNo">715</span>        syncCond.await();<a name="line.715"></a>
+<span class="sourceLineNo">716</span>      }<a name="line.716"></a>
+<span class="sourceLineNo">717</span>    } catch (InterruptedException e) {<a name="line.717"></a>
+<span class="sourceLineNo">718</span>      Thread.currentThread().interrupt();<a name="line.718"></a>
+<span class="sourceLineNo">719</span>      sendAbortProcessSignal();<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      throw new RuntimeException(e);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>    } finally {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>      lock.unlock();<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      if (isSyncAborted()) {<a name="line.723"></a>
+<span class="sourceLineNo">724</span>        throw new RuntimeException("sync aborted", syncException.get());<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      }<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>    return logId;<a name="line.727"></a>
+<span class="sourceLineNo">728</span>  }<a name="line.728"></a>
+<span class="sourceLineNo">729</span><a name="line.729"></a>
+<span class="sourceLineNo">730</span>  private void updateStoreTracker(final PushType type,<a name="line.730"></a>
+<span class="sourceLineNo">731</span>      final long procId, final long[] subProcIds) {<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    switch (type) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span>      case INSERT:<a name="line.733"></a>
+<span class="sourceLineNo">734</span>        if (subProcIds == null) {<a name="line.734"></a>
+<span class="sourceLineNo">735</span>          storeTracker.insert(procId);<a name="line.735"></a>
+<span class="sourceLineNo">736</span>        } else if (procId == Procedure.NO_PROC_ID) {<a name="line.736"></a>
+<span class="sourceLineNo">737</span>          storeTracker.insert(subProcIds);<a name="line.737"></a>
+<span class="sourceLineNo">738</span>        } else {<a name="line.738"></a>
+<span class="sourceLineNo">739</span>          storeTracker.insert(procId, subProcIds);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.740"></a>
+<span class="sourceLineNo">741</span>        }<a name="line.741"></a>
+<span class="sourceLineNo">742</span>        break;<a name="line.742"></a>
+<span class="sourceLineNo">743</span>      case UPDATE:<a name="line.743"></a>
+<span class="sourceLineNo">744</span>        storeTracker.update(procId);<a name="line.744"></a>
+<span class="sourceLineNo">745</span>        holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.745"></a>
+<span class="sourceLineNo">746</span>        break;<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      case DELETE:<a name="line.747"></a>
+<span class="sourceLineNo">748</span>        if (subProcIds != null &amp;&amp; subProcIds.length &gt; 0) {<a name="line.748"></a>
+<span class="sourceLineNo">749</span>          storeTracker.delete(subProcIds);<a name="line.749"></a>
+<span class="sourceLineNo">750</span>          holdingCleanupTracker.setDeletedIfSet(subProcIds);<a name="line.750"></a>
+<span class="sourceLineNo">751</span>        } else {<a name="line.751"></a>
+<span class="sourceLineNo">752</span>          storeTracker.delete(procId);<a name="line.752"></a>
+<span class="sourceLineNo">753</span>          holdingCleanupTracker.setDeletedIfSet(procId);<a name="line.753"></a>
+<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
+<span class="sourceLineNo">755</span>        break;<a name="line.755"></a>
+<span class="sourceLineNo">756</span>      default:<a name="line.756"></a>
+<span class="sourceLineNo">757</span>        throw new RuntimeException("invalid push type " + type);<a name="line.757"></a>
+<span class="sourceLineNo">758</span>    }<a name="line.758"></a>
+<span class="sourceLineNo">759</span>  }<a name="line.759"></a>
+<span class="sourceLineNo">760</span><a name="line.760"></a>
+<span class="sourceLineNo">761</span>  private boolean isSyncAborted() {<a name="line.761"></a>
+<span class="sourceLineNo">762</span>    return syncException.get() != null;<a name="line.762"></a>
+<span class="sourceLineNo">763</span>  }<a name="line.763"></a>
+<span class="sourceLineNo">764</span><a name="line.764"></a>
+<span class="sourceLineNo">765</span>  private void syncLoop() throws Throwable {<a name="line.765"></a>
+<span class="sourceLineNo">766</span>    long totalSyncedToStore = 0;<a name="line.766"></a>
+<span class="sourceLineNo">767</span>    inSync.set(false);<a name="line.767"></a>
+<span class="sourceLineNo">768</span>    lock.lock();<a name="line.768"></a>
+<span class="sourceLineNo">769</span>    try {<a name="line.769"></a>
+<span class="sourceLineNo">770</span>      while (isRunning()) {<a name="line.770"></a>
+<span class="sourceLineNo">771</span>        try {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>          // Wait until new data is available<a name="line.772"></a>
+<span class="sourceLineNo">773</span>          if (slotIndex == 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>            if (!loading.get()) {<a name="line.774"></a>
+<span class="sourceLineNo">775</span>              periodicRoll();<a name="line.775"></a>
 <span class="sourceLineNo">776</span>            }<a name="line.776"></a>
 <span class="sourceLineNo">777</span><a name="line.777"></a>
-<span class="sourceLineNo">778</span>            waitCond.await(getMillisToNextPeriodicRoll(), TimeUnit.MILLISECONDS);<a name="line.778"></a>
-<span class="sourceLineNo">779</span>            if (slotIndex == 0) {<a name="line.779"></a>
-<span class="sourceLineNo">780</span>              // no data.. probably a stop() or a periodic roll<a name="line.780"></a>
-<span class="sourceLineNo">781</span>              continue;<a name="line.781"></a>
-<span class="sourceLineNo">782</span>            }<a name="line.782"></a>
-<span class="sourceLineNo">783</span>          }<a name="line.783"></a>
-<span class="sourceLineNo">784</span>          // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing<a name="line.784"></a>
-<span class="sourceLineNo">785</span>          syncMaxSlot = runningProcCount;<a name="line.785"></a>
-<span class="sourceLineNo">786</span>          assert syncMaxSlot &gt; 0 : "unexpected syncMaxSlot=" + syncMaxSlot;<a name="line.786"></a>
-<span class="sourceLineNo">787</span>          final long syncWaitSt = System.currentTimeMillis();<a name="line.787"></a>
-<span class="sourceLineNo">788</span>          if (slotIndex != syncMaxSlot) {<a name="line.788"></a>
-<span class="sourceLineNo">789</span>            slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);<a name="line.789"></a>
+<span class="sourceLineNo">778</span>            if (LOG.isTraceEnabled()) {<a name="line.778"></a>
+<span class="sourceLineNo">779</span>              float rollTsSec = getMillisFromLastRoll() / 1000.0f;<a name="line.779"></a>
+<span class="sourceLineNo">780</span>              LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)",<a name="line.780"></a>
+<span class="sourceLineNo">781</span>                        StringUtils.humanSize(totalSynced.get()),<a name="line.781"></a>
+<span class="sourceLineNo">782</span>                        StringUtils.humanSize(totalSynced.get() / rollTsSec)));<a name="line.782"></a>
+<span class="sourceLineNo">783</span>            }<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>            waitCond.await(getMillisToNextPeriodicRoll(), TimeUnit.MILLISECONDS);<a name="line.785"></a>
+<span class="sourceLineNo">786</span>            if (slotIndex == 0) {<a name="line.786"></a>
+<span class="sourceLineNo">787</span>              // no data.. probably a stop() or a periodic roll<a name="line.787"></a>
+<span class="sourceLineNo">788</span>              continue;<a name="line.788"></a>
+<span class="sourceLineNo">789</span>            }<a name="line.789"></a>
 <span class="sourceLineNo">790</span>          }<a name="line.790"></a>
-<span class="sourceLineNo">791</span><a name="line.791"></a>
-<span class="sourceLineNo">792</span>          final long currentTs = System.currentTimeMillis();<a name="line.792"></a>
-<span class="sourceLineNo">793</span>          final long syncWaitMs = currentTs - syncWaitSt;<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          final float rollSec = getMillisFromLastRoll() / 1000.0f;<a name="line.794"></a>
-<span class="sourceLineNo">795</span>          final float syncedPerSec = totalSyncedToStore / rollSec;<a name="line.795"></a>
-<span class="sourceLineNo">796</span>          if (LOG.isTraceEnabled() &amp;&amp; (syncWaitMs &gt; 10 || slotIndex &lt; syncMaxSlot)) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span>            LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)",<a name="line.797"></a>
-<span class="sourceLineNo">798</span>                      StringUtils.humanTimeDiff(syncWaitMs), slotIndex,<a name="line.798"></a>
-<span class="sourceLineNo">799</span>                      StringUtils.humanSize(totalSyncedToStore),<a name="line.799"></a>
-<span class="sourceLineNo">800</span>                      StringUtils.humanSize(syncedPerSec)));<a name="line.800"></a>
-<span class="sourceLineNo">801</span>          }<a name="line.801"></a>
-<span class="sourceLineNo">802</span><a name="line.802"></a>
-<span class="sourceLineNo">803</span>          // update webui circular buffers (TODO: get rid of allocations)<a name="line.803"></a>
-<span class="sourceLineNo">804</span>          final SyncMetrics syncMetrics = new SyncMetrics();<a name="line.804"></a>
-<span class="sourceLineNo">805</span>          syncMetrics.timestamp = currentTs;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>          syncMetrics.syncWaitMs = syncWaitMs;<a name="line.806"></a>
-<span class="sourceLineNo">807</span>          syncMetrics.syncedEntries = slotIndex;<a name="line.807"></a>
-<span class="sourceLineNo">808</span>          syncMetrics.totalSyncedBytes = totalSyncedToStore;<a name="line.808"></a>
-<span class="sourceLineNo">809</span>          syncMetrics.syncedPerSec = syncedPerSec;<a name="line.809"></a>
-<span class="sourceLineNo">810</span>          syncMetricsQueue.add(syncMetrics);<a name="line.810"></a>
-<span class="sourceLineNo">811</span><a name="line.811"></a>
-<span class="sourceLineNo">812</span>          // sync<a name="line.812"></a>
-<span class="sourceLineNo">813</span>          inSync.set(true);<a name="line.813"></a>
-<span class="sourceLineNo">814</span>          long slotSize = syncSlots();<a name="line.814"></a>
-<span class="sourceLineNo">815</span>          logs.getLast().addToSize(slotSize);<a name="line.815"></a>
-<span class="sourceLineNo">816</span>          totalSyncedToStore = totalSynced.addAndGet(slotSize);<a name="line.816"></a>
-<span class="sourceLineNo">817</span>          slotIndex = 0;<a name="line.817"></a>
-<span class="sourceLineNo">818</span>          inSync.set(false);<a name="line.818"></a>
-<span class="sourceLineNo">819</span>          syncId.incrementAndGet();<a name="line.819"></a>
-<span class="sourceLineNo">820</span>        } catch (InterruptedException e) {<a name="line.820"></a>
-<span class="sourceLineNo">821</span>          Thread.currentThread().interrupt();<a name="line.821"></a>
-<span class="sourceLineNo">822</span>          syncException.compareAndSet(null, e);<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          sendAbortProcessSignal();<a name="line.823"></a>
-<span class="sourceLineNo">824</span>          throw e;<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        } catch (Throwable t) {<a name="line.825"></a>
-<span class="sourceLineNo">826</span>          syncException.compareAndSet(null, t);<a name="line.826"></a>
-<span class="sourceLineNo">827</span>          sendAbortProcessSignal();<a name="line.827"></a>
-<span class="sourceLineNo">828</span>          throw t;<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        } finally {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          syncCond.signalAll();<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span>    } finally {<a name="line.833"></a>
-<span class="sourceLineNo">834</span>      lock.unlock();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span>  }<a name="line.836"></a>
-<span class="sourceLineNo">837</span><a name="line.837"></a>
-<span class="sourceLineNo">838</span>  public ArrayList&lt;SyncMetrics&gt; getSyncMetrics() {<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    lock.lock();<a name="line.839"></a>
-<span class="sourceLineNo">840</span>    try {<a name="line.840"></a>
-<span class="sourceLineNo">841</span>      return new ArrayList&lt;&gt;(syncMetricsQueue);<a name="line.841"></a>
-<span class="sourceLineNo">842</span>    } finally {<a name="line.842"></a>
-<span class="sourceLineNo">843</span>      lock.unlock();<a name="line.843"></a>
-<span class="sourceLineNo">844</span>    }<a name="line.844"></a>
-<span class="sourceLineNo">845</span>  }<a name="line.845"></a>
-<span class="sourceLineNo">846</span><a name="line.846"></a>
-<span class="sourceLineNo">847</span>  private long syncSlots() throws Throwable {<a name="line.847"></a>
-<span class="sourceLineNo">848</span>    int retry = 0;<a name="line.848"></a>
-<span class="sourceLineNo">849</span>    int logRolled = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span>    long totalSynced = 0;<a name="line.850"></a>
-<span class="sourceLineNo">851</span>    do {<a name="line.851"></a>
-<span class="sourceLineNo">852</span>      try {<a name="line.852"></a>
-<span class="sourceLineNo">853</span>        totalSynced = syncSlots(stream, slots, 0, slotIndex);<a name="line.853"></a>
-<span class="sourceLineNo">854</span>        break;<a name="line.854"></a>
-<span class="sourceLineNo">855</span>      } catch (Throwable e) {<a name="line.855"></a>
-<span class="sourceLineNo">856</span>        LOG.warn("unable to sync slots, retry=" + retry);<a name="line.856"></a>
-<span class="sourceLineNo">857</span>        if (++retry &gt;= maxRetriesBeforeRoll) {<a name="line.857"></a>
-<span class="sourceLineNo">858</span>          if (logRolled &gt;= maxSyncFailureRoll &amp;&amp; isRunning()) {<a name="line.858"></a>
-<span class="sourceLineNo">859</span>            LOG.error("Sync slots after log roll failed, abort.", e);<a name="line.859"></a>
-<span class="sourceLineNo">860</span>            throw e;<a name="line.860"></a>
-<span class="sourceLineNo">861</span>          }<a name="line.861"></a>
-<span class="sourceLineNo">862</span><a name="line.862"></a>
-<span class="sourceLineNo">863</span>          if (!rollWriterWithRetries()) {<a name="line.863"></a>
-<span class="sourceLineNo">864</span>            throw e;<a name="line.864"></a>
-<span class="sourceLineNo">865</span>          }<a name="line.865"></a>
-<span class="sourceLineNo">866</span><a name="line.866"></a>
-<span class="sourceLineNo">867</span>          logRolled++;<a name="line.867"></a>
-<span class="sourceLineNo">868</span>          retry = 0;<a name="line.868"></a>
-<span class="sourceLineNo">869</span>        }<a name="line.869"></a>
-<span class="sourceLineNo">870</span>      }<a name="line.870"></a>
-<span class="sourceLineNo">871</span>    } while (isRunning());<a name="line.871"></a>
-<span class="sourceLineNo">872</span>    return totalSynced;<a name="line.872"></a>
-<span class="sourceLineNo">873</span>  }<a name="line.873"></a>
-<span class="sourceLineNo">874</span><a name="line.874"></a>
-<span class="sourceLineNo">875</span>  protected long syncSlots(final FSDataOutputStream stream, final ByteSlot[] slots,<a name="line.875"></a>
-<span class="sourceLineNo">876</span>      final int offset, final int count) throws IOException {<a name="line.876"></a>
-<span class="sourceLineNo">877</span>    long totalSynced = 0;<a name="line.877"></a>
-<span class="sourceLineNo">878</span>    for (int i = 0; i &lt; count; ++i) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span>      final ByteSlot data = slots[offset + i];<a name="line.879"></a>
-<span class="sourceLineNo">880</span>      data.writeTo(stream);<a name="line.880"></a>
-<span class="sourceLineNo">881</span>      totalSynced += data.size();<a name="line.881"></a>
-<span class="sourceLineNo">882</span>    }<a name="line.882"></a>
-<span class="sourceLineNo">883</span><a name="line.883"></a>
-<span class="sourceLineNo">884</span>    syncStream(stream);<a name="line.884"></a>
-<span class="sourceLineNo">885</span>    sendPostSyncSignal();<a name="line.885"></a>
-<span class="sourceLineNo">886</span><a name="line.886"></a>
-<span class="sourceLineNo">887</span>    if (LOG.isTraceEnabled()) {<a name="line.887"></a>
-<span class="sourceLineNo">888</span>      LOG.trace("Sync slots=" + count + '/' + syncMaxSlot +<a name="line.888"></a>
-<span class="sourceLineNo">889</span>                ", flushed=" + StringUtils.humanSize(totalSynced));<a name="line.889"></a>
-<span class="sourceLineNo">890</span>    }<a name="line.890"></a>
-<span class="sourceLineNo">891</span>    return totalSynced;<a name="line.891"></a>
-<span class="sourceLineNo">892</span>  }<a name="line.892"></a>
+<span class="sourceLineNo">791</span>          // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing<a name="line.791"></a>
+<span class="sourceLineNo">792</span>          syncMaxSlot = runningProcCount;<a name="line.792"></a>
+<span class="sourceLineNo">793</span>          assert syncMaxSlot &gt; 0 : "unexpected syncMaxSlot=" + syncMaxSlot;<a name="line.793"></a>
+<span class="sourceLineNo">794</span>          final long syncWaitSt = System.currentTimeMillis();<a name="line.794"></a>
+<span class="sourceLineNo">795</span>          if (slotIndex != syncMaxSlot) {<a name="line.795"></a>
+<span class="sourceLineNo">796</span>            slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);<a name="line.796"></a>
+<span class="sourceLineNo">797</span>          }<a name="line.797"></a>
+<span class="sourceLineNo">798</span><a name="line.798"></a>
+<span class="sourceLineNo">799</span>          final long currentTs = System.currentTimeMillis();<a name="line.799"></a>
+<span class="sourceLineNo">800</span>          final long syncWaitMs = currentTs - syncWaitSt;<a name="line.800"></a>
+<span class="sourceLineNo">801</span>          final float rollSec = getMillisFromLastRoll() / 1000.0f;<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          final float syncedPerSec = totalSyncedToStore / rollSec;<a name="line.802"></a>
+<span class="sourceLineNo">803</span>          if (LOG.isTraceEnabled() &amp;&amp; (syncWaitMs &gt; 10 || slotIndex &lt; syncMaxSlot)) {<a name="line.803"></a>
+<span class="sourceLineNo">804</span>            LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)",<a name="line.804"></a>
+<span class="sourceLineNo">805</span>                      StringUtils.humanTimeDiff(syncWaitMs), slotIndex,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>                      StringUtils.humanSize(totalSyncedToStore),<a name="line.806"></a>
+<span class="sourceLineNo">807</span>                      StringUtils.humanSize(syncedPerSec)));<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          }<a name="line.808"></a>
+<span class="sourceLineNo">809</span><a name="line.809"></a>
+<span class="sourceLineNo">810</span>          // update webui circular buffers (TODO: get rid of allocations)<a name="line.810"></a>
+<span class="sourceLineNo">811</span>          final SyncMetrics syncMetrics = new SyncMetrics();<a name="line.811"></a>
+<span class="sourceLineNo">812</span>          syncMetrics.timestamp = currentTs;<a name="line.812"></a>
+<span class="sourceLineNo">813</span>          syncMetrics.syncWaitMs = syncWaitMs;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>          syncMetrics.syncedEntries = slotIndex;<a name="line.814"></a>
+<span class="sourceLineNo">815</span>          syncMetrics.totalSyncedBytes = totalSyncedToStore;<a name="line.815"></a>
+<span class="sourceLineNo">816</span>          syncMetrics.syncedPerSec = syncedPerSec;<a name="line.816"></a>
+<span class="sourceLineNo">817</span>          syncMetricsQueue.add(syncMetrics);<a name="line.817"></a>
+<span class="sourceLineNo">818</span><a name="line.818"></a>
+<span class="sourceLineNo">819</span>          // sync<a name="line.819"></a>
+<span class="sourceLineNo">820</span>          inSync.set(true);<a name="line.820"></a>
+<span class="sourceLineNo">821</span>          long slotSize = syncSlots();<a name="line.821"></a>
+<span class="sourceLineNo">822</span>          logs.getLast().addToSize(slotSize);<a name="line.822"></a>
+<span class="sourceLineNo">823</span>          totalSyncedToStore = totalSynced.addAndGet(slotSize);<a name="line.823"></a>
+<span class="sourceLineNo">824</span>          slotIndex = 0;<a name="line.824"></a>
+<span class="sourceLineNo">825</span>          inSync.set(false);<a name="line.825"></a>
+<span class="sourceLineNo">826</span>          syncId.incrementAndGet();<a name="line.826"></a>
+<span class="sourceLineNo">827</span>        } catch (InterruptedException e) {<a name="line.827"></a>
+<span class="sourceLineNo">828</span>          Thread.currentThread().interrupt();<a name="line.828"></a>
+<span class="sourceLineNo">829</span>          syncException.compareAndSet(null, e);<a name="line.829"></a>
+<span class="sourceLineNo">830</span>          sendAbortProcessSignal();<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          throw e;<a name="line.831"></a>
+<span class="sourceLineNo">832</span>        } catch (Throwable t) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>          syncException.compareAndSet(null, t);<a name="line.833"></a>
+<span class="sourceLineNo">834</span>          sendAbortProcessSignal();<a name="line.834"></a>
+<span class="sourceLineNo">835</span>          throw t;<a name="line.835"></a>
+<span class="sourceLineNo">836</span>        } finally {<a name="line.836"></a>
+<span class="sourceLineNo">837</span>          syncCond.signalAll();<a name="line.837"></a>
+<span class="sourceLineNo">838</span>        }<a name="line.838"></a>
+<span class="sourceLineNo">839</span>      }<a name="line.839"></a>
+<span class="sourceLineNo">840</span>    } finally {<a name="line.840"></a>
+<span class="sourceLineNo">841</span>      lock.unlock();<a name="line.841"></a>
+<span class="sourceLineNo">842</span>    }<a name="line.842"></a>
+<span class="sourceLineNo">843</span>  }<a name="line.843"></a>
+<span class="sourceLineNo">844</span><a name="line.844"></a>
+<span class="sourceLineNo">845</span>  public ArrayList&lt;SyncMetrics&gt; getSyncMetrics() {<a name="line.845"></a>
+<span class="sourceLineNo">846</span>    lock.lock();<a name="line.846"></a>
+<span class="sourceLineNo">847</span>    try {<a name="line.847"></a>
+<span class="sourceLineNo">848</span>      return new ArrayList&lt;&gt;(syncMetricsQueue);<a name="line.848"></a>
+<span class="sourceLineNo">849</span>    } finally {<a name="line.849"></a>
+<span class="sourceLineNo">850</span>      lock.unlock();<a name="line.850"></a>
+<span class="sourceLineNo">851</span>    }<a name="line.851"></a>
+<span class="sourceLineNo">852</span>  }<a name="line.852"></a>
+<span class="sourceLineNo">853</span><a name="line.853"></a>
+<span class="sourceLineNo">854</span>  private long syncSlots() throws Throwable {<a name="line.854"></a>
+<span class="sourceLineNo">855</span>    int retry = 0;<a name="line.855"></a>
+<span class="sourceLineNo">856</span>    int logRolled = 0;<a name="line.856"></a>
+<span class="sourceLineNo">857</span>    long totalSynced = 0;<a name="line.857"></a>
+<span class="sourceLineNo">858</span>    do {<a name="line.858"></a>
+<span class="sourceLineNo">859</span>      try {<a name="line.859"></a>
+<span class="sourceLineNo">860</span>        totalSynced = syncSlots(stream, slots, 0, slotIndex);<a name="line.860"></a>
+<span class="sourceLineNo">861</span>        break;<a name="line.861"></a>
+<span class="sourceLineNo">862</span>      } catch (Throwable e) {<a name="line.862"></a>
+<span class="sourceLineNo">863</span>        LOG.warn("unable to sync slots, retry=" + retry);<a name="line.863"></a>
+<span class="sourceLineNo">864</span>        if (++retry &gt;= maxRetriesBeforeRoll) {<a name="line.864"></a>
+<span class="sourceLineNo">865</span>          if (logRolled &gt;= maxSyncFailureRoll &amp;&amp; isRunning()) {<a name="line.865"></a>
+<span class="sourceLineNo">866</span>            LOG.error("Sync slots after log roll failed, abort.", e);<a name="line.866"></a>
+<span class="sourceLineNo">867</span>            throw e;<a name="line.867"></a>
+<span class="sourceLineNo">868</span>          }<a name="line.868"></a>
+<span class="sourceLineNo">869</span><a name="line.869"></a>
+<span class="sourceLineNo">870</span>          if (!rollWriterWithRetries()) {<a name="line.870"></a>
+<span class="sourceLineNo">871</span>            throw e;<a name="line.871"></a>
+<span class="sourceLineNo">872</span>          }<a name="line.872"></a>
+<span class="sourceLineNo">873</span><a name="line.873"></a>
+<span class="sourceLineNo">874</span>          logRolled++;<a name="line.874"></a>
+<span class="sourceLineNo">875</span>          retry = 0;<a name="line.875"></a>
+<span class="sourceLineNo">876</span>        }<a name="line.876"></a>
+<span class="sourceLineNo">877</span>      }<a name="line.877"></a>
+<span class="sourceLineNo">878</span>    } while (isRunning());<a name="line.878"></a>
+<span class="sourceLineNo">879</span>    return totalSynced;<a name="line.879"></a>
+<span class="sourceLineNo">880</span>  }<a name="line.880"></a>
+<span class="sourceLineNo">881</span><a name="line.881"></a>
+<span class="sourceLineNo">882</span>  protected long syncSlots(final FSDataOutputStream stream, final ByteSlot[] slots,<a name="line.882"></a>
+<span class="sourceLineNo">883</span>      final int offset, final int count) throws IOException {<a name="line.883"></a>
+<span class="sourceLineNo">884</span>    long totalSynced = 0;<a name="line.884"></a>
+<span class="sourceLineNo">885</span>    for (int i = 0; i &lt; count; ++i) {<a name="line.885"></a>
+<span class="sourceLineNo">886</span>      final ByteSlot data = slots[offset + i];<a name="line.886"></a>
+<span class="sourceLineNo">887</span>      data.writeTo(stream);<a name="line.887"></a>
+<span class="sourceLineNo">888</span>      totalSynced += data.size();<a name="line.888"></a>
+<span class="sourceLineNo">889</span>    }<a name="line.889"></a>
+<span class="sourceLineNo">890</span><a name="line.890"></a>
+<span class="sourceLineNo">891</span>    syncStream(stream);<a name="line.891"></a>
+<span class="sourceLineNo">892</span>    sendPostSyncSignal();<a name="line.892"></a>
 <span class="sourceLineNo">893</span><a name="line.893"></a>
-<span class="sourceLineNo">894</span>  protected void syncStream(final FSDataOutputStream stream) throws IOException {<a name="line.894"></a>
-<span class="sourceLineNo">895</span>    if (useHsync) {<a name="line.895"></a>
-<span class="sourceLineNo">896</span>      stream.hsync();<a name="line.896"></a>
-<span class="sourceLineNo">897</span>    } else {<a name="line.897"></a>
-<span class="sourceLineNo">898</span>      stream.hflush();<a name="line.898"></a>
-<span class="sourceLineNo">899</span>    }<a name="line.899"></a>
-<span class="sourceLineNo">900</span>  }<a name="line.900"></a>
-<span class="sourceLineNo">901</span><a name="line.901"></a>
-<span class="sourceLineNo">902</span>  private boolean rollWriterWithRetries() {<a name="line.902"></a>
-<span class="sourceLineNo">903</span>    for (int i = 0; i &lt; rollRetries &amp;&amp; isRunning(); ++i) {<a name="line.903"></a>
-<span class="sourceLineNo">904</span>      if (i &gt; 0) Threads.sleepWithoutInterrupt(waitBeforeRoll * i);<a name="line.904"></a>
-<span class="sourceLineNo">905</span><a name="line.905"></a>
-<span class="sourceLineNo">906</span>      try {<a name="line.906"></a>
-<span class="sourceLineNo">907</span>        if (rollWriter()) {<a name="line.907"></a>
-<span class="sourceLineNo">908</span>          return true;<a name="line.908"></a>
-<span class="sourceLineNo">909</span>        }<a name="line.909"></a>
-<span class="sourceLineNo">910</span>      } catch (IOException e) {<a name="line.910"></a>
-<span class="sourceLineNo">911</span>        LOG.warn("Unable to roll the log, attempt=" + (i + 1), e);<a name="line.911"></a>
-<span class="sourceLineNo">912</span>      }<a name="line.912"></a>
-<span class="sourceLineNo">913</span>    }<a name="line.913"></a>
-<span class="sourceLineNo">914</span>    LOG.error(HBaseMarkers.FATAL, "Unable to roll the log");<a name="line.914"></a>
-<span class="sourceLineNo">915</span>    return false;<a name="line.915"></a>
-<span class="sourceLineNo">916</span>  }<a name="line.916"></a>
-<span class="sourceLineNo">917</span><a name="line.917"></a>
-<span class="sourceLineNo">918</span>  private boolean tryRollWriter() {<a name="line.918"></a>
-<span class="sourceLineNo">919</span>    try {<a name="line.919"></a>
-<span class="sourceLineNo">920</span>      return rollWriter();<a name="line.920"></a>
-<span class="sourceLineNo">921</span>    } catch (IOException e) {<a name="line.921"></a>
-<span class="sourceLineNo">922</span>      LOG.warn("Unable to roll the log", e);<a name="line.922"></a>
-<span class="sourceLineNo">9

<TRUNCATED>

[08/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.html
index 681e263..7cdf584 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.html
@@ -182,7 +182,7 @@
 <span class="sourceLineNo">174</span>      }<a name="line.174"></a>
 <span class="sourceLineNo">175</span>      FileStatus[] logfiles = FSUtils.listStatus(fs, logDir, filter);<a name="line.175"></a>
 <span class="sourceLineNo">176</span>      if (logfiles == null || logfiles.length == 0) {<a name="line.176"></a>
-<span class="sourceLineNo">177</span>        LOG.info(logDir + " is empty dir, no logs to split");<a name="line.177"></a>
+<span class="sourceLineNo">177</span>        LOG.info("{} dir is empty, no logs to split.", logDir);<a name="line.177"></a>
 <span class="sourceLineNo">178</span>      } else {<a name="line.178"></a>
 <span class="sourceLineNo">179</span>        Collections.addAll(fileStatus, logfiles);<a name="line.179"></a>
 <span class="sourceLineNo">180</span>      }<a name="line.180"></a>
@@ -243,426 +243,430 @@
 <span class="sourceLineNo">235</span>      PathFilter filter) throws IOException {<a name="line.235"></a>
 <span class="sourceLineNo">236</span>    MonitoredTask status = TaskMonitor.get().createStatus("Doing distributed log split in " +<a name="line.236"></a>
 <span class="sourceLineNo">237</span>      logDirs + " for serverName=" + serverNames);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    status.setStatus("Checking directory contents...");<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.240"></a>
-<span class="sourceLineNo">241</span>    LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.241"></a>
-<span class="sourceLineNo">242</span>      " for " + serverNames);<a name="line.242"></a>
-<span class="sourceLineNo">243</span>    long t = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
-<span class="sourceLineNo">244</span>    long totalSize = 0;<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    TaskBatch batch = new TaskBatch();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    for (FileStatus lf : logfiles) {<a name="line.246"></a>
-<span class="sourceLineNo">247</span>      // TODO If the log file is still being written to - which is most likely<a name="line.247"></a>
-<span class="sourceLineNo">248</span>      // the case for the last log file - then its length will show up here<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      // as zero. The size of such a file can only be retrieved after<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      // recover-lease is done. totalSize will be under in most cases and the<a name="line.250"></a>
-<span class="sourceLineNo">251</span>      // metrics that it drives will also be under-reported.<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      totalSize += lf.getLen();<a name="line.252"></a>
-<span class="sourceLineNo">253</span>      String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.253"></a>
-<span class="sourceLineNo">254</span>      if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.254"></a>
-<span class="sourceLineNo">255</span>        throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.255"></a>
-<span class="sourceLineNo">256</span>      }<a name="line.256"></a>
-<span class="sourceLineNo">257</span>    }<a name="line.257"></a>
-<span class="sourceLineNo">258</span>    waitForSplittingCompletion(batch, status);<a name="line.258"></a>
-<span class="sourceLineNo">259</span><a name="line.259"></a>
-<span class="sourceLineNo">260</span>    if (batch.done != batch.installed) {<a name="line.260"></a>
-<span class="sourceLineNo">261</span>      batch.isDead = true;<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.262"></a>
-<span class="sourceLineNo">263</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.263"></a>
-<span class="sourceLineNo">264</span>          + " but only " + batch.done + " done");<a name="line.264"></a>
-<span class="sourceLineNo">265</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      status.abort(msg);<a name="line.266"></a>
-<span class="sourceLineNo">267</span>      throw new IOException(msg);<a name="line.267"></a>
-<span class="sourceLineNo">268</span>    }<a name="line.268"></a>
-<span class="sourceLineNo">269</span>    for (Path logDir : logDirs) {<a name="line.269"></a>
-<span class="sourceLineNo">270</span>      status.setStatus("Cleaning up log directory...");<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      try {<a name="line.272"></a>
-<span class="sourceLineNo">273</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.273"></a>
-<span class="sourceLineNo">274</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.274"></a>
-<span class="sourceLineNo">275</span>        }<a name="line.275"></a>
-<span class="sourceLineNo">276</span>      } catch (IOException ioe) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.277"></a>
-<span class="sourceLineNo">278</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.278"></a>
-<span class="sourceLineNo">279</span>          LOG.warn("Returning success without actually splitting and "<a name="line.279"></a>
-<span class="sourceLineNo">280</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.280"></a>
-<span class="sourceLineNo">281</span>              + Arrays.toString(files), ioe);<a name="line.281"></a>
-<span class="sourceLineNo">282</span>        } else {<a name="line.282"></a>
-<span class="sourceLineNo">283</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.283"></a>
-<span class="sourceLineNo">284</span>        }<a name="line.284"></a>
-<span class="sourceLineNo">285</span>      }<a name="line.285"></a>
-<span class="sourceLineNo">286</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.286"></a>
-<span class="sourceLineNo">287</span>    }<a name="line.287"></a>
-<span class="sourceLineNo">288</span>    String msg =<a name="line.288"></a>
-<span class="sourceLineNo">289</span>        "finished splitting (more than or equal to) " + totalSize + " bytes in " + batch.installed<a name="line.289"></a>
-<span class="sourceLineNo">290</span>            + " log files in " + logDirs + " in "<a name="line.290"></a>
-<span class="sourceLineNo">291</span>            + (EnvironmentEdgeManager.currentTime() - t) + "ms";<a name="line.291"></a>
-<span class="sourceLineNo">292</span>    status.markComplete(msg);<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    LOG.info(msg);<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    return totalSize;<a name="line.294"></a>
-<span class="sourceLineNo">295</span>  }<a name="line.295"></a>
-<span class="sourceLineNo">296</span><a name="line.296"></a>
-<span class="sourceLineNo">297</span>  /**<a name="line.297"></a>
-<span class="sourceLineNo">298</span>   * Add a task entry to coordination if it is not already there.<a name="line.298"></a>
-<span class="sourceLineNo">299</span>   * @param taskname the path of the log to be split<a name="line.299"></a>
-<span class="sourceLineNo">300</span>   * @param batch the batch this task belongs to<a name="line.300"></a>
-<span class="sourceLineNo">301</span>   * @return true if a new entry is created, false if it is already there.<a name="line.301"></a>
-<span class="sourceLineNo">302</span>   */<a name="line.302"></a>
-<span class="sourceLineNo">303</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.303"></a>
-<span class="sourceLineNo">304</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.304"></a>
-<span class="sourceLineNo">305</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.305"></a>
-<span class="sourceLineNo">306</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.306"></a>
-<span class="sourceLineNo">307</span>    if (oldtask == null) {<a name="line.307"></a>
-<span class="sourceLineNo">308</span>      // publish the task in the coordination engine<a name="line.308"></a>
-<span class="sourceLineNo">309</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.309"></a>
-<span class="sourceLineNo">310</span>      return true;<a name="line.310"></a>
-<span class="sourceLineNo">311</span>    }<a name="line.311"></a>
-<span class="sourceLineNo">312</span>    return false;<a name="line.312"></a>
-<span class="sourceLineNo">313</span>  }<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.315"></a>
-<span class="sourceLineNo">316</span>    synchronized (batch) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.317"></a>
-<span class="sourceLineNo">318</span>        try {<a name="line.318"></a>
-<span class="sourceLineNo">319</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.319"></a>
-<span class="sourceLineNo">320</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.320"></a>
-<span class="sourceLineNo">321</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.321"></a>
-<span class="sourceLineNo">322</span>          int actual = activeTasks(batch);<a name="line.322"></a>
-<span class="sourceLineNo">323</span>          if (remaining != actual) {<a name="line.323"></a>
-<span class="sourceLineNo">324</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.324"></a>
-<span class="sourceLineNo">325</span>          }<a name="line.325"></a>
-<span class="sourceLineNo">326</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.326"></a>
-<span class="sourceLineNo">327</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.327"></a>
-<span class="sourceLineNo">328</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.328"></a>
-<span class="sourceLineNo">329</span>                + remainingTasks);<a name="line.329"></a>
-<span class="sourceLineNo">330</span>          }<a name="line.330"></a>
-<span class="sourceLineNo">331</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span>            LOG.warn("No more task remaining, splitting "<a name="line.332"></a>
-<span class="sourceLineNo">333</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.333"></a>
-<span class="sourceLineNo">334</span>                + ", active tasks in map " + actual);<a name="line.334"></a>
-<span class="sourceLineNo">335</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.335"></a>
-<span class="sourceLineNo">336</span>              return;<a name="line.336"></a>
-<span class="sourceLineNo">337</span>            }<a name="line.337"></a>
-<span class="sourceLineNo">338</span>          }<a name="line.338"></a>
-<span class="sourceLineNo">339</span>          batch.wait(100);<a name="line.339"></a>
-<span class="sourceLineNo">340</span>          if (server.isStopped()) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.341"></a>
-<span class="sourceLineNo">342</span>            return;<a name="line.342"></a>
-<span class="sourceLineNo">343</span>          }<a name="line.343"></a>
-<span class="sourceLineNo">344</span>        } catch (InterruptedException e) {<a name="line.344"></a>
-<span class="sourceLineNo">345</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.345"></a>
-<span class="sourceLineNo">346</span>          Thread.currentThread().interrupt();<a name="line.346"></a>
-<span class="sourceLineNo">347</span>          return;<a name="line.347"></a>
-<span class="sourceLineNo">348</span>        }<a name="line.348"></a>
-<span class="sourceLineNo">349</span>      }<a name="line.349"></a>
-<span class="sourceLineNo">350</span>    }<a name="line.350"></a>
-<span class="sourceLineNo">351</span>  }<a name="line.351"></a>
-<span class="sourceLineNo">352</span><a name="line.352"></a>
-<span class="sourceLineNo">353</span>  @VisibleForTesting<a name="line.353"></a>
-<span class="sourceLineNo">354</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.354"></a>
-<span class="sourceLineNo">355</span>    return tasks;<a name="line.355"></a>
-<span class="sourceLineNo">356</span>  }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span>  private int activeTasks(final TaskBatch batch) {<a name="line.358"></a>
-<span class="sourceLineNo">359</span>    int count = 0;<a name="line.359"></a>
-<span class="sourceLineNo">360</span>    for (Task t : tasks.values()) {<a name="line.360"></a>
-<span class="sourceLineNo">361</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.361"></a>
-<span class="sourceLineNo">362</span>        count++;<a name="line.362"></a>
-<span class="sourceLineNo">363</span>      }<a name="line.363"></a>
-<span class="sourceLineNo">364</span>    }<a name="line.364"></a>
-<span class="sourceLineNo">365</span>    return count;<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span>  }<a name="line.367"></a>
-<span class="sourceLineNo">368</span><a name="line.368"></a>
-<span class="sourceLineNo">369</span>  /**<a name="line.369"></a>
-<span class="sourceLineNo">370</span>   * @param path<a name="line.370"></a>
-<span class="sourceLineNo">371</span>   * @param batch<a name="line.371"></a>
-<span class="sourceLineNo">372</span>   * @return null on success, existing task on error<a name="line.372"></a>
-<span class="sourceLineNo">373</span>   */<a name="line.373"></a>
-<span class="sourceLineNo">374</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span>    Task oldtask;<a name="line.375"></a>
-<span class="sourceLineNo">376</span>    // batch.installed is only changed via this function and<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    // a single thread touches batch.installed.<a name="line.377"></a>
-<span class="sourceLineNo">378</span>    Task newtask = new Task();<a name="line.378"></a>
-<span class="sourceLineNo">379</span>    newtask.batch = batch;<a name="line.379"></a>
-<span class="sourceLineNo">380</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    if (oldtask == null) {<a name="line.381"></a>
-<span class="sourceLineNo">382</span>      batch.installed++;<a name="line.382"></a>
-<span class="sourceLineNo">383</span>      return null;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    }<a name="line.384"></a>
-<span class="sourceLineNo">385</span>    // new task was not used.<a name="line.385"></a>
-<span class="sourceLineNo">386</span>    synchronized (oldtask) {<a name="line.386"></a>
-<span class="sourceLineNo">387</span>      if (oldtask.isOrphan()) {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>        if (oldtask.status == SUCCESS) {<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          // The task is already done. Do not install the batch for this<a name="line.389"></a>
-<span class="sourceLineNo">390</span>          // task because it might be too late for setDone() to update<a name="line.390"></a>
-<span class="sourceLineNo">391</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.391"></a>
-<span class="sourceLineNo">392</span>          // this task to complete.<a name="line.392"></a>
-<span class="sourceLineNo">393</span>          return (null);<a name="line.393"></a>
-<span class="sourceLineNo">394</span>        }<a name="line.394"></a>
-<span class="sourceLineNo">395</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.395"></a>
-<span class="sourceLineNo">396</span>          oldtask.batch = batch;<a name="line.396"></a>
-<span class="sourceLineNo">397</span>          batch.installed++;<a name="line.397"></a>
-<span class="sourceLineNo">398</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.398"></a>
-<span class="sourceLineNo">399</span>          return null;<a name="line.399"></a>
-<span class="sourceLineNo">400</span>        }<a name="line.400"></a>
-<span class="sourceLineNo">401</span>        while (oldtask.status == FAILURE) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.402"></a>
-<span class="sourceLineNo">403</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.403"></a>
-<span class="sourceLineNo">404</span>          try {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>            oldtask.wait();<a name="line.405"></a>
-<span class="sourceLineNo">406</span>          } catch (InterruptedException e) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span>            Thread.currentThread().interrupt();<a name="line.407"></a>
-<span class="sourceLineNo">408</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.408"></a>
-<span class="sourceLineNo">409</span>            // fall through to return failure<a name="line.409"></a>
-<span class="sourceLineNo">410</span>            break;<a name="line.410"></a>
-<span class="sourceLineNo">411</span>          }<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        }<a name="line.412"></a>
-<span class="sourceLineNo">413</span>        if (oldtask.status != DELETED) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span>          LOG.warn("Failure because previously failed task"<a name="line.414"></a>
-<span class="sourceLineNo">415</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.415"></a>
-<span class="sourceLineNo">416</span>          return oldtask;<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        }<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        // reinsert the newTask and it must succeed this time<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        if (t == null) {<a name="line.420"></a>
-<span class="sourceLineNo">421</span>          batch.installed++;<a name="line.421"></a>
-<span class="sourceLineNo">422</span>          return null;<a name="line.422"></a>
-<span class="sourceLineNo">423</span>        }<a name="line.423"></a>
-<span class="sourceLineNo">424</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        assert false : "Deleted task still present in tasks map";<a name="line.425"></a>
-<span class="sourceLineNo">426</span>        return t;<a name="line.426"></a>
-<span class="sourceLineNo">427</span>      }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.428"></a>
-<span class="sourceLineNo">429</span>      return oldtask;<a name="line.429"></a>
-<span class="sourceLineNo">430</span>    }<a name="line.430"></a>
-<span class="sourceLineNo">431</span>  }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span>  public void stop() {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (choreService != null) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      choreService.shutdown();<a name="line.435"></a>
-<span class="sourceLineNo">436</span>    }<a name="line.436"></a>
-<span class="sourceLineNo">437</span>    if (timeoutMonitor != null) {<a name="line.437"></a>
-<span class="sourceLineNo">438</span>      timeoutMonitor.cancel(true);<a name="line.438"></a>
-<span class="sourceLineNo">439</span>    }<a name="line.439"></a>
-<span class="sourceLineNo">440</span>  }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>  void handleDeadWorker(ServerName workerName) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.443"></a>
-<span class="sourceLineNo">444</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.444"></a>
-<span class="sourceLineNo">445</span>    synchronized (deadWorkersLock) {<a name="line.445"></a>
-<span class="sourceLineNo">446</span>      if (deadWorkers == null) {<a name="line.446"></a>
-<span class="sourceLineNo">447</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>      }<a name="line.448"></a>
-<span class="sourceLineNo">449</span>      deadWorkers.add(workerName);<a name="line.449"></a>
-<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    LOG.info("dead splitlog worker " + workerName);<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  }<a name="line.452"></a>
-<span class="sourceLineNo">453</span><a name="line.453"></a>
-<span class="sourceLineNo">454</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.454"></a>
-<span class="sourceLineNo">455</span>    synchronized (deadWorkersLock) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span>      if (deadWorkers == null) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      }<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      deadWorkers.addAll(serverNames);<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.461"></a>
-<span class="sourceLineNo">462</span>  }<a name="line.462"></a>
-<span class="sourceLineNo">463</span><a name="line.463"></a>
-<span class="sourceLineNo">464</span>  /**<a name="line.464"></a>
-<span class="sourceLineNo">465</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.465"></a>
-<span class="sourceLineNo">466</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.466"></a>
-<span class="sourceLineNo">467</span>   * &lt;p&gt;<a name="line.467"></a>
-<span class="sourceLineNo">468</span>   * All access is synchronized.<a name="line.468"></a>
-<span class="sourceLineNo">469</span>   */<a name="line.469"></a>
-<span class="sourceLineNo">470</span>  @InterfaceAudience.Private<a name="line.470"></a>
-<span class="sourceLineNo">471</span>  public static class TaskBatch {<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    public int installed = 0;<a name="line.472"></a>
-<span class="sourceLineNo">473</span>    public int done = 0;<a name="line.473"></a>
-<span class="sourceLineNo">474</span>    public int error = 0;<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    public volatile boolean isDead = false;<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span>    @Override<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    public String toString() {<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.479"></a>
-<span class="sourceLineNo">480</span>    }<a name="line.480"></a>
-<span class="sourceLineNo">481</span>  }<a name="line.481"></a>
-<span class="sourceLineNo">482</span><a name="line.482"></a>
-<span class="sourceLineNo">483</span>  /**<a name="line.483"></a>
-<span class="sourceLineNo">484</span>   * in memory state of an active task.<a name="line.484"></a>
-<span class="sourceLineNo">485</span>   */<a name="line.485"></a>
-<span class="sourceLineNo">486</span>  @InterfaceAudience.Private<a name="line.486"></a>
-<span class="sourceLineNo">487</span>  public static class Task {<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    public volatile long last_update;<a name="line.488"></a>
-<span class="sourceLineNo">489</span>    public volatile int last_version;<a name="line.489"></a>
-<span class="sourceLineNo">490</span>    public volatile ServerName cur_worker_name;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>    public volatile TaskBatch batch;<a name="line.491"></a>
-<span class="sourceLineNo">492</span>    public volatile TerminationStatus status;<a name="line.492"></a>
-<span class="sourceLineNo">493</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.493"></a>
-<span class="sourceLineNo">494</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    public volatile boolean resubmitThresholdReached;<a name="line.495"></a>
-<span class="sourceLineNo">496</span><a name="line.496"></a>
-<span class="sourceLineNo">497</span>    @Override<a name="line.497"></a>
-<span class="sourceLineNo">498</span>    public String toString() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.499"></a>
-<span class="sourceLineNo">500</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.500"></a>
-<span class="sourceLineNo">501</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.501"></a>
-<span class="sourceLineNo">502</span>    }<a name="line.502"></a>
-<span class="sourceLineNo">503</span><a name="line.503"></a>
-<span class="sourceLineNo">504</span>    public Task() {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      last_version = -1;<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      status = IN_PROGRESS;<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      setUnassigned();<a name="line.507"></a>
-<span class="sourceLineNo">508</span>    }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span>    public boolean isOrphan() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      return (batch == null || batch.isDead);<a name="line.511"></a>
+<span class="sourceLineNo">238</span>    long totalSize = 0;<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    TaskBatch batch = null;<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    long startTime = 0;<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.241"></a>
+<span class="sourceLineNo">242</span>    if (logfiles.length != 0) {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>      status.setStatus("Checking directory contents...");<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.245"></a>
+<span class="sourceLineNo">246</span>          " for " + serverNames);<a name="line.246"></a>
+<span class="sourceLineNo">247</span>      startTime = EnvironmentEdgeManager.currentTime();<a name="line.247"></a>
+<span class="sourceLineNo">248</span>      batch = new TaskBatch();<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      for (FileStatus lf : logfiles) {<a name="line.249"></a>
+<span class="sourceLineNo">250</span>        // TODO If the log file is still being written to - which is most likely<a name="line.250"></a>
+<span class="sourceLineNo">251</span>        // the case for the last log file - then its length will show up here<a name="line.251"></a>
+<span class="sourceLineNo">252</span>        // as zero. The size of such a file can only be retrieved after<a name="line.252"></a>
+<span class="sourceLineNo">253</span>        // recover-lease is done. totalSize will be under in most cases and the<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        // metrics that it drives will also be under-reported.<a name="line.254"></a>
+<span class="sourceLineNo">255</span>        totalSize += lf.getLen();<a name="line.255"></a>
+<span class="sourceLineNo">256</span>        String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.257"></a>
+<span class="sourceLineNo">258</span>          throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.258"></a>
+<span class="sourceLineNo">259</span>        }<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      }<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      waitForSplittingCompletion(batch, status);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    if (batch != null &amp;&amp; batch.done != batch.installed) {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      batch.isDead = true;<a name="line.265"></a>
+<span class="sourceLineNo">266</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.267"></a>
+<span class="sourceLineNo">268</span>          + " but only " + batch.done + " done");<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.269"></a>
+<span class="sourceLineNo">270</span>      status.abort(msg);<a name="line.270"></a>
+<span class="sourceLineNo">271</span>      throw new IOException(msg);<a name="line.271"></a>
+<span class="sourceLineNo">272</span>    }<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    for (Path logDir : logDirs) {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      status.setStatus("Cleaning up log directory...");<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.275"></a>
+<span class="sourceLineNo">276</span>      try {<a name="line.276"></a>
+<span class="sourceLineNo">277</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.277"></a>
+<span class="sourceLineNo">278</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.278"></a>
+<span class="sourceLineNo">279</span>        }<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      } catch (IOException ioe) {<a name="line.280"></a>
+<span class="sourceLineNo">281</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.281"></a>
+<span class="sourceLineNo">282</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.282"></a>
+<span class="sourceLineNo">283</span>          LOG.warn("Returning success without actually splitting and "<a name="line.283"></a>
+<span class="sourceLineNo">284</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.284"></a>
+<span class="sourceLineNo">285</span>              + Arrays.toString(files), ioe);<a name="line.285"></a>
+<span class="sourceLineNo">286</span>        } else {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.287"></a>
+<span class="sourceLineNo">288</span>        }<a name="line.288"></a>
+<span class="sourceLineNo">289</span>      }<a name="line.289"></a>
+<span class="sourceLineNo">290</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.290"></a>
+<span class="sourceLineNo">291</span>    }<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    String msg = "Finished splitting (more than or equal to) " + totalSize +<a name="line.292"></a>
+<span class="sourceLineNo">293</span>        " bytes in " + ((batch == null)? 0: batch.installed) +<a name="line.293"></a>
+<span class="sourceLineNo">294</span>        " log files in " + logDirs + " in " +<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        ((startTime == 0)? startTime: (EnvironmentEdgeManager.currentTime() - startTime)) + "ms";<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    status.markComplete(msg);<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    LOG.info(msg);<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    return totalSize;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
+<span class="sourceLineNo">300</span><a name="line.300"></a>
+<span class="sourceLineNo">301</span>  /**<a name="line.301"></a>
+<span class="sourceLineNo">302</span>   * Add a task entry to coordination if it is not already there.<a name="line.302"></a>
+<span class="sourceLineNo">303</span>   * @param taskname the path of the log to be split<a name="line.303"></a>
+<span class="sourceLineNo">304</span>   * @param batch the batch this task belongs to<a name="line.304"></a>
+<span class="sourceLineNo">305</span>   * @return true if a new entry is created, false if it is already there.<a name="line.305"></a>
+<span class="sourceLineNo">306</span>   */<a name="line.306"></a>
+<span class="sourceLineNo">307</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.307"></a>
+<span class="sourceLineNo">308</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.309"></a>
+<span class="sourceLineNo">310</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.310"></a>
+<span class="sourceLineNo">311</span>    if (oldtask == null) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span>      // publish the task in the coordination engine<a name="line.312"></a>
+<span class="sourceLineNo">313</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.313"></a>
+<span class="sourceLineNo">314</span>      return true;<a name="line.314"></a>
+<span class="sourceLineNo">315</span>    }<a name="line.315"></a>
+<span class="sourceLineNo">316</span>    return false;<a name="line.316"></a>
+<span class="sourceLineNo">317</span>  }<a name="line.317"></a>
+<span class="sourceLineNo">318</span><a name="line.318"></a>
+<span class="sourceLineNo">319</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    synchronized (batch) {<a name="line.320"></a>
+<span class="sourceLineNo">321</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.321"></a>
+<span class="sourceLineNo">322</span>        try {<a name="line.322"></a>
+<span class="sourceLineNo">323</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.323"></a>
+<span class="sourceLineNo">324</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.324"></a>
+<span class="sourceLineNo">325</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.325"></a>
+<span class="sourceLineNo">326</span>          int actual = activeTasks(batch);<a name="line.326"></a>
+<span class="sourceLineNo">327</span>          if (remaining != actual) {<a name="line.327"></a>
+<span class="sourceLineNo">328</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.328"></a>
+<span class="sourceLineNo">329</span>          }<a name="line.329"></a>
+<span class="sourceLineNo">330</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.330"></a>
+<span class="sourceLineNo">331</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.331"></a>
+<span class="sourceLineNo">332</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.332"></a>
+<span class="sourceLineNo">333</span>                + remainingTasks);<a name="line.333"></a>
+<span class="sourceLineNo">334</span>          }<a name="line.334"></a>
+<span class="sourceLineNo">335</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.335"></a>
+<span class="sourceLineNo">336</span>            LOG.warn("No more task remaining, splitting "<a name="line.336"></a>
+<span class="sourceLineNo">337</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.337"></a>
+<span class="sourceLineNo">338</span>                + ", active tasks in map " + actual);<a name="line.338"></a>
+<span class="sourceLineNo">339</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.339"></a>
+<span class="sourceLineNo">340</span>              return;<a name="line.340"></a>
+<span class="sourceLineNo">341</span>            }<a name="line.341"></a>
+<span class="sourceLineNo">342</span>          }<a name="line.342"></a>
+<span class="sourceLineNo">343</span>          batch.wait(100);<a name="line.343"></a>
+<span class="sourceLineNo">344</span>          if (server.isStopped()) {<a name="line.344"></a>
+<span class="sourceLineNo">345</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.345"></a>
+<span class="sourceLineNo">346</span>            return;<a name="line.346"></a>
+<span class="sourceLineNo">347</span>          }<a name="line.347"></a>
+<span class="sourceLineNo">348</span>        } catch (InterruptedException e) {<a name="line.348"></a>
+<span class="sourceLineNo">349</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.349"></a>
+<span class="sourceLineNo">350</span>          Thread.currentThread().interrupt();<a name="line.350"></a>
+<span class="sourceLineNo">351</span>          return;<a name="line.351"></a>
+<span class="sourceLineNo">352</span>        }<a name="line.352"></a>
+<span class="sourceLineNo">353</span>      }<a name="line.353"></a>
+<span class="sourceLineNo">354</span>    }<a name="line.354"></a>
+<span class="sourceLineNo">355</span>  }<a name="line.355"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span>  @VisibleForTesting<a name="line.357"></a>
+<span class="sourceLineNo">358</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.358"></a>
+<span class="sourceLineNo">359</span>    return tasks;<a name="line.359"></a>
+<span class="sourceLineNo">360</span>  }<a name="line.360"></a>
+<span class="sourceLineNo">361</span><a name="line.361"></a>
+<span class="sourceLineNo">362</span>  private int activeTasks(final TaskBatch batch) {<a name="line.362"></a>
+<span class="sourceLineNo">363</span>    int count = 0;<a name="line.363"></a>
+<span class="sourceLineNo">364</span>    for (Task t : tasks.values()) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.365"></a>
+<span class="sourceLineNo">366</span>        count++;<a name="line.366"></a>
+<span class="sourceLineNo">367</span>      }<a name="line.367"></a>
+<span class="sourceLineNo">368</span>    }<a name="line.368"></a>
+<span class="sourceLineNo">369</span>    return count;<a name="line.369"></a>
+<span class="sourceLineNo">370</span><a name="line.370"></a>
+<span class="sourceLineNo">371</span>  }<a name="line.371"></a>
+<span class="sourceLineNo">372</span><a name="line.372"></a>
+<span class="sourceLineNo">373</span>  /**<a name="line.373"></a>
+<span class="sourceLineNo">374</span>   * @param path<a name="line.374"></a>
+<span class="sourceLineNo">375</span>   * @param batch<a name="line.375"></a>
+<span class="sourceLineNo">376</span>   * @return null on success, existing task on error<a name="line.376"></a>
+<span class="sourceLineNo">377</span>   */<a name="line.377"></a>
+<span class="sourceLineNo">378</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.378"></a>
+<span class="sourceLineNo">379</span>    Task oldtask;<a name="line.379"></a>
+<span class="sourceLineNo">380</span>    // batch.installed is only changed via this function and<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    // a single thread touches batch.installed.<a name="line.381"></a>
+<span class="sourceLineNo">382</span>    Task newtask = new Task();<a name="line.382"></a>
+<span class="sourceLineNo">383</span>    newtask.batch = batch;<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.384"></a>
+<span class="sourceLineNo">385</span>    if (oldtask == null) {<a name="line.385"></a>
+<span class="sourceLineNo">386</span>      batch.installed++;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      return null;<a name="line.387"></a>
+<span class="sourceLineNo">388</span>    }<a name="line.388"></a>
+<span class="sourceLineNo">389</span>    // new task was not used.<a name="line.389"></a>
+<span class="sourceLineNo">390</span>    synchronized (oldtask) {<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      if (oldtask.isOrphan()) {<a name="line.391"></a>
+<span class="sourceLineNo">392</span>        if (oldtask.status == SUCCESS) {<a name="line.392"></a>
+<span class="sourceLineNo">393</span>          // The task is already done. Do not install the batch for this<a name="line.393"></a>
+<span class="sourceLineNo">394</span>          // task because it might be too late for setDone() to update<a name="line.394"></a>
+<span class="sourceLineNo">395</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.395"></a>
+<span class="sourceLineNo">396</span>          // this task to complete.<a name="line.396"></a>
+<span class="sourceLineNo">397</span>          return (null);<a name="line.397"></a>
+<span class="sourceLineNo">398</span>        }<a name="line.398"></a>
+<span class="sourceLineNo">399</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.399"></a>
+<span class="sourceLineNo">400</span>          oldtask.batch = batch;<a name="line.400"></a>
+<span class="sourceLineNo">401</span>          batch.installed++;<a name="line.401"></a>
+<span class="sourceLineNo">402</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.402"></a>
+<span class="sourceLineNo">403</span>          return null;<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        }<a name="line.404"></a>
+<span class="sourceLineNo">405</span>        while (oldtask.status == FAILURE) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.406"></a>
+<span class="sourceLineNo">407</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.407"></a>
+<span class="sourceLineNo">408</span>          try {<a name="line.408"></a>
+<span class="sourceLineNo">409</span>            oldtask.wait();<a name="line.409"></a>
+<span class="sourceLineNo">410</span>          } catch (InterruptedException e) {<a name="line.410"></a>
+<span class="sourceLineNo">411</span>            Thread.currentThread().interrupt();<a name="line.411"></a>
+<span class="sourceLineNo">412</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.412"></a>
+<span class="sourceLineNo">413</span>            // fall through to return failure<a name="line.413"></a>
+<span class="sourceLineNo">414</span>            break;<a name="line.414"></a>
+<span class="sourceLineNo">415</span>          }<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        }<a name="line.416"></a>
+<span class="sourceLineNo">417</span>        if (oldtask.status != DELETED) {<a name="line.417"></a>
+<span class="sourceLineNo">418</span>          LOG.warn("Failure because previously failed task"<a name="line.418"></a>
+<span class="sourceLineNo">419</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.419"></a>
+<span class="sourceLineNo">420</span>          return oldtask;<a name="line.420"></a>
+<span class="sourceLineNo">421</span>        }<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        // reinsert the newTask and it must succeed this time<a name="line.422"></a>
+<span class="sourceLineNo">423</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        if (t == null) {<a name="line.424"></a>
+<span class="sourceLineNo">425</span>          batch.installed++;<a name="line.425"></a>
+<span class="sourceLineNo">426</span>          return null;<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        }<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.428"></a>
+<span class="sourceLineNo">429</span>        assert false : "Deleted task still present in tasks map";<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        return t;<a name="line.430"></a>
+<span class="sourceLineNo">431</span>      }<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.432"></a>
+<span class="sourceLineNo">433</span>      return oldtask;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>    }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>  }<a name="line.435"></a>
+<span class="sourceLineNo">436</span><a name="line.436"></a>
+<span class="sourceLineNo">437</span>  public void stop() {<a name="line.437"></a>
+<span class="sourceLineNo">438</span>    if (choreService != null) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span>      choreService.shutdown();<a name="line.439"></a>
+<span class="sourceLineNo">440</span>    }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    if (timeoutMonitor != null) {<a name="line.441"></a>
+<span class="sourceLineNo">442</span>      timeoutMonitor.cancel(true);<a name="line.442"></a>
+<span class="sourceLineNo">443</span>    }<a name="line.443"></a>
+<span class="sourceLineNo">444</span>  }<a name="line.444"></a>
+<span class="sourceLineNo">445</span><a name="line.445"></a>
+<span class="sourceLineNo">446</span>  void handleDeadWorker(ServerName workerName) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.447"></a>
+<span class="sourceLineNo">448</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.448"></a>
+<span class="sourceLineNo">449</span>    synchronized (deadWorkersLock) {<a name="line.449"></a>
+<span class="sourceLineNo">450</span>      if (deadWorkers == null) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      }<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      deadWorkers.add(workerName);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    }<a name="line.454"></a>
+<span class="sourceLineNo">455</span>    LOG.info("Dead splitlog worker {}", workerName);<a name="line.455"></a>
+<span class="sourceLineNo">456</span>  }<a name="line.456"></a>
+<span class="sourceLineNo">457</span><a name="line.457"></a>
+<span class="sourceLineNo">458</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>    synchronized (deadWorkersLock) {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>      if (deadWorkers == null) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.461"></a>
+<span class="sourceLineNo">462</span>      }<a name="line.462"></a>
+<span class="sourceLineNo">463</span>      deadWorkers.addAll(serverNames);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    }<a name="line.464"></a>
+<span class="sourceLineNo">465</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.465"></a>
+<span class="sourceLineNo">466</span>  }<a name="line.466"></a>
+<span class="sourceLineNo">467</span><a name="line.467"></a>
+<span class="sourceLineNo">468</span>  /**<a name="line.468"></a>
+<span class="sourceLineNo">469</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.469"></a>
+<span class="sourceLineNo">470</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.470"></a>
+<span class="sourceLineNo">471</span>   * &lt;p&gt;<a name="line.471"></a>
+<span class="sourceLineNo">472</span>   * All access is synchronized.<a name="line.472"></a>
+<span class="sourceLineNo">473</span>   */<a name="line.473"></a>
+<span class="sourceLineNo">474</span>  @InterfaceAudience.Private<a name="line.474"></a>
+<span class="sourceLineNo">475</span>  public static class TaskBatch {<a name="line.475"></a>
+<span class="sourceLineNo">476</span>    public int installed = 0;<a name="line.476"></a>
+<span class="sourceLineNo">477</span>    public int done = 0;<a name="line.477"></a>
+<span class="sourceLineNo">478</span>    public int error = 0;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    public volatile boolean isDead = false;<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span>    @Override<a name="line.481"></a>
+<span class="sourceLineNo">482</span>    public String toString() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    }<a name="line.484"></a>
+<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
+<span class="sourceLineNo">486</span><a name="line.486"></a>
+<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
+<span class="sourceLineNo">488</span>   * in memory state of an active task.<a name="line.488"></a>
+<span class="sourceLineNo">489</span>   */<a name="line.489"></a>
+<span class="sourceLineNo">490</span>  @InterfaceAudience.Private<a name="line.490"></a>
+<span class="sourceLineNo">491</span>  public static class Task {<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    public volatile long last_update;<a name="line.492"></a>
+<span class="sourceLineNo">493</span>    public volatile int last_version;<a name="line.493"></a>
+<span class="sourceLineNo">494</span>    public volatile ServerName cur_worker_name;<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    public volatile TaskBatch batch;<a name="line.495"></a>
+<span class="sourceLineNo">496</span>    public volatile TerminationStatus status;<a name="line.496"></a>
+<span class="sourceLineNo">497</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.497"></a>
+<span class="sourceLineNo">498</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.498"></a>
+<span class="sourceLineNo">499</span>    public volatile boolean resubmitThresholdReached;<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span>    @Override<a name="line.501"></a>
+<span class="sourceLineNo">502</span>    public String toString() {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.503"></a>
+<span class="sourceLineNo">504</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.504"></a>
+<span class="sourceLineNo">505</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.505"></a>
+<span class="sourceLineNo">506</span>    }<a name="line.506"></a>
+<span class="sourceLineNo">507</span><a name="line.507"></a>
+<span class="sourceLineNo">508</span>    public Task() {<a name="line.508"></a>
+<span class="sourceLineNo">509</span>      last_version = -1;<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      status = IN_PROGRESS;<a name="line.510"></a>
+<span class="sourceLineNo">511</span>      setUnassigned();<a name="line.511"></a>
 <span class="sourceLineNo">512</span>    }<a name="line.512"></a>
 <span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span>    public boolean isUnassigned() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span>      return (cur_worker_name == null);<a name="line.515"></a>
+<span class="sourceLineNo">514</span>    public boolean isOrphan() {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>      return (batch == null || batch.isDead);<a name="line.515"></a>
 <span class="sourceLineNo">516</span>    }<a name="line.516"></a>
 <span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span>    public void heartbeatNoDetails(long time) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span>      last_update = time;<a name="line.519"></a>
+<span class="sourceLineNo">518</span>    public boolean isUnassigned() {<a name="line.518"></a>
+<span class="sourceLineNo">519</span>      return (cur_worker_name == null);<a name="line.519"></a>
 <span class="sourceLineNo">520</span>    }<a name="line.520"></a>
 <span class="sourceLineNo">521</span><a name="line.521"></a>
-<span class="sourceLineNo">522</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      last_version = version;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      last_update = time;<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      cur_worker_name = worker;<a name="line.525"></a>
-<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span>    public void setUnassigned() {<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      cur_worker_name = null;<a name="line.529"></a>
-<span class="sourceLineNo">530</span>      last_update = -1;<a name="line.530"></a>
-<span class="sourceLineNo">531</span>    }<a name="line.531"></a>
-<span class="sourceLineNo">532</span>  }<a name="line.532"></a>
-<span class="sourceLineNo">533</span><a name="line.533"></a>
-<span class="sourceLineNo">534</span>  /**<a name="line.534"></a>
-<span class="sourceLineNo">535</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.535"></a>
-<span class="sourceLineNo">536</span>   */<a name="line.536"></a>
-<span class="sourceLineNo">537</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.537"></a>
-<span class="sourceLineNo">538</span>    private long lastLog = 0;<a name="line.538"></a>
-<span class="sourceLineNo">539</span><a name="line.539"></a>
-<span class="sourceLineNo">540</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.540"></a>
-<span class="sourceLineNo">541</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.541"></a>
-<span class="sourceLineNo">542</span>    }<a name="line.542"></a>
+<span class="sourceLineNo">522</span>    public void heartbeatNoDetails(long time) {<a name="line.522"></a>
+<span class="sourceLineNo">523</span>      last_update = time;<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    }<a name="line.524"></a>
+<span class="sourceLineNo">525</span><a name="line.525"></a>
+<span class="sourceLineNo">526</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span>      last_version = version;<a name="line.527"></a>
+<span class="sourceLineNo">528</span>      last_update = time;<a name="line.528"></a>
+<span class="sourceLineNo">529</span>      cur_worker_name = worker;<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    }<a name="line.530"></a>
+<span class="sourceLineNo">531</span><a name="line.531"></a>
+<span class="sourceLineNo">532</span>    public void setUnassigned() {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>      cur_worker_name = null;<a name="line.533"></a>
+<span class="sourceLineNo">534</span>      last_update = -1;<a name="line.534"></a>
+<span class="sourceLineNo">535</span>    }<a name="line.535"></a>
+<span class="sourceLineNo">536</span>  }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>  /**<a name="line.538"></a>
+<span class="sourceLineNo">539</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.539"></a>
+<span class="sourceLineNo">540</span>   */<a name="line.540"></a>
+<span class="sourceLineNo">541</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.541"></a>
+<span class="sourceLineNo">542</span>    private long lastLog = 0;<a name="line.542"></a>
 <span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>    @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    protected void chore() {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.546"></a>
+<span class="sourceLineNo">544</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    }<a name="line.546"></a>
 <span class="sourceLineNo">547</span><a name="line.547"></a>
-<span class="sourceLineNo">548</span>      int resubmitted = 0;<a name="line.548"></a>
-<span class="sourceLineNo">549</span>      int unassigned = 0;<a name="line.549"></a>
-<span class="sourceLineNo">550</span>      int tot = 0;<a name="line.550"></a>
-<span class="sourceLineNo">551</span>      boolean found_assigned_task = false;<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.552"></a>
-<span class="sourceLineNo">553</span><a name="line.553"></a>
-<span class="sourceLineNo">554</span>      synchronized (deadWorkersLock) {<a name="line.554"></a>
-<span class="sourceLineNo">555</span>        localDeadWorkers = deadWorkers;<a name="line.555"></a>
-<span class="sourceLineNo">556</span>        deadWorkers = null;<a name="line.556"></a>
-<span class="sourceLineNo">557</span>      }<a name="line.557"></a>
-<span class="sourceLineNo">558</span><a name="line.558"></a>
-<span class="sourceLineNo">559</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>        String path = e.getKey();<a name="line.560"></a>
-<span class="sourceLineNo">561</span>        Task task = e.getValue();<a name="line.561"></a>
-<span class="sourceLineNo">562</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.562"></a>
-<span class="sourceLineNo">563</span>        tot++;<a name="line.563"></a>
-<span class="sourceLineNo">564</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.564"></a>
-<span class="sourceLineNo">565</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.565"></a>
-<span class="sourceLineNo">566</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.566"></a>
-<span class="sourceLineNo">567</span>        // time. If we want progress when there are no region servers then we<a name="line.567"></a>
-<span class="sourceLineNo">568</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.568"></a>
-<span class="sourceLineNo">569</span>        if (task.isUnassigned()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>          unassigned++;<a name="line.570"></a>
-<span class="sourceLineNo">571</span>          continue;<a name="line.571"></a>
-<span class="sourceLineNo">572</span>        }<a name="line.572"></a>
-<span class="sourceLineNo">573</span>        found_assigned_task = true;<a name="line.573"></a>
-<span class="sourceLineNo">574</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.575"></a>
-<span class="sourceLineNo">576</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.576"></a>
-<span class="sourceLineNo">577</span>            resubmitted++;<a name="line.577"></a>
-<span class="sourceLineNo">578</span>          } else {<a name="line.578"></a>
-<span class="sourceLineNo">579</span>            handleDeadWorker(cur_worker);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.580"></a>
-<span class="sourceLineNo">581</span>                + ", will retry.");<a name="line.581"></a>
-<span class="sourceLineNo">582</span>          }<a name="line.582"></a>
-<span class="sourceLineNo">583</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.583"></a>
-<span class="sourceLineNo">584</span>          resubmitted++;<a name="line.584"></a>
-<span class="sourceLineNo">585</span>        }<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      }<a name="line.586"></a>
-<span class="sourceLineNo">587</span>      if (tot &gt; 0) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.588"></a>
-<span class="sourceLineNo">589</span>        if (now &gt; lastLog + 5000) {<a name="line.589"></a>
-<span class="sourceLineNo">590</span>          lastLog = now;<a name="line.590"></a>
-<span class="sourceLineNo">591</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.591"></a>
-<span class="sourceLineNo">592</span>        }<a name="line.592"></a>
-<span class="sourceLineNo">593</span>      }<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      if (resubmitted &gt; 0) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.595"></a>
-<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
-<span class="sourceLineNo">597</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.598"></a>
-<span class="sourceLineNo">599</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // because a. it is very unlikely that every worker had a<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      // transient error when trying to grab the task b. if there are no<a name="line.601"></a>
-<span class="sourceLineNo">602</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.604"></a>
-<span class="sourceLineNo">605</span>      // that there is always one worker in the system<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      if (tot &gt; 0<a name="line.606"></a>
-<span class="sourceLineNo">607</span>          &amp;&amp; !found_assigned_task<a name="line.607"></a>
-<span class="sourceLineNo">608</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.608"></a>
-<span class="sourceLineNo">609</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.609"></a>
-<span class="sourceLineNo">610</span>          String key = e.getKey();<a name="line.610"></a>
-<span class="sourceLineNo">611</span>          Task task = e.getValue();<a name="line.611"></a>
-<span class="sourceLineNo">612</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.612"></a>
-<span class="sourceLineNo">613</span>          // have been asynchronously assigned. There is no locking required<a name="line.613"></a>
-<span class="sourceLineNo">614</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.614"></a>
-<span class="sourceLineNo">615</span>          // called unnecessarily for a taskpath<a name="line.615"></a>
-<span class="sourceLineNo">616</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>            // We just touch the znode to make sure its still there<a name="line.617"></a>
-<span class="sourceLineNo">618</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.618"></a>
-<span class="sourceLineNo">619</span>          }<a name="line.619"></a>
-<span class="sourceLineNo">620</span>        }<a name="line.620"></a>
-<span class="sourceLineNo">621</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.621"></a>
-<span class="sourceLineNo">622</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.623"></a>
-<span class="sourceLineNo">624</span>      }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      Set&lt;String&gt; failedDeletions =<a name="line.625"></a>
-<span class="sourceLineNo">626</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      // Retry previously failed deletes<a name="line.627"></a>
-<span class="sourceLineNo">628</span>      if (failedDeletions.size() &gt; 0) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.629"></a>
-<span class="sourceLineNo">630</span>        for (String tmpPath : tmpPaths) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>          // deleteNode is an async call<a name="line.631"></a>
-<span class="sourceLineNo">632</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.632"></a>
-<span class="sourceLineNo">633</span>        }<a name="line.633"></a>
-<span class="sourceLineNo">634</span>        failedDeletions.removeAll(tmpPaths);<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      }<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span>  }<a name="line.637"></a>
-<span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>  public enum ResubmitDirective {<a name="line.639"></a>
-<span class="sourceLineNo">640</span>    CHECK(), FORCE()<a name="line.640"></a>
+<span class="sourceLineNo">548</span>    @Override<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    protected void chore() {<a name="line.549"></a>
+<span class="sourceLineNo">550</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>      int resubmitted = 0;<a name="line.552"></a>
+<span class="sourceLineNo">553</span>      int unassigned = 0;<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      int tot = 0;<a name="line.554"></a>
+<span class="sourceLineNo">555</span>      boolean found_assigned_task = false;<a name="line.555"></a>
+<span class="sourceLineNo">556</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>      synchronized (deadWorkersLock) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>        localDeadWorkers = deadWorkers;<a name="line.559"></a>
+<span class="sourceLineNo">560</span>        deadWorkers = null;<a name="line.560"></a>
+<span class="sourceLineNo">561</span>      }<a name="line.561"></a>
+<span class="sourceLineNo">562</span><a name="line.562"></a>
+<span class="sourceLineNo">563</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.563"></a>
+<span class="sourceLineNo">564</span>        String path = e.getKey();<a name="line.564"></a>
+<span class="sourceLineNo">565</span>        Task task = e.getValue();<a name="line.565"></a>
+<span class="sourceLineNo">566</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.566"></a>
+<span class="sourceLineNo">567</span>        tot++;<a name="line.567"></a>
+<span class="sourceLineNo">568</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.568"></a>
+<span class="sourceLineNo">569</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.569"></a>
+<span class="sourceLineNo">570</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.570"></a>
+<span class="sourceLineNo">571</span>        // time. If we want progress when there are no region servers then we<a name="line.571"></a>
+<span class="sourceLineNo">572</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.572"></a>
+<span class="sourceLineNo">573</span>        if (task.isUnassigned()) {<a name="line.573"></a>
+<span class="sourceLineNo">574</span>          unassigned++;<a name="line.574"></a>
+<span class="sourceLineNo">575</span>          continue;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>        }<a name="line.576"></a>
+<span class="sourceLineNo">577</span>        found_assigned_task = true;<a name="line.577"></a>
+<span class="sourceLineNo">578</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.578"></a>
+<span class="sourceLineNo">579</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.579"></a>
+<span class="sourceLineNo">580</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.580"></a>
+<span class="sourceLineNo">581</span>            resubmitted++;<a name="line.581"></a>
+<span class="sourceLineNo">582</span>          } else {<a name="line.582"></a>
+<span class="sourceLineNo">583</span>            handleDeadWorker(cur_worker);<a name="line.583"></a>
+<span class="sourceLineNo">584</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.584"></a>
+<span class="sourceLineNo">585</span>                + ", will retry.");<a name="line.585"></a>
+<span class="sourceLineNo">586</span>          }<a name="line.586"></a>
+<span class="sourceLineNo">587</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>          resubmitted++;<a name="line.588"></a>
+<span class="sourceLineNo">589</span>        }<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      }<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      if (tot &gt; 0) {<a name="line.591"></a>
+<span class="sourceLineNo">592</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.592"></a>
+<span class="sourceLineNo">593</span>        if (now &gt; lastLog + 5000) {<a name="line.593"></a>
+<span class="sourceLineNo">594</span>          lastLog = now;<a name="line.594"></a>
+<span class="sourceLineNo">595</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.595"></a>
+<span class="sourceLineNo">596</span>        }<a name="line.596"></a>
+<span class="sourceLineNo">597</span>      }<a name="line.597"></a>
+<span class="sourceLineNo">598</span>      if (resubmitted &gt; 0) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.599"></a>
+<span class="sourceLineNo">600</span>      }<a name="line.600"></a>
+<span class="sourceLineNo">601</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.602"></a>
+<span class="sourceLineNo">603</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.603"></a>
+<span class="sourceLineNo">604</span>      // because a. it is very unlikely that every worker had a<a name="line.604"></a>
+<span class="sourceLineNo">605</span>      // transient error when trying to grab the task b. if there are no<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.608"></a>
+<span class="sourceLineNo">609</span>      // that there is always one worker in the system<a name="line.609"></a>
+<span class="sourceLineNo">610</span>      if (tot &gt; 0<a name="line.610"></a>
+<span class="sourceLineNo">611</span>          &amp;&amp; !found_assigned_task<a name="line.611"></a>
+<span class="sourceLineNo">612</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>          String key = e.getKey();<a name="line.614"></a>
+<span class="sourceLineNo">615</span>          Task task = e.getValue();<a name="line.615"></a>
+<span class="sourceLineNo">616</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.616"></a>
+<span class="sourceLineNo">617</span>          // have been asynchronously assigned. There is no locking required<a name="line.617"></a>
+<span class="sourceLineNo">618</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.618"></a>
+<span class="sourceLineNo">619</span>          // called unnecessarily for a taskpath<a name="line.619"></a>
+<span class="sourceLineNo">620</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.620"></a>
+<span class="sourceLineNo">621</span>            // We just touch the znode to make sure its still there<a name="line.621"></a>
+<span class="sourceLineNo">622</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.622"></a>
+<span class="sourceLineNo">623</span>          }<a name="line.623"></a>
+<span class="sourceLineNo">624</span>        }<a name="line.624"></a>
+<span class="sourceLineNo">625</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.625"></a>
+<span class="sourceLineNo">626</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.626"></a>
+<span class="sourceLineNo">627</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      }<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      Set&lt;String&gt; failedDeletions =<a name="line.629"></a>
+<span class="sourceLineNo">630</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.630"></a>
+<span class="sourceLineNo">631</span>      // Retry previously failed deletes<a name="line.631"></a>
+<span class="sourceLineNo">632</span>      if (failedDeletions.size() &gt; 0) {<a name="line.632"></a>
+<span class="sourceLineNo">633</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.633"></a>
+<span class="sourceLineNo">634</span>        for (String tmpPath : tmpPaths) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>          // deleteNode is an async call<a name="line.635"></a>
+<span class="sourceLineNo">636</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.636"></a>
+<span class="sourceLineNo">637</span>        }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>        failedDeletions.removeAll(tmpPaths);<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      }<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    }<a name="line.640"></a>
 <span class="sourceLineNo">641</span>  }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>  public enum TerminationStatus {<a name="line.643"></a>
-<span class="sourceLineNo">644</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.644"></a>
-<span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    final String statusMsg;<a name="line.646"></a>
-<span class="sourceLineNo">647</span><a name="line.647"></a>
-<span class="sourceLineNo">648</span>    TerminationStatus(String msg) {<a name="line.648"></a>
-<span class="sourceLineNo">649</span>      statusMsg = msg;<a name="line.649"></a>
-<span class="sourceLineNo">650</span>    }<a name="line.650"></a>
+<span class="sourceLineNo">643</span>  public enum ResubmitDirective {<a name="line.643"></a>
+<span class="sourceLineNo">644</span>    CHECK(), FORCE()<a name="line.644"></a>
+<span class="sourceLineNo">645</span>  }<a name="line.645"></a>
+<span class="sourceLineNo">646</span><a name="line.646"></a>
+<span class="sourceLineNo">647</span>  public enum TerminationStatus {<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.648"></a>
+<span class="sourceLineNo">649</span><a name="line.649"></a>
+<span class="sourceLineNo">650</span>    final String statusMsg;<a name="line.650"></a>
 <span class="sourceLineNo">651</span><a name="line.651"></a>
-<span class="sourceLineNo">652</span>    @Override<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    public String toString() {<a name="line.653"></a>
-<span class="sourceLineNo">654</span>      return statusMsg;<a name="line.654"></a>
-<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
-<span class="sourceLineNo">657</span>}<a name="line.657"></a>
+<span class="sourceLineNo">652</span>    TerminationStatus(String msg) {<a name="line.652"></a>
+<span class="sourceLineNo">653</span>      statusMsg = msg;<a name="line.653"></a>
+<span class="sourceLineNo">654</span>    }<a name="line.654"></a>
+<span class="sourceLineNo">655</span><a name="line.655"></a>
+<span class="sourceLineNo">656</span>    @Override<a name="line.656"></a>
+<span class="sourceLineNo">657</span>    public String toString() {<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      return statusMsg;<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    }<a name="line.659"></a>
+<span class="sourceLineNo">660</span>  }<a name="line.660"></a>
+<span class="sourceLineNo">661</span>}<a name="line.661"></a>
 
 
 


[15/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html
index a990153..d2da8f4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html
@@ -593,7 +593,7 @@
 <span class="sourceLineNo">585</span>              abort(error, t);<a name="line.585"></a>
 <span class="sourceLineNo">586</span>            }<a name="line.586"></a>
 <span class="sourceLineNo">587</span>          }<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        }));<a name="line.588"></a>
+<span class="sourceLineNo">588</span>        }), getName() + ":becomeActiveMaster");<a name="line.588"></a>
 <span class="sourceLineNo">589</span>      }<a name="line.589"></a>
 <span class="sourceLineNo">590</span>      // Fall in here even if we have been aborted. Need to run the shutdown services and<a name="line.590"></a>
 <span class="sourceLineNo">591</span>      // the super run call will do this for us.<a name="line.591"></a>
@@ -932,7 +932,7 @@
 <span class="sourceLineNo">924</span>          HBaseFsck.createLockRetryCounterFactory(this.conf).create());<a name="line.924"></a>
 <span class="sourceLineNo">925</span>    }<a name="line.925"></a>
 <span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span>    status.setStatus("Initialze ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
+<span class="sourceLineNo">927</span>    status.setStatus("Initialize ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
 <span class="sourceLineNo">928</span>    this.serverManager = createServerManager(this);<a name="line.928"></a>
 <span class="sourceLineNo">929</span>    createProcedureExecutor();<a name="line.929"></a>
 <span class="sourceLineNo">930</span>    @SuppressWarnings("rawtypes")<a name="line.930"></a>
@@ -954,2890 +954,2893 @@
 <span class="sourceLineNo">946</span>        .collect(Collectors.toList());<a name="line.946"></a>
 <span class="sourceLineNo">947</span>    this.assignmentManager.setupRIT(ritList);<a name="line.947"></a>
 <span class="sourceLineNo">948</span><a name="line.948"></a>
-<span class="sourceLineNo">949</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.949"></a>
-<span class="sourceLineNo">950</span>    this.regionServerTracker.start(<a name="line.950"></a>
-<span class="sourceLineNo">951</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.951"></a>
-<span class="sourceLineNo">952</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.952"></a>
-<span class="sourceLineNo">953</span>      walManager.getLiveServersFromWALDir());<a name="line.953"></a>
-<span class="sourceLineNo">954</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.954"></a>
-<span class="sourceLineNo">955</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.955"></a>
-<span class="sourceLineNo">956</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.956"></a>
-<span class="sourceLineNo">957</span>    this.tableStateManager =<a name="line.957"></a>
-<span class="sourceLineNo">958</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.958"></a>
-<span class="sourceLineNo">959</span>        ?<a name="line.959"></a>
-<span class="sourceLineNo">960</span>        new MirroringTableStateManager(this):<a name="line.960"></a>
-<span class="sourceLineNo">961</span>        new TableStateManager(this);<a name="line.961"></a>
-<span class="sourceLineNo">962</span><a name="line.962"></a>
-<span class="sourceLineNo">963</span>    status.setStatus("Initializing ZK system trackers");<a name="line.963"></a>
-<span class="sourceLineNo">964</span>    initializeZKBasedSystemTrackers();<a name="line.964"></a>
-<span class="sourceLineNo">965</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.965"></a>
-<span class="sourceLineNo">966</span>    try {<a name="line.966"></a>
-<span class="sourceLineNo">967</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.967"></a>
-<span class="sourceLineNo">968</span>    } catch (IOException e) {<a name="line.968"></a>
-<span class="sourceLineNo">969</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.969"></a>
-<span class="sourceLineNo">970</span>          + " from file system", e);<a name="line.970"></a>
-<span class="sourceLineNo">971</span>    }<a name="line.971"></a>
-<span class="sourceLineNo">972</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.972"></a>
-<span class="sourceLineNo">973</span>    this.activeMaster = true;<a name="line.973"></a>
-<span class="sourceLineNo">974</span><a name="line.974"></a>
-<span class="sourceLineNo">975</span>    // This is for backwards compatibility<a name="line.975"></a>
-<span class="sourceLineNo">976</span>    // See HBASE-11393<a name="line.976"></a>
-<span class="sourceLineNo">977</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.977"></a>
-<span class="sourceLineNo">978</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.978"></a>
-<span class="sourceLineNo">979</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.979"></a>
-<span class="sourceLineNo">980</span>    tableCFsUpdater.copyTableCFs();<a name="line.980"></a>
-<span class="sourceLineNo">981</span><a name="line.981"></a>
-<span class="sourceLineNo">982</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.982"></a>
-<span class="sourceLineNo">983</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.983"></a>
-<span class="sourceLineNo">984</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.984"></a>
-<span class="sourceLineNo">985</span>      updateConfigurationForQuotasObserver(conf);<a name="line.985"></a>
-<span class="sourceLineNo">986</span>    }<a name="line.986"></a>
-<span class="sourceLineNo">987</span>    // initialize master side coprocessors before we start handling requests<a name="line.987"></a>
-<span class="sourceLineNo">988</span>    status.setStatus("Initializing master coprocessors");<a name="line.988"></a>
-<span class="sourceLineNo">989</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.989"></a>
-<span class="sourceLineNo">990</span><a name="line.990"></a>
-<span class="sourceLineNo">991</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.991"></a>
-<span class="sourceLineNo">992</span>    InitMetaProcedure initMetaProc = null;<a name="line.992"></a>
-<span class="sourceLineNo">993</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.993"></a>
-<span class="sourceLineNo">994</span>      .isOffline()) {<a name="line.994"></a>
-<span class="sourceLineNo">995</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.995"></a>
-<span class="sourceLineNo">996</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.996"></a>
-<span class="sourceLineNo">997</span>      if (optProc.isPresent()) {<a name="line.997"></a>
-<span class="sourceLineNo">998</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.998"></a>
-<span class="sourceLineNo">999</span>      } else {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span>        initMetaProc = new InitMetaProcedure();<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span>      }<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span>    }<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1006"></a>
+<span class="sourceLineNo">949</span>    // Start RegionServerTracker with listing of servers found with exiting SCPs -- these should<a name="line.949"></a>
+<span class="sourceLineNo">950</span>    // be registered in the deadServers set -- and with the list of servernames out on the<a name="line.950"></a>
+<span class="sourceLineNo">951</span>    // filesystem that COULD BE 'alive' (we'll schedule SCPs for each and let SCP figure it out).<a name="line.951"></a>
+<span class="sourceLineNo">952</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.952"></a>
+<span class="sourceLineNo">953</span>    this.regionServerTracker.start(<a name="line.953"></a>
+<span class="sourceLineNo">954</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.954"></a>
+<span class="sourceLineNo">955</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.955"></a>
+<span class="sourceLineNo">956</span>      walManager.getLiveServersFromWALDir());<a name="line.956"></a>
+<span class="sourceLineNo">957</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.957"></a>
+<span class="sourceLineNo">958</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.958"></a>
+<span class="sourceLineNo">959</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.959"></a>
+<span class="sourceLineNo">960</span>    this.tableStateManager =<a name="line.960"></a>
+<span class="sourceLineNo">961</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.961"></a>
+<span class="sourceLineNo">962</span>        ?<a name="line.962"></a>
+<span class="sourceLineNo">963</span>        new MirroringTableStateManager(this):<a name="line.963"></a>
+<span class="sourceLineNo">964</span>        new TableStateManager(this);<a name="line.964"></a>
+<span class="sourceLineNo">965</span><a name="line.965"></a>
+<span class="sourceLineNo">966</span>    status.setStatus("Initializing ZK system trackers");<a name="line.966"></a>
+<span class="sourceLineNo">967</span>    initializeZKBasedSystemTrackers();<a name="line.967"></a>
+<span class="sourceLineNo">968</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.968"></a>
+<span class="sourceLineNo">969</span>    try {<a name="line.969"></a>
+<span class="sourceLineNo">970</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.970"></a>
+<span class="sourceLineNo">971</span>    } catch (IOException e) {<a name="line.971"></a>
+<span class="sourceLineNo">972</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.972"></a>
+<span class="sourceLineNo">973</span>          + " from file system", e);<a name="line.973"></a>
+<span class="sourceLineNo">974</span>    }<a name="line.974"></a>
+<span class="sourceLineNo">975</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.975"></a>
+<span class="sourceLineNo">976</span>    this.activeMaster = true;<a name="line.976"></a>
+<span class="sourceLineNo">977</span><a name="line.977"></a>
+<span class="sourceLineNo">978</span>    // This is for backwards compatibility<a name="line.978"></a>
+<span class="sourceLineNo">979</span>    // See HBASE-11393<a name="line.979"></a>
+<span class="sourceLineNo">980</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.980"></a>
+<span class="sourceLineNo">981</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.981"></a>
+<span class="sourceLineNo">982</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.982"></a>
+<span class="sourceLineNo">983</span>    tableCFsUpdater.copyTableCFs();<a name="line.983"></a>
+<span class="sourceLineNo">984</span><a name="line.984"></a>
+<span class="sourceLineNo">985</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.985"></a>
+<span class="sourceLineNo">986</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.986"></a>
+<span class="sourceLineNo">987</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.987"></a>
+<span class="sourceLineNo">988</span>      updateConfigurationForQuotasObserver(conf);<a name="line.988"></a>
+<span class="sourceLineNo">989</span>    }<a name="line.989"></a>
+<span class="sourceLineNo">990</span>    // initialize master side coprocessors before we start handling requests<a name="line.990"></a>
+<span class="sourceLineNo">991</span>    status.setStatus("Initializing master coprocessors");<a name="line.991"></a>
+<span class="sourceLineNo">992</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.992"></a>
+<span class="sourceLineNo">993</span><a name="line.993"></a>
+<span class="sourceLineNo">994</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.994"></a>
+<span class="sourceLineNo">995</span>    InitMetaProcedure initMetaProc = null;<a name="line.995"></a>
+<span class="sourceLineNo">996</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.996"></a>
+<span class="sourceLineNo">997</span>      .isOffline()) {<a name="line.997"></a>
+<span class="sourceLineNo">998</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.998"></a>
+<span class="sourceLineNo">999</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.999"></a>
+<span class="sourceLineNo">1000</span>      if (optProc.isPresent()) {<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span>      } else {<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span>        initMetaProc = new InitMetaProcedure();<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span>      }<a name="line.1006"></a>
 <span class="sourceLineNo">1007</span>    }<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span><a name="line.1008"></a>
-<span class="sourceLineNo">1009</span>    // initialize load balancer<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span>    this.balancer.setMasterServices(this);<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span>    this.balancer.initialize();<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span><a name="line.1013"></a>
-<span class="sourceLineNo">1014</span>    // start up all service threads.<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span>    status.setStatus("Initializing master service threads");<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span>    startServiceThreads();<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span>    // wait meta to be initialized after we start procedure executor<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span>    if (initMetaProc != null) {<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span>      initMetaProc.await();<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span>    }<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span>    tableStateManager.start();<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span>    // Wake up this server to check in<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span>    sleeper.skipSleepCycle();<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span>    // Wait for region servers to report in.<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span>    String statusStr = "Wait for region servers to report in";<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span>    status.setStatus(statusStr);<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span>    LOG.info(Objects.toString(status));<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span>    waitForRegionServers(status);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span><a name="line.1032"></a>
-<span class="sourceLineNo">1033</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span>    if (isStopped()) {<a name="line.1034"></a>
-<span class="sourceLineNo">1035</span>      return;<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span>    }<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span><a name="line.1037"></a>
-<span class="sourceLineNo">1038</span>    //Initialize after meta as it scans meta<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span>    if (favoredNodesManager != null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span>      snapshotOfRegionAssignment.initialize();<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span>    }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span><a name="line.1045"></a>
-<span class="sourceLineNo">1046</span>    // Fix up assignment manager status<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span>    status.setStatus("Starting assignment manager");<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span>    this.assignmentManager.joinCluster();<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span><a name="line.1049"></a>
-<span class="sourceLineNo">1050</span>    // set cluster status again after user regions are assigned<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1051"></a>
+<span class="sourceLineNo">1008</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span>    }<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span><a name="line.1011"></a>
+<span class="sourceLineNo">1012</span>    // initialize load balancer<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span>    this.balancer.setMasterServices(this);<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span>    this.balancer.initialize();<a name="line.1015"></a>
+<span class="sourceLineNo">1016</span><a name="line.1016"></a>
+<span class="sourceLineNo">1017</span>    // start up all service threads.<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span>    status.setStatus("Initializing master service threads");<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span>    startServiceThreads();<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span>    // wait meta to be initialized after we start procedure executor<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span>    if (initMetaProc != null) {<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span>      initMetaProc.await();<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span>    }<a name="line.1023"></a>
+<span class="sourceLineNo">1024</span>    tableStateManager.start();<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span>    // Wake up this server to check in<a name="line.1025"></a>
+<span class="sourceLineNo">1026</span>    sleeper.skipSleepCycle();<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span><a name="line.1027"></a>
+<span class="sourceLineNo">1028</span>    // Wait for region servers to report in.<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span>    String statusStr = "Wait for region servers to report in";<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span>    status.setStatus(statusStr);<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span>    LOG.info(Objects.toString(status));<a name="line.1033"></a>
+<span class="sourceLineNo">1034</span>    waitForRegionServers(status);<a name="line.1034"></a>
+<span class="sourceLineNo">1035</span><a name="line.1035"></a>
+<span class="sourceLineNo">1036</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1036"></a>
+<span class="sourceLineNo">1037</span>    if (isStopped()) {<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span>      return;<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span>    }<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span><a name="line.1040"></a>
+<span class="sourceLineNo">1041</span>    //Initialize after meta as it scans meta<a name="line.1041"></a>
+<span class="sourceLineNo">1042</span>    if (favoredNodesManager != null) {<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span>      snapshotOfRegionAssignment.initialize();<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span>    }<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span><a name="line.1048"></a>
+<span class="sourceLineNo">1049</span>    // Fix up assignment manager status<a name="line.1049"></a>
+<span class="sourceLineNo">1050</span>    status.setStatus("Starting assignment manager");<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span>    this.assignmentManager.joinCluster();<a name="line.1051"></a>
 <span class="sourceLineNo">1052</span><a name="line.1052"></a>
-<span class="sourceLineNo">1053</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span>    this.balancerChore = new BalancerChore(this);<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span>    this.serverManager.startChore();<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span>    status.setStatus("Starting cluster schema service");<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span>    initClusterSchemaService();<a name="line.1066"></a>
+<span class="sourceLineNo">1053</span>    // set cluster status again after user regions are assigned<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span><a name="line.1055"></a>
+<span class="sourceLineNo">1056</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1057"></a>
+<span class="sourceLineNo">1058</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span>    this.balancerChore = new BalancerChore(this);<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1065"></a>
+<span class="sourceLineNo">1066</span>    this.serverManager.startChore();<a name="line.1066"></a>
 <span class="sourceLineNo">1067</span><a name="line.1067"></a>
-<span class="sourceLineNo">1068</span>    if (this.cpHost != null) {<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span>      try {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span>        this.cpHost.preMasterInitialization();<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span>      } catch (IOException e) {<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span>      }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span>    }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span><a name="line.1075"></a>
-<span class="sourceLineNo">1076</span>    status.markComplete("Initialization successful");<a name="line.1076"></a>
-<span class="sourceLineNo">1077</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span>    configurationManager.registerObserver(this.balancer);<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1082"></a>
-<span class="sourceLineNo">1083</span>    // Set master as 'initialized'.<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span>    setInitialized(true);<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span>    status.setStatus("Assign meta replicas");<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span>    metaBootstrap.assignMetaReplicas();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span>    status.setStatus("Starting quota manager");<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span>    initQuotaManager();<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span>      // Create the quota snapshot notifier<a name="line.1092"></a>
-<span class="sourceLineNo">1093</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span><a name="line.1098"></a>
-<span class="sourceLineNo">1099</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1100"></a>
-<span class="sourceLineNo">1101</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span>    }<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span><a name="line.1103"></a>
-<span class="sourceLineNo">1104</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span>    // master initialization. See HBASE-5916.<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span><a name="line.1108"></a>
-<span class="sourceLineNo">1109</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span><a name="line.1112"></a>
-<span class="sourceLineNo">1113</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span>    initMobCleaner();<a name="line.1114"></a>
+<span class="sourceLineNo">1068</span>    status.setStatus("Starting cluster schema service");<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span>    initClusterSchemaService();<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span><a name="line.1070"></a>
+<span class="sourceLineNo">1071</span>    if (this.cpHost != null) {<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span>      try {<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span>        this.cpHost.preMasterInitialization();<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span>      } catch (IOException e) {<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1075"></a>
+<span class="sourceLineNo">1076</span>      }<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span>    }<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span><a name="line.1078"></a>
+<span class="sourceLineNo">1079</span>    status.markComplete("Initialization successful");<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1081"></a>
+<span class="sourceLineNo">1082</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span>    configurationManager.registerObserver(this.balancer);<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1085"></a>
+<span class="sourceLineNo">1086</span>    // Set master as 'initialized'.<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span>    setInitialized(true);<a name="line.1087"></a>
+<span class="sourceLineNo">1088</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span>    status.setStatus("Assign meta replicas");<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span>    metaBootstrap.assignMetaReplicas();<a name="line.1091"></a>
+<span class="sourceLineNo">1092</span>    status.setStatus("Starting quota manager");<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span>    initQuotaManager();<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span>      // Create the quota snapshot notifier<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1097"></a>
+<span class="sourceLineNo">1098</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1099"></a>
+<span class="sourceLineNo">1100</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span><a name="line.1101"></a>
+<span class="sourceLineNo">1102</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span>    }<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span><a name="line.1106"></a>
+<span class="sourceLineNo">1107</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1107"></a>
+<span class="sourceLineNo">1108</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span>    // master initialization. See HBASE-5916.<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span><a name="line.1111"></a>
+<span class="sourceLineNo">1112</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1112"></a>
+<span class="sourceLineNo">1113</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1114"></a>
 <span class="sourceLineNo">1115</span><a name="line.1115"></a>
-<span class="sourceLineNo">1116</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1116"></a>
-<span class="sourceLineNo">1117</span>    if (this.cpHost != null) {<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span>      // don't let cp initialization errors kill the master<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span>      try {<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span>        this.cpHost.postStartMaster();<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span>      } catch (IOException ioe) {<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span>      }<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span>    }<a name="line.1124"></a>
-<span class="sourceLineNo">1125</span><a name="line.1125"></a>
-<span class="sourceLineNo">1126</span>    zombieDetector.interrupt();<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span><a name="line.1127"></a>
-<span class="sourceLineNo">1128</span>    /*<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span>     * in activeMasterManager thread, it should be fine.<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span>     */<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span>    long start = System.currentTimeMillis();<a name="line.1132"></a>
-<span class="sourceLineNo">1133</span>    this.balancer.postMasterStartupInitialize();<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span>    if (LOG.isDebugEnabled()) {<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span>    }<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span>  }<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span><a name="line.1139"></a>
-<span class="sourceLineNo">1140</span>  /**<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span>   */<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span>  @VisibleForTesting<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span>    if (!conf.getBoolean(<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1148"></a>
-<span class="sourceLineNo">1149</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span>      return;<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span>    }<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1152"></a>
-<span class="sourceLineNo">1153</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span>    if (length &gt; 0) {<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span>    }<a name="line.1157"></a>
-<span class="sourceLineNo">1158</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1159"></a>
-<span class="sourceLineNo">1160</span>  }<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span><a name="line.1161"></a>
-<span class="sourceLineNo">1162</span>  private void initMobCleaner() {<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1163"></a>
-<span class="sourceLineNo">1164</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span><a name="line.1165"></a>
-<span class="sourceLineNo">1166</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1167"></a>
-<span class="sourceLineNo">1168</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1168"></a>
-<span class="sourceLineNo">1169</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span>    } else {<a name="line.1171"></a>
-<span class="sourceLineNo">1172</span>      LOG<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span>    }<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>  }<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span><a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>  /**<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>   * &lt;p&gt;<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>   * &lt;/p&gt;<a name="line.1181"></a>
+<span class="sourceLineNo">1116</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span>    initMobCleaner();<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span><a name="line.1118"></a>
+<span class="sourceLineNo">1119</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span>    if (this.cpHost != null) {<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span>      // don't let cp initialization errors kill the master<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span>      try {<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span>        this.cpHost.postStartMaster();<a name="line.1123"></a>
+<span class="sourceLineNo">1124</span>      } catch (IOException ioe) {<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span>      }<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span>    }<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span><a name="line.1128"></a>
+<span class="sourceLineNo">1129</span>    zombieDetector.interrupt();<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span><a name="line.1130"></a>
+<span class="sourceLineNo">1131</span>    /*<a name="line.1131"></a>
+<span class="sourceLineNo">1132</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span>     * in activeMasterManager thread, it should be fine.<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span>     */<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span>    long start = System.currentTimeMillis();<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span>    this.balancer.postMasterStartupInitialize();<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span>    if (LOG.isDebugEnabled()) {<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span>    }<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span>  }<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span><a name="line.1142"></a>
+<span class="sourceLineNo">1143</span>  /**<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span>   */<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span>  @VisibleForTesting<a name="line.1147"></a>
+<span class="sourceLineNo">1148</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span>    if (!conf.getBoolean(<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1151"></a>
+<span class="sourceLineNo">1152</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span>      return;<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span>    }<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1155"></a>
+<span class="sourceLineNo">1156</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span>    if (length &gt; 0) {<a name="line.1158"></a>
+<span class="sourceLineNo">1159</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span>    }<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1162"></a>
+<span class="sourceLineNo">1163</span>  }<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span><a name="line.1164"></a>
+<span class="sourceLineNo">1165</span>  private void initMobCleaner() {<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1166"></a>
+<span class="sourceLineNo">1167</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span><a name="line.1168"></a>
+<span class="sourceLineNo">1169</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1170"></a>
+<span class="sourceLineNo">1171</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span>    } else {<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span>      LOG<a name="line.1175"></a>
+<span class="sourceLineNo">1176</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>    }<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>  }<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span><a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>  /**<a name="line.1181"></a>
 <span class="sourceLineNo">1182</span>   * &lt;p&gt;<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>   * Will be overridden in tests.<a name="line.1183"></a>
+<span class="sourceLineNo">1183</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1183"></a>
 <span class="sourceLineNo">1184</span>   * &lt;/p&gt;<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>   */<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>  @VisibleForTesting<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>    return new MasterMetaBootstrap(this);<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>  }<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span><a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>  /**<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>   * &lt;p&gt;<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>   * Create a {@link ServerManager} instance.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>   * &lt;/p&gt;<a name="line.1196"></a>
+<span class="sourceLineNo">1185</span>   * &lt;p&gt;<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>   * Will be overridden in tests.<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>   * &lt;/p&gt;<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>   */<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>  @VisibleForTesting<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span>    return new MasterMetaBootstrap(this);<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>  }<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span><a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>  /**<a name="line.1196"></a>
 <span class="sourceLineNo">1197</span>   * &lt;p&gt;<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>   * Will be overridden in tests.<a name="line.1198"></a>
+<span class="sourceLineNo">1198</span>   * Create a {@link ServerManager} instance.<a name="line.1198"></a>
 <span class="sourceLineNo">1199</span>   * &lt;/p&gt;<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>   */<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>  @VisibleForTesting<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>    // w/ a mocked up ServerManager.<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>    setupClusterConnection();<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    return new ServerManager(master);<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span>  }<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span><a name="line.1208"></a>
-<span class="sourceLineNo">1209</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>      throws IOException, InterruptedException {<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span>    this.serverManager.waitForRegionServers(status);<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>  }<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span><a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>  // Will be overridden in tests<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>  @VisibleForTesting<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    this.clusterSchemaService.startAsync();<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span>    try {<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span>    } catch (TimeoutException toe) {<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>    }<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>  }<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span><a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>  private void initQuotaManager() throws IOException {<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>    quotaManager.start();<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>    this.quotaManager = quotaManager;<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>  }<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span><a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>    return notifier;<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span>  }<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span><a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>  boolean isCatalogJanitorEnabled() {<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>    return catalogJanitorChore != null ?<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>  }<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span><a name="line.1244"></a>
-<span class="sourceLineNo">1245</span>  boolean isCleanerChoreEnabled() {<a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1246"></a>
+<span class="sourceLineNo">1200</span>   * &lt;p&gt;<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>   * Will be overridden in tests.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>   * &lt;/p&gt;<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>   */<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>  @VisibleForTesting<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>    // w/ a mocked up ServerManager.<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span>    setupClusterConnection();<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>    return new ServerManager(master);<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>  }<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span><a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>      throws IOException, InterruptedException {<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>    this.serverManager.waitForRegionServers(status);<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>  }<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span><a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>  // Will be overridden in tests<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span>  @VisibleForTesting<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>    this.clusterSchemaService.startAsync();<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>    try {<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>    } catch (TimeoutException toe) {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>    }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>  }<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span><a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>  private void initQuotaManager() throws IOException {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>    quotaManager.start();<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>    this.quotaManager = quotaManager;<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>  }<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span><a name="line.1236"></a>
+<span class="sourceLineNo">1237</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>    return notifier;<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>  }<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span><a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>  boolean isCatalogJanitorEnabled() {<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span>    return catalogJanitorChore != null ?<a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>  }<a name="line.1246"></a>
 <span class="sourceLineNo">1247</span><a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>    if (hfileCleaner != null) {<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span>    }<a name="line.1250"></a>
-<span class="sourceLineNo">1251</span><a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>    if (logCleaner != null) {<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span>    }<a name="line.1254"></a>
-<span class="sourceLineNo">1255</span><a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>  }<a name="line.1257"></a>
+<span class="sourceLineNo">1248</span>  boolean isCleanerChoreEnabled() {<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1249"></a>
+<span class="sourceLineNo">1250</span><a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>    if (hfileCleaner != null) {<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span>    }<a name="line.1253"></a>
+<span class="sourceLineNo">1254</span><a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>    if (logCleaner != null) {<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span>    }<a name="line.1257"></a>
 <span class="sourceLineNo">1258</span><a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>  @Override<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>  public TableDescriptors getTableDescriptors() {<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>    return this.tableDescriptors;<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>  }<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span><a name="line.1263"></a>
-<span class="sourceLineNo">1264</span>  @Override<a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>  public ServerManager getServerManager() {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    return this.serverManager;<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>  }<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span><a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>  @Override<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>    return this.fileSystemManager;<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span>  }<a name="line.1272"></a>
-<span class="sourceLineNo">1273</span><a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>  @Override<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>  public MasterWalManager getMasterWalManager() {<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>    return this.walManager;<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>  }<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span><a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>  @Override<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>  public TableStateManager getTableStateManager() {<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>    return tableStateManager;<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span>  }<a name="line.1282"></a>
-<span class="sourceLineNo">1283</span><a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>  /*<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>   * then they just die with a logged message.  This should be fine because<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>   *  need to install an unexpected exception handler.<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>   */<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>  private void startServiceThreads() throws IOException{<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span>   // Start the executor service pools<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span><a name="line.1303"></a>
-<span class="sourceLineNo">1304</span>   // We depend on there being only one instance of this executor running<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>   // tables.<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   // AccessController#postCompletedCreateTableAction<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   startProcedureExecutor();<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span><a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>    // Initial cleaner chore<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>    CleanerChore.initChorePool(conf);<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>   // Start log cleaner thread<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>   this.logCleaner =<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>      new LogCleaner(cleanerInterval,<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>         getMasterWalManager().getOldLogDir());<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span><a name="line.1321"></a>
-<span class="sourceLineNo">1322</span>    // start the hfile archive cleaner thread<a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>    params.put(MASTER, this);<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>        .getFileSystem(), archiveDir, params);<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span><a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>      replicationPeerManager);<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1332"></a>
-<span class="sourceLineNo">1333</span><a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>    serviceStarted = true;<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>    if (LOG.isTraceEnabled()) {<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>      LOG.trace("Started service threads");<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>    }<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>  }<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span><a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>  @Override<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>  protected void stopServiceThreads() {<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>    if (masterJettyServer != null) {<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>      LOG.info("Stopping master jetty server");<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>      try {<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>        masterJettyServer.stop();<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>      } catch (Exception e) {<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>      }<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span>    }<a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>    stopChores();<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>    if (this.mobCompactThread != null) {<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>      this.mobCompactThread.close();<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>    }<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>    super.stopServiceThreads();<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>    CleanerChore.shutDownChorePool();<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span><a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>    LOG.debug("Stopping service threads");<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span><a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>    if (this.quotaManager != null) {<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>      this.quotaManager.stop();<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>    }<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span><a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>    if (this.activeMasterManager != null) {<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>      this.activeMasterManager.stop();<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>    }<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span>    if (this.serverManager != null) {<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      this.serverManager.stop();<a name="line.1367"></a>
+<span class="sourceLineNo">1259</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>  }<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span><a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>  @Override<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span>  public TableDescriptors getTableDescriptors() {<a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>    return this.tableDescriptors;<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>  }<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span><a name="line.1266"></a>
+<span class="sourceLineNo">1267</span>  @Override<a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>  public ServerManager getServerManager() {<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>    return this.serverManager;<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>  }<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span><a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>  @Override<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span>    return this.fileSystemManager;<a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>  }<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span><a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>  @Override<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>  public MasterWalManager getMasterWalManager() {<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    return this.walManager;<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>  }<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span><a name="line.1281"></a>
+<span class="sourceLineNo">1282</span>  @Override<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>  public TableStateManager getTableStateManager() {<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>    return tableStateManager;<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>  }<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span><a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>  /*<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>   * then they just die with a logged message.  This should be fine because<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span>   *  need to install an unexpected exception handler.<a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>   */<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>  private void startServiceThreads() throws IOException{<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   // Start the executor service pools<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span><a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   // We depend on there being only one instance of this executor running<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   // tables.<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>   // AccessController#postCompletedCreateTableAction<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>   startProcedureExecutor();<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span><a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>    // Initial cleaner chore<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>    CleanerChore.initChorePool(conf);<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>   // Start log cleaner thread<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>   this.logCleaner =<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>      new LogCleaner(cleanerInterval,<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>         getMasterWalManager().getOldLogDir());<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span><a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>    // start the hfile archive cleaner thread<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>    params.put(MASTER, this);<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>        .getFileSystem(), archiveDir, params);<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1331"></a>
+<span class="sourceLineNo">1332</span><a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>      replicationPeerManager);<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span><a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>    serviceStarted = true;<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>    if (LOG.isTraceEnabled()) {<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>      LOG.trace("Started service threads");<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>    }<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>  }<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span><a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>  @Override<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>  protected void stopServiceThreads() {<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>    if (masterJettyServer != null) {<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>      LOG.info("Stopping master jetty server");<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>      try {<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span>        masterJettyServer.stop();<a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>      } catch (Exception e) {<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>      }<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>    }<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span>    stopChores();<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>    if (this.mobCompactThread != null) {<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>      this.mobCompactThread.close();<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span>    }<a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>    super.stopServiceThreads();<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>    CleanerChore.shutDownChorePool();<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span><a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>    LOG.debug("Stopping service threads");<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span><a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>    if (this.quotaManager != null) {<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>      this.quotaManager.stop();<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>    }<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span><a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>    if (this.activeMasterManager != null) {<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>      this.activeMasterManager.stop();<a name="line.1367"></a>
 <span class="sourceLineNo">1368</span>    }<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>    if (this.assignmentManager != null) {<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>      this.assignmentManager.stop();<a name="line.1370"></a>
+<span class="sourceLineNo">1369</span>    if (this.serverManager != null) {<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      this.serverManager.stop();<a name="line.1370"></a>
 <span class="sourceLineNo">1371</span>    }<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span><a name="line.1372"></a>
-<span class="sourceLineNo">1373</span>    stopProcedureExecutor();<a name="line.1373"></a>
-<span class="sourceLineNo">1374</span><a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>    if (this.walManager != null) {<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>      this.walManager.stop();<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>    }<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>    if (this.fileSystemManager != null) {<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>      this.fileSystemManager.stop();<a name="line.1379"></a>
+<span class="sourceLineNo">1372</span>    if (this.assignmentManager != null) {<a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>      this.assignmentManager.stop();<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>    }<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span><a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>    stopProcedureExecutor();<a name="line.1376"></a>
+<span class="sourceLineNo">1377</span><a name="line.1377"></a>
+<span class="sourceLineNo">1378</

<TRUNCATED>

[14/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/MasterWalManager.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/MasterWalManager.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/MasterWalManager.html
index 498695e..a7df46f 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/MasterWalManager.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/MasterWalManager.html
@@ -151,191 +151,196 @@
 <span class="sourceLineNo">143</span>    return this.fsOk;<a name="line.143"></a>
 <span class="sourceLineNo">144</span>  }<a name="line.144"></a>
 <span class="sourceLineNo">145</span><a name="line.145"></a>
-<span class="sourceLineNo">146</span>  public Set&lt;ServerName&gt; getLiveServersFromWALDir() throws IOException {<a name="line.146"></a>
-<span class="sourceLineNo">147</span>    Path walDirPath = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    FileStatus[] walDirForLiveServers = FSUtils.listStatus(fs, walDirPath,<a name="line.148"></a>
-<span class="sourceLineNo">149</span>      p -&gt; !p.getName().endsWith(AbstractFSWALProvider.SPLITTING_EXT));<a name="line.149"></a>
-<span class="sourceLineNo">150</span>    if (walDirForLiveServers == null) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>      return Collections.emptySet();<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    }<a name="line.152"></a>
-<span class="sourceLineNo">153</span>    return Stream.of(walDirForLiveServers).map(s -&gt; {<a name="line.153"></a>
-<span class="sourceLineNo">154</span>      ServerName serverName = AbstractFSWALProvider.getServerNameFromWALDirectoryName(s.getPath());<a name="line.154"></a>
-<span class="sourceLineNo">155</span>      if (serverName == null) {<a name="line.155"></a>
-<span class="sourceLineNo">156</span>        LOG.warn("Log folder {} doesn't look like its name includes a " +<a name="line.156"></a>
-<span class="sourceLineNo">157</span>          "region server name; leaving in place. If you see later errors about missing " +<a name="line.157"></a>
-<span class="sourceLineNo">158</span>          "write ahead logs they may be saved in this location.", s.getPath());<a name="line.158"></a>
-<span class="sourceLineNo">159</span>        return null;<a name="line.159"></a>
-<span class="sourceLineNo">160</span>      }<a name="line.160"></a>
-<span class="sourceLineNo">161</span>      return serverName;<a name="line.161"></a>
-<span class="sourceLineNo">162</span>    }).filter(s -&gt; s != null).collect(Collectors.toSet());<a name="line.162"></a>
-<span class="sourceLineNo">163</span>  }<a name="line.163"></a>
-<span class="sourceLineNo">164</span><a name="line.164"></a>
-<span class="sourceLineNo">165</span>  /**<a name="line.165"></a>
-<span class="sourceLineNo">166</span>   * Inspect the log directory to find dead servers which need recovery work<a name="line.166"></a>
-<span class="sourceLineNo">167</span>   * @return A set of ServerNames which aren't running but still have WAL files left in file system<a name="line.167"></a>
-<span class="sourceLineNo">168</span>   * @deprecated With proc-v2, we can record the crash server with procedure store, so do not need<a name="line.168"></a>
-<span class="sourceLineNo">169</span>   *             to scan the wal directory to find out the splitting wal directory any more. Leave<a name="line.169"></a>
-<span class="sourceLineNo">170</span>   *             it here only because {@code RecoverMetaProcedure}(which is also deprecated) uses<a name="line.170"></a>
-<span class="sourceLineNo">171</span>   *             it.<a name="line.171"></a>
-<span class="sourceLineNo">172</span>   */<a name="line.172"></a>
-<span class="sourceLineNo">173</span>  @Deprecated<a name="line.173"></a>
-<span class="sourceLineNo">174</span>  public Set&lt;ServerName&gt; getFailedServersFromLogFolders() {<a name="line.174"></a>
-<span class="sourceLineNo">175</span>    boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",<a name="line.175"></a>
-<span class="sourceLineNo">176</span>        WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);<a name="line.176"></a>
-<span class="sourceLineNo">177</span><a name="line.177"></a>
-<span class="sourceLineNo">178</span>    Set&lt;ServerName&gt; serverNames = new HashSet&lt;&gt;();<a name="line.178"></a>
-<span class="sourceLineNo">179</span>    Path logsDirPath = new Path(this.rootDir, HConstants.HREGION_LOGDIR_NAME);<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span>    do {<a name="line.181"></a>
-<span class="sourceLineNo">182</span>      if (services.isStopped()) {<a name="line.182"></a>
-<span class="sourceLineNo">183</span>        LOG.warn("Master stopped while trying to get failed servers.");<a name="line.183"></a>
-<span class="sourceLineNo">184</span>        break;<a name="line.184"></a>
-<span class="sourceLineNo">185</span>      }<a name="line.185"></a>
-<span class="sourceLineNo">186</span>      try {<a name="line.186"></a>
-<span class="sourceLineNo">187</span>        if (!this.fs.exists(logsDirPath)) return serverNames;<a name="line.187"></a>
-<span class="sourceLineNo">188</span>        FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);<a name="line.188"></a>
-<span class="sourceLineNo">189</span>        // Get online servers after getting log folders to avoid log folder deletion of newly<a name="line.189"></a>
-<span class="sourceLineNo">190</span>        // checked in region servers . see HBASE-5916<a name="line.190"></a>
-<span class="sourceLineNo">191</span>        Set&lt;ServerName&gt; onlineServers = services.getServerManager().getOnlineServers().keySet();<a name="line.191"></a>
-<span class="sourceLineNo">192</span><a name="line.192"></a>
-<span class="sourceLineNo">193</span>        if (logFolders == null || logFolders.length == 0) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span>          LOG.debug("No log files to split, proceeding...");<a name="line.194"></a>
-<span class="sourceLineNo">195</span>          return serverNames;<a name="line.195"></a>
-<span class="sourceLineNo">196</span>        }<a name="line.196"></a>
-<span class="sourceLineNo">197</span>        for (FileStatus status : logFolders) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>          FileStatus[] curLogFiles = FSUtils.listStatus(this.fs, status.getPath(), null);<a name="line.198"></a>
-<span class="sourceLineNo">199</span>          if (curLogFiles == null || curLogFiles.length == 0) {<a name="line.199"></a>
-<span class="sourceLineNo">200</span>            // Empty log folder. No recovery needed<a name="line.200"></a>
-<span class="sourceLineNo">201</span>            continue;<a name="line.201"></a>
-<span class="sourceLineNo">202</span>          }<a name="line.202"></a>
-<span class="sourceLineNo">203</span>          final ServerName serverName = AbstractFSWALProvider.getServerNameFromWALDirectoryName(<a name="line.203"></a>
-<span class="sourceLineNo">204</span>              status.getPath());<a name="line.204"></a>
-<span class="sourceLineNo">205</span>          if (null == serverName) {<a name="line.205"></a>
-<span class="sourceLineNo">206</span>            LOG.warn("Log folder " + status.getPath() + " doesn't look like its name includes a " +<a name="line.206"></a>
-<span class="sourceLineNo">207</span>                "region server name; leaving in place. If you see later errors about missing " +<a name="line.207"></a>
-<span class="sourceLineNo">208</span>                "write ahead logs they may be saved in this location.");<a name="line.208"></a>
-<span class="sourceLineNo">209</span>          } else if (!onlineServers.contains(serverName)) {<a name="line.209"></a>
-<span class="sourceLineNo">210</span>            LOG.info("Log folder " + status.getPath() + " doesn't belong "<a name="line.210"></a>
-<span class="sourceLineNo">211</span>                + "to a known region server, splitting");<a name="line.211"></a>
-<span class="sourceLineNo">212</span>            serverNames.add(serverName);<a name="line.212"></a>
-<span class="sourceLineNo">213</span>          } else {<a name="line.213"></a>
-<span class="sourceLineNo">214</span>            LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");<a name="line.214"></a>
-<span class="sourceLineNo">215</span>          }<a name="line.215"></a>
-<span class="sourceLineNo">216</span>        }<a name="line.216"></a>
-<span class="sourceLineNo">217</span>        retrySplitting = false;<a name="line.217"></a>
-<span class="sourceLineNo">218</span>      } catch (IOException ioe) {<a name="line.218"></a>
-<span class="sourceLineNo">219</span>        LOG.warn("Failed getting failed servers to be recovered.", ioe);<a name="line.219"></a>
-<span class="sourceLineNo">220</span>        if (!checkFileSystem()) {<a name="line.220"></a>
-<span class="sourceLineNo">221</span>          LOG.warn("Bad Filesystem, exiting");<a name="line.221"></a>
-<span class="sourceLineNo">222</span>          Runtime.getRuntime().halt(1);<a name="line.222"></a>
-<span class="sourceLineNo">223</span>        }<a name="line.223"></a>
-<span class="sourceLineNo">224</span>        try {<a name="line.224"></a>
-<span class="sourceLineNo">225</span>          if (retrySplitting) {<a name="line.225"></a>
-<span class="sourceLineNo">226</span>            Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));<a name="line.226"></a>
-<span class="sourceLineNo">227</span>          }<a name="line.227"></a>
-<span class="sourceLineNo">228</span>        } catch (InterruptedException e) {<a name="line.228"></a>
-<span class="sourceLineNo">229</span>          LOG.warn("Interrupted, aborting since cannot return w/o splitting");<a name="line.229"></a>
-<span class="sourceLineNo">230</span>          Thread.currentThread().interrupt();<a name="line.230"></a>
-<span class="sourceLineNo">231</span>          retrySplitting = false;<a name="line.231"></a>
-<span class="sourceLineNo">232</span>          Runtime.getRuntime().halt(1);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>        }<a name="line.233"></a>
-<span class="sourceLineNo">234</span>      }<a name="line.234"></a>
-<span class="sourceLineNo">235</span>    } while (retrySplitting);<a name="line.235"></a>
-<span class="sourceLineNo">236</span><a name="line.236"></a>
-<span class="sourceLineNo">237</span>    return serverNames;<a name="line.237"></a>
-<span class="sourceLineNo">238</span>  }<a name="line.238"></a>
-<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span>  public void splitLog(final ServerName serverName) throws IOException {<a name="line.240"></a>
-<span class="sourceLineNo">241</span>    splitLog(Collections.&lt;ServerName&gt;singleton(serverName));<a name="line.241"></a>
-<span class="sourceLineNo">242</span>  }<a name="line.242"></a>
-<span class="sourceLineNo">243</span><a name="line.243"></a>
-<span class="sourceLineNo">244</span>  /**<a name="line.244"></a>
-<span class="sourceLineNo">245</span>   * Specialized method to handle the splitting for meta WAL<a name="line.245"></a>
-<span class="sourceLineNo">246</span>   * @param serverName logs belonging to this server will be split<a name="line.246"></a>
-<span class="sourceLineNo">247</span>   */<a name="line.247"></a>
-<span class="sourceLineNo">248</span>  public void splitMetaLog(final ServerName serverName) throws IOException {<a name="line.248"></a>
-<span class="sourceLineNo">249</span>    splitMetaLog(Collections.&lt;ServerName&gt;singleton(serverName));<a name="line.249"></a>
-<span class="sourceLineNo">250</span>  }<a name="line.250"></a>
-<span class="sourceLineNo">251</span><a name="line.251"></a>
-<span class="sourceLineNo">252</span>  /**<a name="line.252"></a>
-<span class="sourceLineNo">253</span>   * Specialized method to handle the splitting for meta WAL<a name="line.253"></a>
-<span class="sourceLineNo">254</span>   * @param serverNames logs belonging to these servers will be split<a name="line.254"></a>
-<span class="sourceLineNo">255</span>   */<a name="line.255"></a>
-<span class="sourceLineNo">256</span>  public void splitMetaLog(final Set&lt;ServerName&gt; serverNames) throws IOException {<a name="line.256"></a>
-<span class="sourceLineNo">257</span>    splitLog(serverNames, META_FILTER);<a name="line.257"></a>
-<span class="sourceLineNo">258</span>  }<a name="line.258"></a>
-<span class="sourceLineNo">259</span><a name="line.259"></a>
-<span class="sourceLineNo">260</span>  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK", justification=<a name="line.260"></a>
-<span class="sourceLineNo">261</span>      "We only release this lock when we set it. Updates to code that uses it should verify use " +<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      "of the guard boolean.")<a name="line.262"></a>
-<span class="sourceLineNo">263</span>  private List&lt;Path&gt; getLogDirs(final Set&lt;ServerName&gt; serverNames) throws IOException {<a name="line.263"></a>
-<span class="sourceLineNo">264</span>    List&lt;Path&gt; logDirs = new ArrayList&lt;&gt;();<a name="line.264"></a>
-<span class="sourceLineNo">265</span>    boolean needReleaseLock = false;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>    if (!this.services.isInitialized()) {<a name="line.266"></a>
-<span class="sourceLineNo">267</span>      // during master initialization, we could have multiple places splitting a same wal<a name="line.267"></a>
-<span class="sourceLineNo">268</span>      // XXX: Does this still exist after we move to proc-v2?<a name="line.268"></a>
-<span class="sourceLineNo">269</span>      this.splitLogLock.lock();<a name="line.269"></a>
-<span class="sourceLineNo">270</span>      needReleaseLock = true;<a name="line.270"></a>
-<span class="sourceLineNo">271</span>    }<a name="line.271"></a>
-<span class="sourceLineNo">272</span>    try {<a name="line.272"></a>
-<span class="sourceLineNo">273</span>      for (ServerName serverName : serverNames) {<a name="line.273"></a>
-<span class="sourceLineNo">274</span>        Path logDir = new Path(this.rootDir,<a name="line.274"></a>
-<span class="sourceLineNo">275</span>          AbstractFSWALProvider.getWALDirectoryName(serverName.toString()));<a name="line.275"></a>
-<span class="sourceLineNo">276</span>        Path splitDir = logDir.suffix(AbstractFSWALProvider.SPLITTING_EXT);<a name="line.276"></a>
-<span class="sourceLineNo">277</span>        // Rename the directory so a rogue RS doesn't create more WALs<a name="line.277"></a>
-<span class="sourceLineNo">278</span>        if (fs.exists(logDir)) {<a name="line.278"></a>
-<span class="sourceLineNo">279</span>          if (!this.fs.rename(logDir, splitDir)) {<a name="line.279"></a>
-<span class="sourceLineNo">280</span>            throw new IOException("Failed fs.rename for log split: " + logDir);<a name="line.280"></a>
-<span class="sourceLineNo">281</span>          }<a name="line.281"></a>
-<span class="sourceLineNo">282</span>          logDir = splitDir;<a name="line.282"></a>
-<span class="sourceLineNo">283</span>          LOG.debug("Renamed region directory: " + splitDir);<a name="line.283"></a>
-<span class="sourceLineNo">284</span>        } else if (!fs.exists(splitDir)) {<a name="line.284"></a>
-<span class="sourceLineNo">285</span>          LOG.info("Log dir for server " + serverName + " does not exist");<a name="line.285"></a>
-<span class="sourceLineNo">286</span>          continue;<a name="line.286"></a>
-<span class="sourceLineNo">287</span>        }<a name="line.287"></a>
-<span class="sourceLineNo">288</span>        logDirs.add(splitDir);<a name="line.288"></a>
-<span class="sourceLineNo">289</span>      }<a name="line.289"></a>
-<span class="sourceLineNo">290</span>    } catch (IOException ioe) {<a name="line.290"></a>
-<span class="sourceLineNo">291</span>      if (!checkFileSystem()) {<a name="line.291"></a>
-<span class="sourceLineNo">292</span>        this.services.abort("Aborting due to filesystem unavailable", ioe);<a name="line.292"></a>
-<span class="sourceLineNo">293</span>        throw ioe;<a name="line.293"></a>
+<span class="sourceLineNo">146</span>  /**<a name="line.146"></a>
+<span class="sourceLineNo">147</span>   * @return listing of ServerNames found in the filesystem under the WAL directory<a name="line.147"></a>
+<span class="sourceLineNo">148</span>   *   that COULD BE 'alive'; excludes those that have a '-splitting' suffix as these are already<a name="line.148"></a>
+<span class="sourceLineNo">149</span>   *   being split -- they cannot be 'alive'.<a name="line.149"></a>
+<span class="sourceLineNo">150</span>   */<a name="line.150"></a>
+<span class="sourceLineNo">151</span>  public Set&lt;ServerName&gt; getLiveServersFromWALDir() throws IOException {<a name="line.151"></a>
+<span class="sourceLineNo">152</span>    Path walDirPath = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);<a name="line.152"></a>
+<span class="sourceLineNo">153</span>    FileStatus[] walDirForLiveServers = FSUtils.listStatus(fs, walDirPath,<a name="line.153"></a>
+<span class="sourceLineNo">154</span>      p -&gt; !p.getName().endsWith(AbstractFSWALProvider.SPLITTING_EXT));<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    if (walDirForLiveServers == null) {<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      return Collections.emptySet();<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    return Stream.of(walDirForLiveServers).map(s -&gt; {<a name="line.158"></a>
+<span class="sourceLineNo">159</span>      ServerName serverName = AbstractFSWALProvider.getServerNameFromWALDirectoryName(s.getPath());<a name="line.159"></a>
+<span class="sourceLineNo">160</span>      if (serverName == null) {<a name="line.160"></a>
+<span class="sourceLineNo">161</span>        LOG.warn("Log folder {} doesn't look like its name includes a " +<a name="line.161"></a>
+<span class="sourceLineNo">162</span>          "region server name; leaving in place. If you see later errors about missing " +<a name="line.162"></a>
+<span class="sourceLineNo">163</span>          "write ahead logs they may be saved in this location.", s.getPath());<a name="line.163"></a>
+<span class="sourceLineNo">164</span>        return null;<a name="line.164"></a>
+<span class="sourceLineNo">165</span>      }<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      return serverName;<a name="line.166"></a>
+<span class="sourceLineNo">167</span>    }).filter(s -&gt; s != null).collect(Collectors.toSet());<a name="line.167"></a>
+<span class="sourceLineNo">168</span>  }<a name="line.168"></a>
+<span class="sourceLineNo">169</span><a name="line.169"></a>
+<span class="sourceLineNo">170</span>  /**<a name="line.170"></a>
+<span class="sourceLineNo">171</span>   * Inspect the log directory to find dead servers which need recovery work<a name="line.171"></a>
+<span class="sourceLineNo">172</span>   * @return A set of ServerNames which aren't running but still have WAL files left in file system<a name="line.172"></a>
+<span class="sourceLineNo">173</span>   * @deprecated With proc-v2, we can record the crash server with procedure store, so do not need<a name="line.173"></a>
+<span class="sourceLineNo">174</span>   *             to scan the wal directory to find out the splitting wal directory any more. Leave<a name="line.174"></a>
+<span class="sourceLineNo">175</span>   *             it here only because {@code RecoverMetaProcedure}(which is also deprecated) uses<a name="line.175"></a>
+<span class="sourceLineNo">176</span>   *             it.<a name="line.176"></a>
+<span class="sourceLineNo">177</span>   */<a name="line.177"></a>
+<span class="sourceLineNo">178</span>  @Deprecated<a name="line.178"></a>
+<span class="sourceLineNo">179</span>  public Set&lt;ServerName&gt; getFailedServersFromLogFolders() {<a name="line.179"></a>
+<span class="sourceLineNo">180</span>    boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",<a name="line.180"></a>
+<span class="sourceLineNo">181</span>        WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);<a name="line.181"></a>
+<span class="sourceLineNo">182</span><a name="line.182"></a>
+<span class="sourceLineNo">183</span>    Set&lt;ServerName&gt; serverNames = new HashSet&lt;&gt;();<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    Path logsDirPath = new Path(this.rootDir, HConstants.HREGION_LOGDIR_NAME);<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span>    do {<a name="line.186"></a>
+<span class="sourceLineNo">187</span>      if (services.isStopped()) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span>        LOG.warn("Master stopped while trying to get failed servers.");<a name="line.188"></a>
+<span class="sourceLineNo">189</span>        break;<a name="line.189"></a>
+<span class="sourceLineNo">190</span>      }<a name="line.190"></a>
+<span class="sourceLineNo">191</span>      try {<a name="line.191"></a>
+<span class="sourceLineNo">192</span>        if (!this.fs.exists(logsDirPath)) return serverNames;<a name="line.192"></a>
+<span class="sourceLineNo">193</span>        FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);<a name="line.193"></a>
+<span class="sourceLineNo">194</span>        // Get online servers after getting log folders to avoid log folder deletion of newly<a name="line.194"></a>
+<span class="sourceLineNo">195</span>        // checked in region servers . see HBASE-5916<a name="line.195"></a>
+<span class="sourceLineNo">196</span>        Set&lt;ServerName&gt; onlineServers = services.getServerManager().getOnlineServers().keySet();<a name="line.196"></a>
+<span class="sourceLineNo">197</span><a name="line.197"></a>
+<span class="sourceLineNo">198</span>        if (logFolders == null || logFolders.length == 0) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span>          LOG.debug("No log files to split, proceeding...");<a name="line.199"></a>
+<span class="sourceLineNo">200</span>          return serverNames;<a name="line.200"></a>
+<span class="sourceLineNo">201</span>        }<a name="line.201"></a>
+<span class="sourceLineNo">202</span>        for (FileStatus status : logFolders) {<a name="line.202"></a>
+<span class="sourceLineNo">203</span>          FileStatus[] curLogFiles = FSUtils.listStatus(this.fs, status.getPath(), null);<a name="line.203"></a>
+<span class="sourceLineNo">204</span>          if (curLogFiles == null || curLogFiles.length == 0) {<a name="line.204"></a>
+<span class="sourceLineNo">205</span>            // Empty log folder. No recovery needed<a name="line.205"></a>
+<span class="sourceLineNo">206</span>            continue;<a name="line.206"></a>
+<span class="sourceLineNo">207</span>          }<a name="line.207"></a>
+<span class="sourceLineNo">208</span>          final ServerName serverName = AbstractFSWALProvider.getServerNameFromWALDirectoryName(<a name="line.208"></a>
+<span class="sourceLineNo">209</span>              status.getPath());<a name="line.209"></a>
+<span class="sourceLineNo">210</span>          if (null == serverName) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>            LOG.warn("Log folder " + status.getPath() + " doesn't look like its name includes a " +<a name="line.211"></a>
+<span class="sourceLineNo">212</span>                "region server name; leaving in place. If you see later errors about missing " +<a name="line.212"></a>
+<span class="sourceLineNo">213</span>                "write ahead logs they may be saved in this location.");<a name="line.213"></a>
+<span class="sourceLineNo">214</span>          } else if (!onlineServers.contains(serverName)) {<a name="line.214"></a>
+<span class="sourceLineNo">215</span>            LOG.info("Log folder " + status.getPath() + " doesn't belong "<a name="line.215"></a>
+<span class="sourceLineNo">216</span>                + "to a known region server, splitting");<a name="line.216"></a>
+<span class="sourceLineNo">217</span>            serverNames.add(serverName);<a name="line.217"></a>
+<span class="sourceLineNo">218</span>          } else {<a name="line.218"></a>
+<span class="sourceLineNo">219</span>            LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");<a name="line.219"></a>
+<span class="sourceLineNo">220</span>          }<a name="line.220"></a>
+<span class="sourceLineNo">221</span>        }<a name="line.221"></a>
+<span class="sourceLineNo">222</span>        retrySplitting = false;<a name="line.222"></a>
+<span class="sourceLineNo">223</span>      } catch (IOException ioe) {<a name="line.223"></a>
+<span class="sourceLineNo">224</span>        LOG.warn("Failed getting failed servers to be recovered.", ioe);<a name="line.224"></a>
+<span class="sourceLineNo">225</span>        if (!checkFileSystem()) {<a name="line.225"></a>
+<span class="sourceLineNo">226</span>          LOG.warn("Bad Filesystem, exiting");<a name="line.226"></a>
+<span class="sourceLineNo">227</span>          Runtime.getRuntime().halt(1);<a name="line.227"></a>
+<span class="sourceLineNo">228</span>        }<a name="line.228"></a>
+<span class="sourceLineNo">229</span>        try {<a name="line.229"></a>
+<span class="sourceLineNo">230</span>          if (retrySplitting) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span>            Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));<a name="line.231"></a>
+<span class="sourceLineNo">232</span>          }<a name="line.232"></a>
+<span class="sourceLineNo">233</span>        } catch (InterruptedException e) {<a name="line.233"></a>
+<span class="sourceLineNo">234</span>          LOG.warn("Interrupted, aborting since cannot return w/o splitting");<a name="line.234"></a>
+<span class="sourceLineNo">235</span>          Thread.currentThread().interrupt();<a name="line.235"></a>
+<span class="sourceLineNo">236</span>          retrySplitting = false;<a name="line.236"></a>
+<span class="sourceLineNo">237</span>          Runtime.getRuntime().halt(1);<a name="line.237"></a>
+<span class="sourceLineNo">238</span>        }<a name="line.238"></a>
+<span class="sourceLineNo">239</span>      }<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    } while (retrySplitting);<a name="line.240"></a>
+<span class="sourceLineNo">241</span><a name="line.241"></a>
+<span class="sourceLineNo">242</span>    return serverNames;<a name="line.242"></a>
+<span class="sourceLineNo">243</span>  }<a name="line.243"></a>
+<span class="sourceLineNo">244</span><a name="line.244"></a>
+<span class="sourceLineNo">245</span>  public void splitLog(final ServerName serverName) throws IOException {<a name="line.245"></a>
+<span class="sourceLineNo">246</span>    splitLog(Collections.&lt;ServerName&gt;singleton(serverName));<a name="line.246"></a>
+<span class="sourceLineNo">247</span>  }<a name="line.247"></a>
+<span class="sourceLineNo">248</span><a name="line.248"></a>
+<span class="sourceLineNo">249</span>  /**<a name="line.249"></a>
+<span class="sourceLineNo">250</span>   * Specialized method to handle the splitting for meta WAL<a name="line.250"></a>
+<span class="sourceLineNo">251</span>   * @param serverName logs belonging to this server will be split<a name="line.251"></a>
+<span class="sourceLineNo">252</span>   */<a name="line.252"></a>
+<span class="sourceLineNo">253</span>  public void splitMetaLog(final ServerName serverName) throws IOException {<a name="line.253"></a>
+<span class="sourceLineNo">254</span>    splitMetaLog(Collections.&lt;ServerName&gt;singleton(serverName));<a name="line.254"></a>
+<span class="sourceLineNo">255</span>  }<a name="line.255"></a>
+<span class="sourceLineNo">256</span><a name="line.256"></a>
+<span class="sourceLineNo">257</span>  /**<a name="line.257"></a>
+<span class="sourceLineNo">258</span>   * Specialized method to handle the splitting for meta WAL<a name="line.258"></a>
+<span class="sourceLineNo">259</span>   * @param serverNames logs belonging to these servers will be split<a name="line.259"></a>
+<span class="sourceLineNo">260</span>   */<a name="line.260"></a>
+<span class="sourceLineNo">261</span>  public void splitMetaLog(final Set&lt;ServerName&gt; serverNames) throws IOException {<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    splitLog(serverNames, META_FILTER);<a name="line.262"></a>
+<span class="sourceLineNo">263</span>  }<a name="line.263"></a>
+<span class="sourceLineNo">264</span><a name="line.264"></a>
+<span class="sourceLineNo">265</span>  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK", justification=<a name="line.265"></a>
+<span class="sourceLineNo">266</span>      "We only release this lock when we set it. Updates to code that uses it should verify use " +<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      "of the guard boolean.")<a name="line.267"></a>
+<span class="sourceLineNo">268</span>  private List&lt;Path&gt; getLogDirs(final Set&lt;ServerName&gt; serverNames) throws IOException {<a name="line.268"></a>
+<span class="sourceLineNo">269</span>    List&lt;Path&gt; logDirs = new ArrayList&lt;&gt;();<a name="line.269"></a>
+<span class="sourceLineNo">270</span>    boolean needReleaseLock = false;<a name="line.270"></a>
+<span class="sourceLineNo">271</span>    if (!this.services.isInitialized()) {<a name="line.271"></a>
+<span class="sourceLineNo">272</span>      // during master initialization, we could have multiple places splitting a same wal<a name="line.272"></a>
+<span class="sourceLineNo">273</span>      // XXX: Does this still exist after we move to proc-v2?<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      this.splitLogLock.lock();<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      needReleaseLock = true;<a name="line.275"></a>
+<span class="sourceLineNo">276</span>    }<a name="line.276"></a>
+<span class="sourceLineNo">277</span>    try {<a name="line.277"></a>
+<span class="sourceLineNo">278</span>      for (ServerName serverName : serverNames) {<a name="line.278"></a>
+<span class="sourceLineNo">279</span>        Path logDir = new Path(this.rootDir,<a name="line.279"></a>
+<span class="sourceLineNo">280</span>          AbstractFSWALProvider.getWALDirectoryName(serverName.toString()));<a name="line.280"></a>
+<span class="sourceLineNo">281</span>        Path splitDir = logDir.suffix(AbstractFSWALProvider.SPLITTING_EXT);<a name="line.281"></a>
+<span class="sourceLineNo">282</span>        // Rename the directory so a rogue RS doesn't create more WALs<a name="line.282"></a>
+<span class="sourceLineNo">283</span>        if (fs.exists(logDir)) {<a name="line.283"></a>
+<span class="sourceLineNo">284</span>          if (!this.fs.rename(logDir, splitDir)) {<a name="line.284"></a>
+<span class="sourceLineNo">285</span>            throw new IOException("Failed fs.rename for log split: " + logDir);<a name="line.285"></a>
+<span class="sourceLineNo">286</span>          }<a name="line.286"></a>
+<span class="sourceLineNo">287</span>          logDir = splitDir;<a name="line.287"></a>
+<span class="sourceLineNo">288</span>          LOG.debug("Renamed region directory: " + splitDir);<a name="line.288"></a>
+<span class="sourceLineNo">289</span>        } else if (!fs.exists(splitDir)) {<a name="line.289"></a>
+<span class="sourceLineNo">290</span>          LOG.info("Log dir for server " + serverName + " does not exist");<a name="line.290"></a>
+<span class="sourceLineNo">291</span>          continue;<a name="line.291"></a>
+<span class="sourceLineNo">292</span>        }<a name="line.292"></a>
+<span class="sourceLineNo">293</span>        logDirs.add(splitDir);<a name="line.293"></a>
 <span class="sourceLineNo">294</span>      }<a name="line.294"></a>
-<span class="sourceLineNo">295</span>    } finally {<a name="line.295"></a>
-<span class="sourceLineNo">296</span>      if (needReleaseLock) {<a name="line.296"></a>
-<span class="sourceLineNo">297</span>        this.splitLogLock.unlock();<a name="line.297"></a>
-<span class="sourceLineNo">298</span>      }<a name="line.298"></a>
-<span class="sourceLineNo">299</span>    }<a name="line.299"></a>
-<span class="sourceLineNo">300</span>    return logDirs;<a name="line.300"></a>
-<span class="sourceLineNo">301</span>  }<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span>  public void splitLog(final Set&lt;ServerName&gt; serverNames) throws IOException {<a name="line.303"></a>
-<span class="sourceLineNo">304</span>    splitLog(serverNames, NON_META_FILTER);<a name="line.304"></a>
-<span class="sourceLineNo">305</span>  }<a name="line.305"></a>
-<span class="sourceLineNo">306</span><a name="line.306"></a>
-<span class="sourceLineNo">307</span>  /**<a name="line.307"></a>
-<span class="sourceLineNo">308</span>   * This method is the base split method that splits WAL files matching a filter. Callers should<a name="line.308"></a>
-<span class="sourceLineNo">309</span>   * pass the appropriate filter for meta and non-meta WALs.<a name="line.309"></a>
-<span class="sourceLineNo">310</span>   * @param serverNames logs belonging to these servers will be split; this will rename the log<a name="line.310"></a>
-<span class="sourceLineNo">311</span>   *                    directory out from under a soft-failed server<a name="line.311"></a>
-<span class="sourceLineNo">312</span>   */<a name="line.312"></a>
-<span class="sourceLineNo">313</span>  public void splitLog(final Set&lt;ServerName&gt; serverNames, PathFilter filter) throws IOException {<a name="line.313"></a>
-<span class="sourceLineNo">314</span>    long splitTime = 0, splitLogSize = 0;<a name="line.314"></a>
-<span class="sourceLineNo">315</span>    List&lt;Path&gt; logDirs = getLogDirs(serverNames);<a name="line.315"></a>
-<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span>    splitLogManager.handleDeadWorkers(serverNames);<a name="line.317"></a>
-<span class="sourceLineNo">318</span>    splitTime = EnvironmentEdgeManager.currentTime();<a name="line.318"></a>
-<span class="sourceLineNo">319</span>    splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);<a name="line.319"></a>
-<span class="sourceLineNo">320</span>    splitTime = EnvironmentEdgeManager.currentTime() - splitTime;<a name="line.320"></a>
+<span class="sourceLineNo">295</span>    } catch (IOException ioe) {<a name="line.295"></a>
+<span class="sourceLineNo">296</span>      if (!checkFileSystem()) {<a name="line.296"></a>
+<span class="sourceLineNo">297</span>        this.services.abort("Aborting due to filesystem unavailable", ioe);<a name="line.297"></a>
+<span class="sourceLineNo">298</span>        throw ioe;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>      }<a name="line.299"></a>
+<span class="sourceLineNo">300</span>    } finally {<a name="line.300"></a>
+<span class="sourceLineNo">301</span>      if (needReleaseLock) {<a name="line.301"></a>
+<span class="sourceLineNo">302</span>        this.splitLogLock.unlock();<a name="line.302"></a>
+<span class="sourceLineNo">303</span>      }<a name="line.303"></a>
+<span class="sourceLineNo">304</span>    }<a name="line.304"></a>
+<span class="sourceLineNo">305</span>    return logDirs;<a name="line.305"></a>
+<span class="sourceLineNo">306</span>  }<a name="line.306"></a>
+<span class="sourceLineNo">307</span><a name="line.307"></a>
+<span class="sourceLineNo">308</span>  public void splitLog(final Set&lt;ServerName&gt; serverNames) throws IOException {<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    splitLog(serverNames, NON_META_FILTER);<a name="line.309"></a>
+<span class="sourceLineNo">310</span>  }<a name="line.310"></a>
+<span class="sourceLineNo">311</span><a name="line.311"></a>
+<span class="sourceLineNo">312</span>  /**<a name="line.312"></a>
+<span class="sourceLineNo">313</span>   * This method is the base split method that splits WAL files matching a filter. Callers should<a name="line.313"></a>
+<span class="sourceLineNo">314</span>   * pass the appropriate filter for meta and non-meta WALs.<a name="line.314"></a>
+<span class="sourceLineNo">315</span>   * @param serverNames logs belonging to these servers will be split; this will rename the log<a name="line.315"></a>
+<span class="sourceLineNo">316</span>   *                    directory out from under a soft-failed server<a name="line.316"></a>
+<span class="sourceLineNo">317</span>   */<a name="line.317"></a>
+<span class="sourceLineNo">318</span>  public void splitLog(final Set&lt;ServerName&gt; serverNames, PathFilter filter) throws IOException {<a name="line.318"></a>
+<span class="sourceLineNo">319</span>    long splitTime = 0, splitLogSize = 0;<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    List&lt;Path&gt; logDirs = getLogDirs(serverNames);<a name="line.320"></a>
 <span class="sourceLineNo">321</span><a name="line.321"></a>
-<span class="sourceLineNo">322</span>    if (this.metricsMasterFilesystem != null) {<a name="line.322"></a>
-<span class="sourceLineNo">323</span>      if (filter == META_FILTER) {<a name="line.323"></a>
-<span class="sourceLineNo">324</span>        this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);<a name="line.324"></a>
-<span class="sourceLineNo">325</span>      } else {<a name="line.325"></a>
-<span class="sourceLineNo">326</span>        this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);<a name="line.326"></a>
-<span class="sourceLineNo">327</span>      }<a name="line.327"></a>
-<span class="sourceLineNo">328</span>    }<a name="line.328"></a>
-<span class="sourceLineNo">329</span>  }<a name="line.329"></a>
-<span class="sourceLineNo">330</span>}<a name="line.330"></a>
+<span class="sourceLineNo">322</span>    splitLogManager.handleDeadWorkers(serverNames);<a name="line.322"></a>
+<span class="sourceLineNo">323</span>    splitTime = EnvironmentEdgeManager.currentTime();<a name="line.323"></a>
+<span class="sourceLineNo">324</span>    splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);<a name="line.324"></a>
+<span class="sourceLineNo">325</span>    splitTime = EnvironmentEdgeManager.currentTime() - splitTime;<a name="line.325"></a>
+<span class="sourceLineNo">326</span><a name="line.326"></a>
+<span class="sourceLineNo">327</span>    if (this.metricsMasterFilesystem != null) {<a name="line.327"></a>
+<span class="sourceLineNo">328</span>      if (filter == META_FILTER) {<a name="line.328"></a>
+<span class="sourceLineNo">329</span>        this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);<a name="line.329"></a>
+<span class="sourceLineNo">330</span>      } else {<a name="line.330"></a>
+<span class="sourceLineNo">331</span>        this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);<a name="line.331"></a>
+<span class="sourceLineNo">332</span>      }<a name="line.332"></a>
+<span class="sourceLineNo">333</span>    }<a name="line.333"></a>
+<span class="sourceLineNo">334</span>  }<a name="line.334"></a>
+<span class="sourceLineNo">335</span>}<a name="line.335"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html
index 256ed63..1b8c609 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html
@@ -119,79 +119,81 @@
 <span class="sourceLineNo">111</span>   * In this method, we will also construct the region server sets in {@link ServerManager}. If a<a name="line.111"></a>
 <span class="sourceLineNo">112</span>   * region server is dead between the crash of the previous master instance and the start of the<a name="line.112"></a>
 <span class="sourceLineNo">113</span>   * current master instance, we will schedule a SCP for it. This is done in<a name="line.113"></a>
-<span class="sourceLineNo">114</span>   * {@link ServerManager#findOutDeadServersAndProcess(Set, Set)}, we call it here under the lock<a name="line.114"></a>
+<span class="sourceLineNo">114</span>   * {@link ServerManager#findDeadServersAndProcess(Set, Set)}, we call it here under the lock<a name="line.114"></a>
 <span class="sourceLineNo">115</span>   * protection to prevent concurrency issues with server expiration operation.<a name="line.115"></a>
 <span class="sourceLineNo">116</span>   * @param deadServersFromPE the region servers which already have SCP associated.<a name="line.116"></a>
 <span class="sourceLineNo">117</span>   * @param liveServersFromWALDir the live region servers from wal directory.<a name="line.117"></a>
 <span class="sourceLineNo">118</span>   */<a name="line.118"></a>
 <span class="sourceLineNo">119</span>  public void start(Set&lt;ServerName&gt; deadServersFromPE, Set&lt;ServerName&gt; liveServersFromWALDir)<a name="line.119"></a>
 <span class="sourceLineNo">120</span>      throws KeeperException, IOException {<a name="line.120"></a>
-<span class="sourceLineNo">121</span>    watcher.registerListener(this);<a name="line.121"></a>
-<span class="sourceLineNo">122</span>    synchronized (this) {<a name="line.122"></a>
-<span class="sourceLineNo">123</span>      List&lt;String&gt; servers =<a name="line.123"></a>
-<span class="sourceLineNo">124</span>        ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.getZNodePaths().rsZNode);<a name="line.124"></a>
-<span class="sourceLineNo">125</span>      for (String n : servers) {<a name="line.125"></a>
-<span class="sourceLineNo">126</span>        Pair&lt;ServerName, RegionServerInfo&gt; pair = getServerInfo(n);<a name="line.126"></a>
-<span class="sourceLineNo">127</span>        ServerName serverName = pair.getFirst();<a name="line.127"></a>
-<span class="sourceLineNo">128</span>        RegionServerInfo info = pair.getSecond();<a name="line.128"></a>
-<span class="sourceLineNo">129</span>        regionServers.add(serverName);<a name="line.129"></a>
-<span class="sourceLineNo">130</span>        ServerMetrics serverMetrics = info != null ? ServerMetricsBuilder.of(serverName,<a name="line.130"></a>
-<span class="sourceLineNo">131</span>          VersionInfoUtil.getVersionNumber(info.getVersionInfo()),<a name="line.131"></a>
-<span class="sourceLineNo">132</span>          info.getVersionInfo().getVersion()) : ServerMetricsBuilder.of(serverName);<a name="line.132"></a>
-<span class="sourceLineNo">133</span>        serverManager.checkAndRecordNewServer(serverName, serverMetrics);<a name="line.133"></a>
-<span class="sourceLineNo">134</span>      }<a name="line.134"></a>
-<span class="sourceLineNo">135</span>      serverManager.findOutDeadServersAndProcess(deadServersFromPE, liveServersFromWALDir);<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    }<a name="line.136"></a>
-<span class="sourceLineNo">137</span>  }<a name="line.137"></a>
-<span class="sourceLineNo">138</span><a name="line.138"></a>
-<span class="sourceLineNo">139</span>  public void stop() {<a name="line.139"></a>
-<span class="sourceLineNo">140</span>    executor.shutdownNow();<a name="line.140"></a>
-<span class="sourceLineNo">141</span>  }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span>  private synchronized void refresh() {<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    List&lt;String&gt; names;<a name="line.144"></a>
-<span class="sourceLineNo">145</span>    try {<a name="line.145"></a>
-<span class="sourceLineNo">146</span>      names = ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.getZNodePaths().rsZNode);<a name="line.146"></a>
-<span class="sourceLineNo">147</span>    } catch (KeeperException e) {<a name="line.147"></a>
-<span class="sourceLineNo">148</span>      // here we need to abort as we failed to set watcher on the rs node which means that we can<a name="line.148"></a>
-<span class="sourceLineNo">149</span>      // not track the node deleted evetnt any more.<a name="line.149"></a>
-<span class="sourceLineNo">150</span>      server.abort("Unexpected zk exception getting RS nodes", e);<a name="line.150"></a>
-<span class="sourceLineNo">151</span>      return;<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    }<a name="line.152"></a>
-<span class="sourceLineNo">153</span>    Set&lt;ServerName&gt; servers =<a name="line.153"></a>
-<span class="sourceLineNo">154</span>      names.stream().map(ServerName::parseServerName).collect(Collectors.toSet());<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    for (Iterator&lt;ServerName&gt; iter = regionServers.iterator(); iter.hasNext();) {<a name="line.155"></a>
-<span class="sourceLineNo">156</span>      ServerName sn = iter.next();<a name="line.156"></a>
-<span class="sourceLineNo">157</span>      if (!servers.contains(sn)) {<a name="line.157"></a>
-<span class="sourceLineNo">158</span>        LOG.info("RegionServer ephemeral node deleted, processing expiration [{}]", sn);<a name="line.158"></a>
-<span class="sourceLineNo">159</span>        serverManager.expireServer(sn);<a name="line.159"></a>
-<span class="sourceLineNo">160</span>        iter.remove();<a name="line.160"></a>
-<span class="sourceLineNo">161</span>      }<a name="line.161"></a>
-<span class="sourceLineNo">162</span>    }<a name="line.162"></a>
-<span class="sourceLineNo">163</span>    // here we do not need to parse the region server info as it is useless now, we only need the<a name="line.163"></a>
-<span class="sourceLineNo">164</span>    // server name.<a name="line.164"></a>
-<span class="sourceLineNo">165</span>    boolean newServerAdded = false;<a name="line.165"></a>
-<span class="sourceLineNo">166</span>    for (ServerName sn : servers) {<a name="line.166"></a>
-<span class="sourceLineNo">167</span>      if (regionServers.add(sn)) {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>        newServerAdded = true;<a name="line.168"></a>
-<span class="sourceLineNo">169</span>        LOG.info("RegionServer ephemeral node created, adding [" + sn + "]");<a name="line.169"></a>
-<span class="sourceLineNo">170</span>      }<a name="line.170"></a>
-<span class="sourceLineNo">171</span>    }<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    if (newServerAdded &amp;&amp; server.isInitialized()) {<a name="line.172"></a>
-<span class="sourceLineNo">173</span>      // Only call the check to move servers if a RegionServer was added to the cluster; in this<a name="line.173"></a>
-<span class="sourceLineNo">174</span>      // case it could be a server with a new version so it makes sense to run the check.<a name="line.174"></a>
-<span class="sourceLineNo">175</span>      server.checkIfShouldMoveSystemRegionAsync();<a name="line.175"></a>
-<span class="sourceLineNo">176</span>    }<a name="line.176"></a>
-<span class="sourceLineNo">177</span>  }<a name="line.177"></a>
-<span class="sourceLineNo">178</span><a name="line.178"></a>
-<span class="sourceLineNo">179</span>  @Override<a name="line.179"></a>
-<span class="sourceLineNo">180</span>  public void nodeChildrenChanged(String path) {<a name="line.180"></a>
-<span class="sourceLineNo">181</span>    if (path.equals(watcher.getZNodePaths().rsZNode) &amp;&amp; !server.isAborted() &amp;&amp;<a name="line.181"></a>
-<span class="sourceLineNo">182</span>      !server.isStopped()) {<a name="line.182"></a>
-<span class="sourceLineNo">183</span>      executor.execute(this::refresh);<a name="line.183"></a>
-<span class="sourceLineNo">184</span>    }<a name="line.184"></a>
-<span class="sourceLineNo">185</span>  }<a name="line.185"></a>
-<span class="sourceLineNo">186</span>}<a name="line.186"></a>
+<span class="sourceLineNo">121</span>    LOG.info("Starting RegionServerTracker; {} have existing ServerCrashProcedures, {} " +<a name="line.121"></a>
+<span class="sourceLineNo">122</span>        "possibly 'live' servers.", deadServersFromPE.size(), liveServersFromWALDir.size());<a name="line.122"></a>
+<span class="sourceLineNo">123</span>    watcher.registerListener(this);<a name="line.123"></a>
+<span class="sourceLineNo">124</span>    synchronized (this) {<a name="line.124"></a>
+<span class="sourceLineNo">125</span>      List&lt;String&gt; servers =<a name="line.125"></a>
+<span class="sourceLineNo">126</span>        ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.getZNodePaths().rsZNode);<a name="line.126"></a>
+<span class="sourceLineNo">127</span>      for (String n : servers) {<a name="line.127"></a>
+<span class="sourceLineNo">128</span>        Pair&lt;ServerName, RegionServerInfo&gt; pair = getServerInfo(n);<a name="line.128"></a>
+<span class="sourceLineNo">129</span>        ServerName serverName = pair.getFirst();<a name="line.129"></a>
+<span class="sourceLineNo">130</span>        RegionServerInfo info = pair.getSecond();<a name="line.130"></a>
+<span class="sourceLineNo">131</span>        regionServers.add(serverName);<a name="line.131"></a>
+<span class="sourceLineNo">132</span>        ServerMetrics serverMetrics = info != null ? ServerMetricsBuilder.of(serverName,<a name="line.132"></a>
+<span class="sourceLineNo">133</span>          VersionInfoUtil.getVersionNumber(info.getVersionInfo()),<a name="line.133"></a>
+<span class="sourceLineNo">134</span>          info.getVersionInfo().getVersion()) : ServerMetricsBuilder.of(serverName);<a name="line.134"></a>
+<span class="sourceLineNo">135</span>        serverManager.checkAndRecordNewServer(serverName, serverMetrics);<a name="line.135"></a>
+<span class="sourceLineNo">136</span>      }<a name="line.136"></a>
+<span class="sourceLineNo">137</span>      serverManager.findDeadServersAndProcess(deadServersFromPE, liveServersFromWALDir);<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    }<a name="line.138"></a>
+<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span>  public void stop() {<a name="line.141"></a>
+<span class="sourceLineNo">142</span>    executor.shutdownNow();<a name="line.142"></a>
+<span class="sourceLineNo">143</span>  }<a name="line.143"></a>
+<span class="sourceLineNo">144</span><a name="line.144"></a>
+<span class="sourceLineNo">145</span>  private synchronized void refresh() {<a name="line.145"></a>
+<span class="sourceLineNo">146</span>    List&lt;String&gt; names;<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    try {<a name="line.147"></a>
+<span class="sourceLineNo">148</span>      names = ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.getZNodePaths().rsZNode);<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    } catch (KeeperException e) {<a name="line.149"></a>
+<span class="sourceLineNo">150</span>      // here we need to abort as we failed to set watcher on the rs node which means that we can<a name="line.150"></a>
+<span class="sourceLineNo">151</span>      // not track the node deleted evetnt any more.<a name="line.151"></a>
+<span class="sourceLineNo">152</span>      server.abort("Unexpected zk exception getting RS nodes", e);<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      return;<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    }<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    Set&lt;ServerName&gt; servers =<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      names.stream().map(ServerName::parseServerName).collect(Collectors.toSet());<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    for (Iterator&lt;ServerName&gt; iter = regionServers.iterator(); iter.hasNext();) {<a name="line.157"></a>
+<span class="sourceLineNo">158</span>      ServerName sn = iter.next();<a name="line.158"></a>
+<span class="sourceLineNo">159</span>      if (!servers.contains(sn)) {<a name="line.159"></a>
+<span class="sourceLineNo">160</span>        LOG.info("RegionServer ephemeral node deleted, processing expiration [{}]", sn);<a name="line.160"></a>
+<span class="sourceLineNo">161</span>        serverManager.expireServer(sn);<a name="line.161"></a>
+<span class="sourceLineNo">162</span>        iter.remove();<a name="line.162"></a>
+<span class="sourceLineNo">163</span>      }<a name="line.163"></a>
+<span class="sourceLineNo">164</span>    }<a name="line.164"></a>
+<span class="sourceLineNo">165</span>    // here we do not need to parse the region server info as it is useless now, we only need the<a name="line.165"></a>
+<span class="sourceLineNo">166</span>    // server name.<a name="line.166"></a>
+<span class="sourceLineNo">167</span>    boolean newServerAdded = false;<a name="line.167"></a>
+<span class="sourceLineNo">168</span>    for (ServerName sn : servers) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span>      if (regionServers.add(sn)) {<a name="line.169"></a>
+<span class="sourceLineNo">170</span>        newServerAdded = true;<a name="line.170"></a>
+<span class="sourceLineNo">171</span>        LOG.info("RegionServer ephemeral node created, adding [" + sn + "]");<a name="line.171"></a>
+<span class="sourceLineNo">172</span>      }<a name="line.172"></a>
+<span class="sourceLineNo">173</span>    }<a name="line.173"></a>
+<span class="sourceLineNo">174</span>    if (newServerAdded &amp;&amp; server.isInitialized()) {<a name="line.174"></a>
+<span class="sourceLineNo">175</span>      // Only call the check to move servers if a RegionServer was added to the cluster; in this<a name="line.175"></a>
+<span class="sourceLineNo">176</span>      // case it could be a server with a new version so it makes sense to run the check.<a name="line.176"></a>
+<span class="sourceLineNo">177</span>      server.checkIfShouldMoveSystemRegionAsync();<a name="line.177"></a>
+<span class="sourceLineNo">178</span>    }<a name="line.178"></a>
+<span class="sourceLineNo">179</span>  }<a name="line.179"></a>
+<span class="sourceLineNo">180</span><a name="line.180"></a>
+<span class="sourceLineNo">181</span>  @Override<a name="line.181"></a>
+<span class="sourceLineNo">182</span>  public void nodeChildrenChanged(String path) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    if (path.equals(watcher.getZNodePaths().rsZNode) &amp;&amp; !server.isAborted() &amp;&amp;<a name="line.183"></a>
+<span class="sourceLineNo">184</span>      !server.isStopped()) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>      executor.execute(this::refresh);<a name="line.185"></a>
+<span class="sourceLineNo">186</span>    }<a name="line.186"></a>
+<span class="sourceLineNo">187</span>  }<a name="line.187"></a>
+<span class="sourceLineNo">188</span>}<a name="line.188"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.FlushedSequenceIdFlusher.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.FlushedSequenceIdFlusher.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.FlushedSequenceIdFlusher.html
index d98a33d..c13e4a4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.FlushedSequenceIdFlusher.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.FlushedSequenceIdFlusher.html
@@ -358,10 +358,10 @@
 <span class="sourceLineNo">350</span>   * &lt;p/&gt;<a name="line.350"></a>
 <span class="sourceLineNo">351</span>   * Must be called inside the initialization method of {@code RegionServerTracker} to avoid<a name="line.351"></a>
 <span class="sourceLineNo">352</span>   * concurrency issue.<a name="line.352"></a>
-<span class="sourceLineNo">353</span>   * @param deadServersFromPE the region servers which already have SCP associated.<a name="line.353"></a>
+<span class="sourceLineNo">353</span>   * @param deadServersFromPE the region servers which already have a SCP associated.<a name="line.353"></a>
 <span class="sourceLineNo">354</span>   * @param liveServersFromWALDir the live region servers from wal directory.<a name="line.354"></a>
 <span class="sourceLineNo">355</span>   */<a name="line.355"></a>
-<span class="sourceLineNo">356</span>  void findOutDeadServersAndProcess(Set&lt;ServerName&gt; deadServersFromPE,<a name="line.356"></a>
+<span class="sourceLineNo">356</span>  void findDeadServersAndProcess(Set&lt;ServerName&gt; deadServersFromPE,<a name="line.356"></a>
 <span class="sourceLineNo">357</span>      Set&lt;ServerName&gt; liveServersFromWALDir) {<a name="line.357"></a>
 <span class="sourceLineNo">358</span>    deadServersFromPE.forEach(deadservers::add);<a name="line.358"></a>
 <span class="sourceLineNo">359</span>    liveServersFromWALDir.stream().filter(sn -&gt; !onlineServers.containsKey(sn))<a name="line.359"></a>
@@ -629,7 +629,7 @@
 <span class="sourceLineNo">621</span>  public void moveFromOnlineToDeadServers(final ServerName sn) {<a name="line.621"></a>
 <span class="sourceLineNo">622</span>    synchronized (onlineServers) {<a name="line.622"></a>
 <span class="sourceLineNo">623</span>      if (!this.onlineServers.containsKey(sn)) {<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        LOG.warn("Expiration of " + sn + " but server not online");<a name="line.624"></a>
+<span class="sourceLineNo">624</span>        LOG.trace("Expiration of {} but server not online", sn);<a name="line.624"></a>
 <span class="sourceLineNo">625</span>      }<a name="line.625"></a>
 <span class="sourceLineNo">626</span>      // Remove the server from the known servers lists and update load info BUT<a name="line.626"></a>
 <span class="sourceLineNo">627</span>      // add to deadservers first; do this so it'll show in dead servers list if<a name="line.627"></a>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.html
index d98a33d..c13e4a4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/ServerManager.html
@@ -358,10 +358,10 @@
 <span class="sourceLineNo">350</span>   * &lt;p/&gt;<a name="line.350"></a>
 <span class="sourceLineNo">351</span>   * Must be called inside the initialization method of {@code RegionServerTracker} to avoid<a name="line.351"></a>
 <span class="sourceLineNo">352</span>   * concurrency issue.<a name="line.352"></a>
-<span class="sourceLineNo">353</span>   * @param deadServersFromPE the region servers which already have SCP associated.<a name="line.353"></a>
+<span class="sourceLineNo">353</span>   * @param deadServersFromPE the region servers which already have a SCP associated.<a name="line.353"></a>
 <span class="sourceLineNo">354</span>   * @param liveServersFromWALDir the live region servers from wal directory.<a name="line.354"></a>
 <span class="sourceLineNo">355</span>   */<a name="line.355"></a>
-<span class="sourceLineNo">356</span>  void findOutDeadServersAndProcess(Set&lt;ServerName&gt; deadServersFromPE,<a name="line.356"></a>
+<span class="sourceLineNo">356</span>  void findDeadServersAndProcess(Set&lt;ServerName&gt; deadServersFromPE,<a name="line.356"></a>
 <span class="sourceLineNo">357</span>      Set&lt;ServerName&gt; liveServersFromWALDir) {<a name="line.357"></a>
 <span class="sourceLineNo">358</span>    deadServersFromPE.forEach(deadservers::add);<a name="line.358"></a>
 <span class="sourceLineNo">359</span>    liveServersFromWALDir.stream().filter(sn -&gt; !onlineServers.containsKey(sn))<a name="line.359"></a>
@@ -629,7 +629,7 @@
 <span class="sourceLineNo">621</span>  public void moveFromOnlineToDeadServers(final ServerName sn) {<a name="line.621"></a>
 <span class="sourceLineNo">622</span>    synchronized (onlineServers) {<a name="line.622"></a>
 <span class="sourceLineNo">623</span>      if (!this.onlineServers.containsKey(sn)) {<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        LOG.warn("Expiration of " + sn + " but server not online");<a name="line.624"></a>
+<span class="sourceLineNo">624</span>        LOG.trace("Expiration of {} but server not online", sn);<a name="line.624"></a>
 <span class="sourceLineNo">625</span>      }<a name="line.625"></a>
 <span class="sourceLineNo">626</span>      // Remove the server from the known servers lists and update load info BUT<a name="line.626"></a>
 <span class="sourceLineNo">627</span>      // add to deadservers first; do this so it'll show in dead servers list if<a name="line.627"></a>


[21/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/MasterWalManager.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/MasterWalManager.html b/devapidocs/org/apache/hadoop/hbase/master/MasterWalManager.html
index 3462352..2348fa8 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/MasterWalManager.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/MasterWalManager.html
@@ -527,9 +527,13 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getLiveServersFromWALDir</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.146">getLiveServersFromWALDir</a>()
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.151">getLiveServersFromWALDir</a>()
                                          throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
+<dt><span class="returnLabel">Returns:</span></dt>
+<dd>listing of ServerNames found in the filesystem under the WAL directory
+   that COULD BE 'alive'; excludes those that have a '-splitting' suffix as these are already
+   being split -- they cannot be 'alive'.</dd>
 <dt><span class="throwsLabel">Throws:</span></dt>
 <dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></code></dd>
 </dl>
@@ -542,7 +546,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <li class="blockList">
 <h4>getFailedServersFromLogFolders</h4>
 <pre><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Deprecated.html?is-external=true" title="class or interface in java.lang">@Deprecated</a>
-public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.174">getFailedServersFromLogFolders</a>()</pre>
+public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.179">getFailedServersFromLogFolders</a>()</pre>
 <div class="block"><span class="deprecatedLabel">Deprecated.</span>&nbsp;<span class="deprecationComment">With proc-v2, we can record the crash server with procedure store, so do not need
              to scan the wal directory to find out the splitting wal directory any more. Leave
              it here only because <code>RecoverMetaProcedure</code>(which is also deprecated) uses
@@ -560,7 +564,7 @@ public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.htm
 <ul class="blockList">
 <li class="blockList">
 <h4>splitLog</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.240">splitLog</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;serverName)
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.245">splitLog</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;serverName)
               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -574,7 +578,7 @@ public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.htm
 <ul class="blockList">
 <li class="blockList">
 <h4>splitMetaLog</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.248">splitMetaLog</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;serverName)
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.253">splitMetaLog</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;serverName)
                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Specialized method to handle the splitting for meta WAL</div>
 <dl>
@@ -591,7 +595,7 @@ public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.htm
 <ul class="blockList">
 <li class="blockList">
 <h4>splitMetaLog</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.256">splitMetaLog</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames)
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.261">splitMetaLog</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames)
                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Specialized method to handle the splitting for meta WAL</div>
 <dl>
@@ -608,7 +612,7 @@ public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.htm
 <ul class="blockList">
 <li class="blockList">
 <h4>getLogDirs</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;org.apache.hadoop.fs.Path&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.263">getLogDirs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames)
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;org.apache.hadoop.fs.Path&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.268">getLogDirs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames)
                                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -622,7 +626,7 @@ public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.htm
 <ul class="blockList">
 <li class="blockList">
 <h4>splitLog</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.303">splitLog</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames)
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.308">splitLog</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames)
               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -636,7 +640,7 @@ public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.htm
 <ul class="blockListLast">
 <li class="blockList">
 <h4>splitLog</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.313">splitLog</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/MasterWalManager.html#line.318">splitLog</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames,
                      org.apache.hadoop.fs.PathFilter&nbsp;filter)
               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">This method is the base split method that splits WAL files matching a filter. Callers should

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/RegionServerTracker.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/RegionServerTracker.html b/devapidocs/org/apache/hadoop/hbase/master/RegionServerTracker.html
index e0e705a..ea701c9 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/RegionServerTracker.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/RegionServerTracker.html
@@ -363,7 +363,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.htm
  In this method, we will also construct the region server sets in <a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html" title="class in org.apache.hadoop.hbase.master"><code>ServerManager</code></a>. If a
  region server is dead between the crash of the previous master instance and the start of the
  current master instance, we will schedule a SCP for it. This is done in
- <a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html#findOutDeadServersAndProcess-java.util.Set-java.util.Set-"><code>ServerManager.findOutDeadServersAndProcess(Set, Set)</code></a>, we call it here under the lock
+ <a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html#findDeadServersAndProcess-java.util.Set-java.util.Set-"><code>ServerManager.findDeadServersAndProcess(Set, Set)</code></a>, we call it here under the lock
  protection to prevent concurrency issues with server expiration operation.</div>
 <dl>
 <dt><span class="paramLabel">Parameters:</span></dt>
@@ -381,7 +381,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.htm
 <ul class="blockList">
 <li class="blockList">
 <h4>stop</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html#line.139">stop</a>()</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html#line.141">stop</a>()</pre>
 </li>
 </ul>
 <a name="refresh--">
@@ -390,7 +390,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.htm
 <ul class="blockList">
 <li class="blockList">
 <h4>refresh</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html#line.143">refresh</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html#line.145">refresh</a>()</pre>
 </li>
 </ul>
 <a name="nodeChildrenChanged-java.lang.String-">
@@ -399,7 +399,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.htm
 <ul class="blockListLast">
 <li class="blockList">
 <h4>nodeChildrenChanged</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html#line.180">nodeChildrenChanged</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;path)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/RegionServerTracker.html#line.182">nodeChildrenChanged</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;path)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.html#nodeChildrenChanged-java.lang.String-">ZKListener</a></code></span></div>
 <div class="block">Called when an existing node has a child node added or removed.</div>
 <dl>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/ServerManager.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/ServerManager.html b/devapidocs/org/apache/hadoop/hbase/master/ServerManager.html
index 7aceba4..b47e951 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/ServerManager.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/ServerManager.html
@@ -400,8 +400,8 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 </tr>
 <tr id="i11" class="rowColor">
 <td class="colFirst"><code>(package private) void</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html#findOutDeadServersAndProcess-java.util.Set-java.util.Set-">findOutDeadServersAndProcess</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;deadServersFromPE,
-                            <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;liveServersFromWALDir)</code>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html#findDeadServersAndProcess-java.util.Set-java.util.Set-">findDeadServersAndProcess</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;deadServersFromPE,
+                         <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;liveServersFromWALDir)</code>
 <div class="block">Find out the region servers crashed between the crash of the previous master instance and the
  current master instance and schedule SCP for them.</div>
 </td>
@@ -1051,14 +1051,14 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 </dl>
 </li>
 </ul>
-<a name="findOutDeadServersAndProcess-java.util.Set-java.util.Set-">
+<a name="findDeadServersAndProcess-java.util.Set-java.util.Set-">
 <!--   -->
 </a>
 <ul class="blockList">
 <li class="blockList">
-<h4>findOutDeadServersAndProcess</h4>
-<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/ServerManager.html#line.356">findOutDeadServersAndProcess</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;deadServersFromPE,
-                                  <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;liveServersFromWALDir)</pre>
+<h4>findDeadServersAndProcess</h4>
+<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/ServerManager.html#line.356">findDeadServersAndProcess</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;deadServersFromPE,
+                               <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;liveServersFromWALDir)</pre>
 <div class="block">Find out the region servers crashed between the crash of the previous master instance and the
  current master instance and schedule SCP for them.
  <p/>
@@ -1070,7 +1070,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
  concurrency issue.</div>
 <dl>
 <dt><span class="paramLabel">Parameters:</span></dt>
-<dd><code>deadServersFromPE</code> - the region servers which already have SCP associated.</dd>
+<dd><code>deadServersFromPE</code> - the region servers which already have a SCP associated.</dd>
 <dd><code>liveServersFromWALDir</code> - the live region servers from wal directory.</dd>
 </dl>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html
index c7f4d20..f0b6ad5 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>public static enum <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.639">SplitLogManager.ResubmitDirective</a>
+<pre>public static enum <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.643">SplitLogManager.ResubmitDirective</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang">Enum</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.ResubmitDirective</a>&gt;</pre>
 </li>
 </ul>
@@ -210,7 +210,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>CHECK</h4>
-<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.ResubmitDirective</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html#line.640">CHECK</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.ResubmitDirective</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html#line.644">CHECK</a></pre>
 </li>
 </ul>
 <a name="FORCE">
@@ -219,7 +219,7 @@ the order they are declared.</div>
 <ul class="blockListLast">
 <li class="blockList">
 <h4>FORCE</h4>
-<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.ResubmitDirective</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html#line.640">FORCE</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.ResubmitDirective</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html#line.644">FORCE</a></pre>
 </li>
 </ul>
 </li>
@@ -236,7 +236,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>values</h4>
-<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.ResubmitDirective</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html#line.471">values</a>()</pre>
+<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.ResubmitDirective</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html#line.475">values</a>()</pre>
 <div class="block">Returns an array containing the constants of this enum type, in
 the order they are declared.  This method may be used to iterate
 over the constants as follows:
@@ -256,7 +256,7 @@ for (SplitLogManager.ResubmitDirective c : SplitLogManager.ResubmitDirective.val
 <ul class="blockListLast">
 <li class="blockList">
 <h4>valueOf</h4>
-<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.ResubmitDirective</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html#line.471">valueOf</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
+<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.ResubmitDirective</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html#line.475">valueOf</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
 <div class="block">Returns the enum constant of this type with the specified name.
 The string must match <i>exactly</i> an identifier used to declare an
 enum constant in this type.  (Extraneous whitespace characters are 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.Task.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.Task.html b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.Task.html
index 238811f..6f07e74 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.Task.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.Task.html
@@ -114,7 +114,7 @@ var activeTableTab = "activeTableTab";
 <hr>
 <br>
 <pre>@InterfaceAudience.Private
-public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.487">SplitLogManager.Task</a>
+public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.491">SplitLogManager.Task</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
 <div class="block">in memory state of an active task.</div>
 </li>
@@ -253,7 +253,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>last_update</h4>
-<pre>public volatile&nbsp;long <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.488">last_update</a></pre>
+<pre>public volatile&nbsp;long <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.492">last_update</a></pre>
 </li>
 </ul>
 <a name="last_version">
@@ -262,7 +262,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>last_version</h4>
-<pre>public volatile&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.489">last_version</a></pre>
+<pre>public volatile&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.493">last_version</a></pre>
 </li>
 </ul>
 <a name="cur_worker_name">
@@ -271,7 +271,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>cur_worker_name</h4>
-<pre>public volatile&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.490">cur_worker_name</a></pre>
+<pre>public volatile&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.494">cur_worker_name</a></pre>
 </li>
 </ul>
 <a name="batch">
@@ -280,7 +280,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>batch</h4>
-<pre>public volatile&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.TaskBatch</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.491">batch</a></pre>
+<pre>public volatile&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.TaskBatch</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.495">batch</a></pre>
 </li>
 </ul>
 <a name="status">
@@ -289,7 +289,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>status</h4>
-<pre>public volatile&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.492">status</a></pre>
+<pre>public volatile&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.496">status</a></pre>
 </li>
 </ul>
 <a name="incarnation">
@@ -298,7 +298,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>incarnation</h4>
-<pre>public volatile&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicInteger.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicInteger</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.493">incarnation</a></pre>
+<pre>public volatile&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicInteger.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicInteger</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.497">incarnation</a></pre>
 </li>
 </ul>
 <a name="unforcedResubmits">
@@ -307,7 +307,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>unforcedResubmits</h4>
-<pre>public final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicInteger.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicInteger</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.494">unforcedResubmits</a></pre>
+<pre>public final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicInteger.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicInteger</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.498">unforcedResubmits</a></pre>
 </li>
 </ul>
 <a name="resubmitThresholdReached">
@@ -316,7 +316,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>resubmitThresholdReached</h4>
-<pre>public volatile&nbsp;boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.495">resubmitThresholdReached</a></pre>
+<pre>public volatile&nbsp;boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.499">resubmitThresholdReached</a></pre>
 </li>
 </ul>
 </li>
@@ -333,7 +333,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>Task</h4>
-<pre>public&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.504">Task</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.508">Task</a>()</pre>
 </li>
 </ul>
 </li>
@@ -350,7 +350,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>toString</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.498">toString</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.502">toString</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
 <dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#toString--" title="class or interface in java.lang">toString</a></code>&nbsp;in class&nbsp;<code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></code></dd>
@@ -363,7 +363,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isOrphan</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.510">isOrphan</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.514">isOrphan</a>()</pre>
 </li>
 </ul>
 <a name="isUnassigned--">
@@ -372,7 +372,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isUnassigned</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.514">isUnassigned</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.518">isUnassigned</a>()</pre>
 </li>
 </ul>
 <a name="heartbeatNoDetails-long-">
@@ -381,7 +381,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>heartbeatNoDetails</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.518">heartbeatNoDetails</a>(long&nbsp;time)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.522">heartbeatNoDetails</a>(long&nbsp;time)</pre>
 </li>
 </ul>
 <a name="heartbeat-long-int-org.apache.hadoop.hbase.ServerName-">
@@ -390,7 +390,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>heartbeat</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.522">heartbeat</a>(long&nbsp;time,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.526">heartbeat</a>(long&nbsp;time,
                       int&nbsp;version,
                       <a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;worker)</pre>
 </li>
@@ -401,7 +401,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>setUnassigned</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.528">setUnassigned</a>()</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.Task.html#line.532">setUnassigned</a>()</pre>
 </li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html
index 6a7f65f..bbedc60 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html
@@ -114,7 +114,7 @@ var activeTableTab = "activeTableTab";
 <hr>
 <br>
 <pre>@InterfaceAudience.Private
-public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.471">SplitLogManager.TaskBatch</a>
+public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.475">SplitLogManager.TaskBatch</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
 <div class="block">Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().
  Clients threads use this object to wait for all their tasks to be done.
@@ -218,7 +218,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>installed</h4>
-<pre>public&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.472">installed</a></pre>
+<pre>public&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.476">installed</a></pre>
 </li>
 </ul>
 <a name="done">
@@ -227,7 +227,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>done</h4>
-<pre>public&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.473">done</a></pre>
+<pre>public&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.477">done</a></pre>
 </li>
 </ul>
 <a name="error">
@@ -236,7 +236,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>error</h4>
-<pre>public&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.474">error</a></pre>
+<pre>public&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.478">error</a></pre>
 </li>
 </ul>
 <a name="isDead">
@@ -245,7 +245,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>isDead</h4>
-<pre>public volatile&nbsp;boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.475">isDead</a></pre>
+<pre>public volatile&nbsp;boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.479">isDead</a></pre>
 </li>
 </ul>
 </li>
@@ -262,7 +262,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>TaskBatch</h4>
-<pre>public&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.471">TaskBatch</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.475">TaskBatch</a>()</pre>
 </li>
 </ul>
 </li>
@@ -279,7 +279,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>toString</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.478">toString</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html#line.482">toString</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
 <dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#toString--" title="class or interface in java.lang">toString</a></code>&nbsp;in class&nbsp;<code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html
index 0e535d9..2f79173 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>public static enum <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.643">SplitLogManager.TerminationStatus</a>
+<pre>public static enum <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.647">SplitLogManager.TerminationStatus</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang">Enum</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a>&gt;</pre>
 </li>
 </ul>
@@ -239,7 +239,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>IN_PROGRESS</h4>
-<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.644">IN_PROGRESS</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.648">IN_PROGRESS</a></pre>
 </li>
 </ul>
 <a name="SUCCESS">
@@ -248,7 +248,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>SUCCESS</h4>
-<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.644">SUCCESS</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.648">SUCCESS</a></pre>
 </li>
 </ul>
 <a name="FAILURE">
@@ -257,7 +257,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>FAILURE</h4>
-<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.644">FAILURE</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.648">FAILURE</a></pre>
 </li>
 </ul>
 <a name="DELETED">
@@ -266,7 +266,7 @@ the order they are declared.</div>
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DELETED</h4>
-<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.644">DELETED</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.648">DELETED</a></pre>
 </li>
 </ul>
 </li>
@@ -283,7 +283,7 @@ the order they are declared.</div>
 <ul class="blockListLast">
 <li class="blockList">
 <h4>statusMsg</h4>
-<pre>final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.646">statusMsg</a></pre>
+<pre>final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.650">statusMsg</a></pre>
 </li>
 </ul>
 </li>
@@ -300,7 +300,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>values</h4>
-<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.471">values</a>()</pre>
+<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.475">values</a>()</pre>
 <div class="block">Returns an array containing the constants of this enum type, in
 the order they are declared.  This method may be used to iterate
 over the constants as follows:
@@ -320,7 +320,7 @@ for (SplitLogManager.TerminationStatus c : SplitLogManager.TerminationStatus.val
 <ul class="blockList">
 <li class="blockList">
 <h4>valueOf</h4>
-<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.471">valueOf</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
+<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.475">valueOf</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
 <div class="block">Returns the enum constant of this type with the specified name.
 The string must match <i>exactly</i> an identifier used to declare an
 enum constant in this type.  (Extraneous whitespace characters are 
@@ -342,7 +342,7 @@ not permitted.)</div>
 <ul class="blockListLast">
 <li class="blockList">
 <h4>toString</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.653">toString</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html#line.657">toString</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
 <dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true#toString--" title="class or interface in java.lang">toString</a></code>&nbsp;in class&nbsp;<code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang">Enum</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master">SplitLogManager.TerminationStatus</a>&gt;</code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html
index 5bf5c2d..de6ae9d 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>private class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.537">SplitLogManager.TimeoutMonitor</a>
+<pre>private class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.541">SplitLogManager.TimeoutMonitor</a>
 extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" title="class in org.apache.hadoop.hbase">ScheduledChore</a></pre>
 <div class="block">Periodically checks all active tasks and resubmits the ones that have timed out</div>
 </li>
@@ -221,7 +221,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockListLast">
 <li class="blockList">
 <h4>lastLog</h4>
-<pre>private&nbsp;long <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html#line.538">lastLog</a></pre>
+<pre>private&nbsp;long <a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html#line.542">lastLog</a></pre>
 </li>
 </ul>
 </li>
@@ -238,7 +238,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockListLast">
 <li class="blockList">
 <h4>TimeoutMonitor</h4>
-<pre>public&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html#line.540">TimeoutMonitor</a>(int&nbsp;period,
+<pre>public&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html#line.544">TimeoutMonitor</a>(int&nbsp;period,
                       <a href="../../../../../org/apache/hadoop/hbase/Stoppable.html" title="interface in org.apache.hadoop.hbase">Stoppable</a>&nbsp;stopper)</pre>
 </li>
 </ul>
@@ -256,7 +256,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockListLast">
 <li class="blockList">
 <h4>chore</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html#line.545">chore</a>()</pre>
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html#line.549">chore</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html#chore--">ScheduledChore</a></code></span></div>
 <div class="block">The task to execute on each scheduled execution of the Chore</div>
 <dl>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.html b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.html
index 9c78500..308dccf 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/SplitLogManager.html
@@ -633,7 +633,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>enqueueSplitTask</h4>
-<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.303">enqueueSplitTask</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;taskname,
+<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.307">enqueueSplitTask</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;taskname,
                          <a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.TaskBatch</a>&nbsp;batch)</pre>
 <div class="block">Add a task entry to coordination if it is not already there.</div>
 <dl>
@@ -651,7 +651,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>waitForSplittingCompletion</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.315">waitForSplittingCompletion</a>(<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.TaskBatch</a>&nbsp;batch,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.319">waitForSplittingCompletion</a>(<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.TaskBatch</a>&nbsp;batch,
                                         <a href="../../../../../org/apache/hadoop/hbase/monitoring/MonitoredTask.html" title="interface in org.apache.hadoop.hbase.monitoring">MonitoredTask</a>&nbsp;status)</pre>
 </li>
 </ul>
@@ -661,7 +661,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getTasks</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentMap</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.Task.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.Task</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.354">getTasks</a>()</pre>
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentMap</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.Task.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.Task</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.358">getTasks</a>()</pre>
 </li>
 </ul>
 <a name="activeTasks-org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch-">
@@ -670,7 +670,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>activeTasks</h4>
-<pre>private&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.358">activeTasks</a>(<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.TaskBatch</a>&nbsp;batch)</pre>
+<pre>private&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.362">activeTasks</a>(<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.TaskBatch</a>&nbsp;batch)</pre>
 </li>
 </ul>
 <a name="createTaskIfAbsent-java.lang.String-org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch-">
@@ -679,7 +679,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>createTaskIfAbsent</h4>
-<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.Task.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.Task</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.374">createTaskIfAbsent</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;path,
+<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.Task.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.Task</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.378">createTaskIfAbsent</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;path,
                                                 <a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TaskBatch.html" title="class in org.apache.hadoop.hbase.master">SplitLogManager.TaskBatch</a>&nbsp;batch)</pre>
 <dl>
 <dt><span class="paramLabel">Parameters:</span></dt>
@@ -696,7 +696,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>stop</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.433">stop</a>()</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.437">stop</a>()</pre>
 </li>
 </ul>
 <a name="handleDeadWorker-org.apache.hadoop.hbase.ServerName-">
@@ -705,7 +705,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>handleDeadWorker</h4>
-<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.442">handleDeadWorker</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;workerName)</pre>
+<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.446">handleDeadWorker</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;workerName)</pre>
 </li>
 </ul>
 <a name="handleDeadWorkers-java.util.Set-">
@@ -714,7 +714,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>handleDeadWorkers</h4>
-<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.454">handleDeadWorkers</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames)</pre>
+<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/SplitLogManager.html#line.458">handleDeadWorkers</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;serverNames)</pre>
 </li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
index d4fb18d..ec42b1a 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
@@ -454,7 +454,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeHeader</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.103">writeHeader</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/OutputStream.html?is-external=true" title="class or interface in java.io">OutputStream</a>&nbsp;stream,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.104">writeHeader</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/OutputStream.html?is-external=true" title="class or interface in java.io">OutputStream</a>&nbsp;stream,
                                org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALHeader&nbsp;header)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -469,7 +469,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeTrailer</h4>
-<pre>public static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.123">writeTrailer</a>(org.apache.hadoop.fs.FSDataOutputStream&nbsp;stream,
+<pre>public static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.124">writeTrailer</a>(org.apache.hadoop.fs.FSDataOutputStream&nbsp;stream,
                                 <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)
                          throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -484,7 +484,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readHeader</h4>
-<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALHeader&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.141">readHeader</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html?is-external=true" title="class or interface in java.io">InputStream</a>&nbsp;stream)
+<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALHeader&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.142">readHeader</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html?is-external=true" title="class or interface in java.io">InputStream</a>&nbsp;stream)
                                                                                                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -498,7 +498,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readTrailer</h4>
-<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALTrailer&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.166">readTrailer</a>(org.apache.hadoop.fs.FSDataInputStream&nbsp;stream,
+<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALTrailer&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.167">readTrailer</a>(org.apache.hadoop.fs.FSDataInputStream&nbsp;stream,
                                                                                                                 long&nbsp;startPos,
                                                                                                                 long&nbsp;size)
                                                                                                          throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -514,7 +514,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readEntry</h4>
-<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.203">readEntry</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html?is-external=true" title="class or interface in java.io">InputStream</a>&nbsp;stream)
+<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.204">readEntry</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html?is-external=true" title="class or interface in java.io">InputStream</a>&nbsp;stream)
                                                                                                      throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -528,7 +528,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeEntry</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.207">writeEntry</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.208">writeEntry</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                               org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry.Type&nbsp;type,
                               <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc,
                               <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>[]&nbsp;subprocs)
@@ -545,7 +545,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeInsert</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.220">writeInsert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.221">writeInsert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -560,7 +560,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeInsert</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.225">writeInsert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.226">writeInsert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>[]&nbsp;subprocs)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -576,7 +576,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeUpdate</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.230">writeUpdate</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.231">writeUpdate</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -591,7 +591,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeDelete</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.235">writeDelete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.236">writeDelete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                long&nbsp;procId)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -606,7 +606,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>writeDelete</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.243">writeDelete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.244">writeDelete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc,
                                long[]&nbsp;subprocs)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
index a692a51..c3309bf 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>private static enum <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.661">WALProcedureStore.PushType</a>
+<pre>private static enum <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.668">WALProcedureStore.PushType</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang">Enum</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a>&gt;</pre>
 </li>
 </ul>
@@ -213,7 +213,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>INSERT</h4>
-<pre>public static final&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a> <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html#line.661">INSERT</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a> <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html#line.668">INSERT</a></pre>
 </li>
 </ul>
 <a name="UPDATE">
@@ -222,7 +222,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>UPDATE</h4>
-<pre>public static final&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a> <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html#line.661">UPDATE</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a> <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html#line.668">UPDATE</a></pre>
 </li>
 </ul>
 <a name="DELETE">
@@ -231,7 +231,7 @@ the order they are declared.</div>
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DELETE</h4>
-<pre>public static final&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a> <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html#line.661">DELETE</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a> <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html#line.668">DELETE</a></pre>
 </li>
 </ul>
 </li>


[17/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html
index a990153..d2da8f4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html
@@ -593,7 +593,7 @@
 <span class="sourceLineNo">585</span>              abort(error, t);<a name="line.585"></a>
 <span class="sourceLineNo">586</span>            }<a name="line.586"></a>
 <span class="sourceLineNo">587</span>          }<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        }));<a name="line.588"></a>
+<span class="sourceLineNo">588</span>        }), getName() + ":becomeActiveMaster");<a name="line.588"></a>
 <span class="sourceLineNo">589</span>      }<a name="line.589"></a>
 <span class="sourceLineNo">590</span>      // Fall in here even if we have been aborted. Need to run the shutdown services and<a name="line.590"></a>
 <span class="sourceLineNo">591</span>      // the super run call will do this for us.<a name="line.591"></a>
@@ -932,7 +932,7 @@
 <span class="sourceLineNo">924</span>          HBaseFsck.createLockRetryCounterFactory(this.conf).create());<a name="line.924"></a>
 <span class="sourceLineNo">925</span>    }<a name="line.925"></a>
 <span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span>    status.setStatus("Initialze ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
+<span class="sourceLineNo">927</span>    status.setStatus("Initialize ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
 <span class="sourceLineNo">928</span>    this.serverManager = createServerManager(this);<a name="line.928"></a>
 <span class="sourceLineNo">929</span>    createProcedureExecutor();<a name="line.929"></a>
 <span class="sourceLineNo">930</span>    @SuppressWarnings("rawtypes")<a name="line.930"></a>
@@ -954,2890 +954,2893 @@
 <span class="sourceLineNo">946</span>        .collect(Collectors.toList());<a name="line.946"></a>
 <span class="sourceLineNo">947</span>    this.assignmentManager.setupRIT(ritList);<a name="line.947"></a>
 <span class="sourceLineNo">948</span><a name="line.948"></a>
-<span class="sourceLineNo">949</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.949"></a>
-<span class="sourceLineNo">950</span>    this.regionServerTracker.start(<a name="line.950"></a>
-<span class="sourceLineNo">951</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.951"></a>
-<span class="sourceLineNo">952</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.952"></a>
-<span class="sourceLineNo">953</span>      walManager.getLiveServersFromWALDir());<a name="line.953"></a>
-<span class="sourceLineNo">954</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.954"></a>
-<span class="sourceLineNo">955</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.955"></a>
-<span class="sourceLineNo">956</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.956"></a>
-<span class="sourceLineNo">957</span>    this.tableStateManager =<a name="line.957"></a>
-<span class="sourceLineNo">958</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.958"></a>
-<span class="sourceLineNo">959</span>        ?<a name="line.959"></a>
-<span class="sourceLineNo">960</span>        new MirroringTableStateManager(this):<a name="line.960"></a>
-<span class="sourceLineNo">961</span>        new TableStateManager(this);<a name="line.961"></a>
-<span class="sourceLineNo">962</span><a name="line.962"></a>
-<span class="sourceLineNo">963</span>    status.setStatus("Initializing ZK system trackers");<a name="line.963"></a>
-<span class="sourceLineNo">964</span>    initializeZKBasedSystemTrackers();<a name="line.964"></a>
-<span class="sourceLineNo">965</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.965"></a>
-<span class="sourceLineNo">966</span>    try {<a name="line.966"></a>
-<span class="sourceLineNo">967</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.967"></a>
-<span class="sourceLineNo">968</span>    } catch (IOException e) {<a name="line.968"></a>
-<span class="sourceLineNo">969</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.969"></a>
-<span class="sourceLineNo">970</span>          + " from file system", e);<a name="line.970"></a>
-<span class="sourceLineNo">971</span>    }<a name="line.971"></a>
-<span class="sourceLineNo">972</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.972"></a>
-<span class="sourceLineNo">973</span>    this.activeMaster = true;<a name="line.973"></a>
-<span class="sourceLineNo">974</span><a name="line.974"></a>
-<span class="sourceLineNo">975</span>    // This is for backwards compatibility<a name="line.975"></a>
-<span class="sourceLineNo">976</span>    // See HBASE-11393<a name="line.976"></a>
-<span class="sourceLineNo">977</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.977"></a>
-<span class="sourceLineNo">978</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.978"></a>
-<span class="sourceLineNo">979</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.979"></a>
-<span class="sourceLineNo">980</span>    tableCFsUpdater.copyTableCFs();<a name="line.980"></a>
-<span class="sourceLineNo">981</span><a name="line.981"></a>
-<span class="sourceLineNo">982</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.982"></a>
-<span class="sourceLineNo">983</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.983"></a>
-<span class="sourceLineNo">984</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.984"></a>
-<span class="sourceLineNo">985</span>      updateConfigurationForQuotasObserver(conf);<a name="line.985"></a>
-<span class="sourceLineNo">986</span>    }<a name="line.986"></a>
-<span class="sourceLineNo">987</span>    // initialize master side coprocessors before we start handling requests<a name="line.987"></a>
-<span class="sourceLineNo">988</span>    status.setStatus("Initializing master coprocessors");<a name="line.988"></a>
-<span class="sourceLineNo">989</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.989"></a>
-<span class="sourceLineNo">990</span><a name="line.990"></a>
-<span class="sourceLineNo">991</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.991"></a>
-<span class="sourceLineNo">992</span>    InitMetaProcedure initMetaProc = null;<a name="line.992"></a>
-<span class="sourceLineNo">993</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.993"></a>
-<span class="sourceLineNo">994</span>      .isOffline()) {<a name="line.994"></a>
-<span class="sourceLineNo">995</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.995"></a>
-<span class="sourceLineNo">996</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.996"></a>
-<span class="sourceLineNo">997</span>      if (optProc.isPresent()) {<a name="line.997"></a>
-<span class="sourceLineNo">998</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.998"></a>
-<span class="sourceLineNo">999</span>      } else {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span>        initMetaProc = new InitMetaProcedure();<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span>      }<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span>    }<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1006"></a>
+<span class="sourceLineNo">949</span>    // Start RegionServerTracker with listing of servers found with exiting SCPs -- these should<a name="line.949"></a>
+<span class="sourceLineNo">950</span>    // be registered in the deadServers set -- and with the list of servernames out on the<a name="line.950"></a>
+<span class="sourceLineNo">951</span>    // filesystem that COULD BE 'alive' (we'll schedule SCPs for each and let SCP figure it out).<a name="line.951"></a>
+<span class="sourceLineNo">952</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.952"></a>
+<span class="sourceLineNo">953</span>    this.regionServerTracker.start(<a name="line.953"></a>
+<span class="sourceLineNo">954</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.954"></a>
+<span class="sourceLineNo">955</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.955"></a>
+<span class="sourceLineNo">956</span>      walManager.getLiveServersFromWALDir());<a name="line.956"></a>
+<span class="sourceLineNo">957</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.957"></a>
+<span class="sourceLineNo">958</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.958"></a>
+<span class="sourceLineNo">959</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.959"></a>
+<span class="sourceLineNo">960</span>    this.tableStateManager =<a name="line.960"></a>
+<span class="sourceLineNo">961</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.961"></a>
+<span class="sourceLineNo">962</span>        ?<a name="line.962"></a>
+<span class="sourceLineNo">963</span>        new MirroringTableStateManager(this):<a name="line.963"></a>
+<span class="sourceLineNo">964</span>        new TableStateManager(this);<a name="line.964"></a>
+<span class="sourceLineNo">965</span><a name="line.965"></a>
+<span class="sourceLineNo">966</span>    status.setStatus("Initializing ZK system trackers");<a name="line.966"></a>
+<span class="sourceLineNo">967</span>    initializeZKBasedSystemTrackers();<a name="line.967"></a>
+<span class="sourceLineNo">968</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.968"></a>
+<span class="sourceLineNo">969</span>    try {<a name="line.969"></a>
+<span class="sourceLineNo">970</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.970"></a>
+<span class="sourceLineNo">971</span>    } catch (IOException e) {<a name="line.971"></a>
+<span class="sourceLineNo">972</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.972"></a>
+<span class="sourceLineNo">973</span>          + " from file system", e);<a name="line.973"></a>
+<span class="sourceLineNo">974</span>    }<a name="line.974"></a>
+<span class="sourceLineNo">975</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.975"></a>
+<span class="sourceLineNo">976</span>    this.activeMaster = true;<a name="line.976"></a>
+<span class="sourceLineNo">977</span><a name="line.977"></a>
+<span class="sourceLineNo">978</span>    // This is for backwards compatibility<a name="line.978"></a>
+<span class="sourceLineNo">979</span>    // See HBASE-11393<a name="line.979"></a>
+<span class="sourceLineNo">980</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.980"></a>
+<span class="sourceLineNo">981</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.981"></a>
+<span class="sourceLineNo">982</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.982"></a>
+<span class="sourceLineNo">983</span>    tableCFsUpdater.copyTableCFs();<a name="line.983"></a>
+<span class="sourceLineNo">984</span><a name="line.984"></a>
+<span class="sourceLineNo">985</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.985"></a>
+<span class="sourceLineNo">986</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.986"></a>
+<span class="sourceLineNo">987</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.987"></a>
+<span class="sourceLineNo">988</span>      updateConfigurationForQuotasObserver(conf);<a name="line.988"></a>
+<span class="sourceLineNo">989</span>    }<a name="line.989"></a>
+<span class="sourceLineNo">990</span>    // initialize master side coprocessors before we start handling requests<a name="line.990"></a>
+<span class="sourceLineNo">991</span>    status.setStatus("Initializing master coprocessors");<a name="line.991"></a>
+<span class="sourceLineNo">992</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.992"></a>
+<span class="sourceLineNo">993</span><a name="line.993"></a>
+<span class="sourceLineNo">994</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.994"></a>
+<span class="sourceLineNo">995</span>    InitMetaProcedure initMetaProc = null;<a name="line.995"></a>
+<span class="sourceLineNo">996</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.996"></a>
+<span class="sourceLineNo">997</span>      .isOffline()) {<a name="line.997"></a>
+<span class="sourceLineNo">998</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.998"></a>
+<span class="sourceLineNo">999</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.999"></a>
+<span class="sourceLineNo">1000</span>      if (optProc.isPresent()) {<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span>      } else {<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span>        initMetaProc = new InitMetaProcedure();<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span>      }<a name="line.1006"></a>
 <span class="sourceLineNo">1007</span>    }<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span><a name="line.1008"></a>
-<span class="sourceLineNo">1009</span>    // initialize load balancer<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span>    this.balancer.setMasterServices(this);<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span>    this.balancer.initialize();<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span><a name="line.1013"></a>
-<span class="sourceLineNo">1014</span>    // start up all service threads.<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span>    status.setStatus("Initializing master service threads");<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span>    startServiceThreads();<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span>    // wait meta to be initialized after we start procedure executor<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span>    if (initMetaProc != null) {<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span>      initMetaProc.await();<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span>    }<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span>    tableStateManager.start();<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span>    // Wake up this server to check in<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span>    sleeper.skipSleepCycle();<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span>    // Wait for region servers to report in.<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span>    String statusStr = "Wait for region servers to report in";<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span>    status.setStatus(statusStr);<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span>    LOG.info(Objects.toString(status));<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span>    waitForRegionServers(status);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span><a name="line.1032"></a>
-<span class="sourceLineNo">1033</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span>    if (isStopped()) {<a name="line.1034"></a>
-<span class="sourceLineNo">1035</span>      return;<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span>    }<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span><a name="line.1037"></a>
-<span class="sourceLineNo">1038</span>    //Initialize after meta as it scans meta<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span>    if (favoredNodesManager != null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span>      snapshotOfRegionAssignment.initialize();<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span>    }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span><a name="line.1045"></a>
-<span class="sourceLineNo">1046</span>    // Fix up assignment manager status<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span>    status.setStatus("Starting assignment manager");<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span>    this.assignmentManager.joinCluster();<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span><a name="line.1049"></a>
-<span class="sourceLineNo">1050</span>    // set cluster status again after user regions are assigned<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1051"></a>
+<span class="sourceLineNo">1008</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span>    }<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span><a name="line.1011"></a>
+<span class="sourceLineNo">1012</span>    // initialize load balancer<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span>    this.balancer.setMasterServices(this);<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span>    this.balancer.initialize();<a name="line.1015"></a>
+<span class="sourceLineNo">1016</span><a name="line.1016"></a>
+<span class="sourceLineNo">1017</span>    // start up all service threads.<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span>    status.setStatus("Initializing master service threads");<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span>    startServiceThreads();<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span>    // wait meta to be initialized after we start procedure executor<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span>    if (initMetaProc != null) {<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span>      initMetaProc.await();<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span>    }<a name="line.1023"></a>
+<span class="sourceLineNo">1024</span>    tableStateManager.start();<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span>    // Wake up this server to check in<a name="line.1025"></a>
+<span class="sourceLineNo">1026</span>    sleeper.skipSleepCycle();<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span><a name="line.1027"></a>
+<span class="sourceLineNo">1028</span>    // Wait for region servers to report in.<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span>    String statusStr = "Wait for region servers to report in";<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span>    status.setStatus(statusStr);<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span>    LOG.info(Objects.toString(status));<a name="line.1033"></a>
+<span class="sourceLineNo">1034</span>    waitForRegionServers(status);<a name="line.1034"></a>
+<span class="sourceLineNo">1035</span><a name="line.1035"></a>
+<span class="sourceLineNo">1036</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1036"></a>
+<span class="sourceLineNo">1037</span>    if (isStopped()) {<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span>      return;<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span>    }<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span><a name="line.1040"></a>
+<span class="sourceLineNo">1041</span>    //Initialize after meta as it scans meta<a name="line.1041"></a>
+<span class="sourceLineNo">1042</span>    if (favoredNodesManager != null) {<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span>      snapshotOfRegionAssignment.initialize();<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span>    }<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span><a name="line.1048"></a>
+<span class="sourceLineNo">1049</span>    // Fix up assignment manager status<a name="line.1049"></a>
+<span class="sourceLineNo">1050</span>    status.setStatus("Starting assignment manager");<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span>    this.assignmentManager.joinCluster();<a name="line.1051"></a>
 <span class="sourceLineNo">1052</span><a name="line.1052"></a>
-<span class="sourceLineNo">1053</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span>    this.balancerChore = new BalancerChore(this);<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span>    this.serverManager.startChore();<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span>    status.setStatus("Starting cluster schema service");<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span>    initClusterSchemaService();<a name="line.1066"></a>
+<span class="sourceLineNo">1053</span>    // set cluster status again after user regions are assigned<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span><a name="line.1055"></a>
+<span class="sourceLineNo">1056</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1057"></a>
+<span class="sourceLineNo">1058</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span>    this.balancerChore = new BalancerChore(this);<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1065"></a>
+<span class="sourceLineNo">1066</span>    this.serverManager.startChore();<a name="line.1066"></a>
 <span class="sourceLineNo">1067</span><a name="line.1067"></a>
-<span class="sourceLineNo">1068</span>    if (this.cpHost != null) {<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span>      try {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span>        this.cpHost.preMasterInitialization();<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span>      } catch (IOException e) {<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span>      }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span>    }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span><a name="line.1075"></a>
-<span class="sourceLineNo">1076</span>    status.markComplete("Initialization successful");<a name="line.1076"></a>
-<span class="sourceLineNo">1077</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span>    configurationManager.registerObserver(this.balancer);<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1082"></a>
-<span class="sourceLineNo">1083</span>    // Set master as 'initialized'.<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span>    setInitialized(true);<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span>    status.setStatus("Assign meta replicas");<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span>    metaBootstrap.assignMetaReplicas();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span>    status.setStatus("Starting quota manager");<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span>    initQuotaManager();<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span>      // Create the quota snapshot notifier<a name="line.1092"></a>
-<span class="sourceLineNo">1093</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span><a name="line.1098"></a>
-<span class="sourceLineNo">1099</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1100"></a>
-<span class="sourceLineNo">1101</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span>    }<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span><a name="line.1103"></a>
-<span class="sourceLineNo">1104</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span>    // master initialization. See HBASE-5916.<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span><a name="line.1108"></a>
-<span class="sourceLineNo">1109</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span><a name="line.1112"></a>
-<span class="sourceLineNo">1113</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span>    initMobCleaner();<a name="line.1114"></a>
+<span class="sourceLineNo">1068</span>    status.setStatus("Starting cluster schema service");<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span>    initClusterSchemaService();<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span><a name="line.1070"></a>
+<span class="sourceLineNo">1071</span>    if (this.cpHost != null) {<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span>      try {<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span>        this.cpHost.preMasterInitialization();<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span>      } catch (IOException e) {<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1075"></a>
+<span class="sourceLineNo">1076</span>      }<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span>    }<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span><a name="line.1078"></a>
+<span class="sourceLineNo">1079</span>    status.markComplete("Initialization successful");<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1081"></a>
+<span class="sourceLineNo">1082</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span>    configurationManager.registerObserver(this.balancer);<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1085"></a>
+<span class="sourceLineNo">1086</span>    // Set master as 'initialized'.<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span>    setInitialized(true);<a name="line.1087"></a>
+<span class="sourceLineNo">1088</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span>    status.setStatus("Assign meta replicas");<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span>    metaBootstrap.assignMetaReplicas();<a name="line.1091"></a>
+<span class="sourceLineNo">1092</span>    status.setStatus("Starting quota manager");<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span>    initQuotaManager();<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span>      // Create the quota snapshot notifier<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1097"></a>
+<span class="sourceLineNo">1098</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1099"></a>
+<span class="sourceLineNo">1100</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span><a name="line.1101"></a>
+<span class="sourceLineNo">1102</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span>    }<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span><a name="line.1106"></a>
+<span class="sourceLineNo">1107</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1107"></a>
+<span class="sourceLineNo">1108</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span>    // master initialization. See HBASE-5916.<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span><a name="line.1111"></a>
+<span class="sourceLineNo">1112</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1112"></a>
+<span class="sourceLineNo">1113</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1114"></a>
 <span class="sourceLineNo">1115</span><a name="line.1115"></a>
-<span class="sourceLineNo">1116</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1116"></a>
-<span class="sourceLineNo">1117</span>    if (this.cpHost != null) {<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span>      // don't let cp initialization errors kill the master<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span>      try {<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span>        this.cpHost.postStartMaster();<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span>      } catch (IOException ioe) {<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span>      }<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span>    }<a name="line.1124"></a>
-<span class="sourceLineNo">1125</span><a name="line.1125"></a>
-<span class="sourceLineNo">1126</span>    zombieDetector.interrupt();<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span><a name="line.1127"></a>
-<span class="sourceLineNo">1128</span>    /*<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span>     * in activeMasterManager thread, it should be fine.<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span>     */<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span>    long start = System.currentTimeMillis();<a name="line.1132"></a>
-<span class="sourceLineNo">1133</span>    this.balancer.postMasterStartupInitialize();<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span>    if (LOG.isDebugEnabled()) {<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span>    }<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span>  }<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span><a name="line.1139"></a>
-<span class="sourceLineNo">1140</span>  /**<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span>   */<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span>  @VisibleForTesting<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span>    if (!conf.getBoolean(<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1148"></a>
-<span class="sourceLineNo">1149</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span>      return;<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span>    }<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1152"></a>
-<span class="sourceLineNo">1153</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span>    if (length &gt; 0) {<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span>    }<a name="line.1157"></a>
-<span class="sourceLineNo">1158</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1159"></a>
-<span class="sourceLineNo">1160</span>  }<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span><a name="line.1161"></a>
-<span class="sourceLineNo">1162</span>  private void initMobCleaner() {<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1163"></a>
-<span class="sourceLineNo">1164</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span><a name="line.1165"></a>
-<span class="sourceLineNo">1166</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1167"></a>
-<span class="sourceLineNo">1168</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1168"></a>
-<span class="sourceLineNo">1169</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span>    } else {<a name="line.1171"></a>
-<span class="sourceLineNo">1172</span>      LOG<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span>    }<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>  }<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span><a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>  /**<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>   * &lt;p&gt;<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>   * &lt;/p&gt;<a name="line.1181"></a>
+<span class="sourceLineNo">1116</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span>    initMobCleaner();<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span><a name="line.1118"></a>
+<span class="sourceLineNo">1119</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span>    if (this.cpHost != null) {<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span>      // don't let cp initialization errors kill the master<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span>      try {<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span>        this.cpHost.postStartMaster();<a name="line.1123"></a>
+<span class="sourceLineNo">1124</span>      } catch (IOException ioe) {<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span>      }<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span>    }<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span><a name="line.1128"></a>
+<span class="sourceLineNo">1129</span>    zombieDetector.interrupt();<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span><a name="line.1130"></a>
+<span class="sourceLineNo">1131</span>    /*<a name="line.1131"></a>
+<span class="sourceLineNo">1132</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span>     * in activeMasterManager thread, it should be fine.<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span>     */<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span>    long start = System.currentTimeMillis();<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span>    this.balancer.postMasterStartupInitialize();<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span>    if (LOG.isDebugEnabled()) {<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span>    }<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span>  }<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span><a name="line.1142"></a>
+<span class="sourceLineNo">1143</span>  /**<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span>   */<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span>  @VisibleForTesting<a name="line.1147"></a>
+<span class="sourceLineNo">1148</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span>    if (!conf.getBoolean(<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1151"></a>
+<span class="sourceLineNo">1152</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span>      return;<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span>    }<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1155"></a>
+<span class="sourceLineNo">1156</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span>    if (length &gt; 0) {<a name="line.1158"></a>
+<span class="sourceLineNo">1159</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span>    }<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1162"></a>
+<span class="sourceLineNo">1163</span>  }<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span><a name="line.1164"></a>
+<span class="sourceLineNo">1165</span>  private void initMobCleaner() {<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1166"></a>
+<span class="sourceLineNo">1167</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span><a name="line.1168"></a>
+<span class="sourceLineNo">1169</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1170"></a>
+<span class="sourceLineNo">1171</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span>    } else {<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span>      LOG<a name="line.1175"></a>
+<span class="sourceLineNo">1176</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>    }<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>  }<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span><a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>  /**<a name="line.1181"></a>
 <span class="sourceLineNo">1182</span>   * &lt;p&gt;<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>   * Will be overridden in tests.<a name="line.1183"></a>
+<span class="sourceLineNo">1183</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1183"></a>
 <span class="sourceLineNo">1184</span>   * &lt;/p&gt;<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>   */<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>  @VisibleForTesting<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>    return new MasterMetaBootstrap(this);<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>  }<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span><a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>  /**<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>   * &lt;p&gt;<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>   * Create a {@link ServerManager} instance.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>   * &lt;/p&gt;<a name="line.1196"></a>
+<span class="sourceLineNo">1185</span>   * &lt;p&gt;<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>   * Will be overridden in tests.<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>   * &lt;/p&gt;<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>   */<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>  @VisibleForTesting<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span>    return new MasterMetaBootstrap(this);<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>  }<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span><a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>  /**<a name="line.1196"></a>
 <span class="sourceLineNo">1197</span>   * &lt;p&gt;<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>   * Will be overridden in tests.<a name="line.1198"></a>
+<span class="sourceLineNo">1198</span>   * Create a {@link ServerManager} instance.<a name="line.1198"></a>
 <span class="sourceLineNo">1199</span>   * &lt;/p&gt;<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>   */<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>  @VisibleForTesting<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>    // w/ a mocked up ServerManager.<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>    setupClusterConnection();<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    return new ServerManager(master);<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span>  }<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span><a name="line.1208"></a>
-<span class="sourceLineNo">1209</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>      throws IOException, InterruptedException {<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span>    this.serverManager.waitForRegionServers(status);<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>  }<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span><a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>  // Will be overridden in tests<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>  @VisibleForTesting<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    this.clusterSchemaService.startAsync();<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span>    try {<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span>    } catch (TimeoutException toe) {<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>    }<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>  }<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span><a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>  private void initQuotaManager() throws IOException {<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>    quotaManager.start();<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>    this.quotaManager = quotaManager;<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>  }<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span><a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>    return notifier;<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span>  }<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span><a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>  boolean isCatalogJanitorEnabled() {<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>    return catalogJanitorChore != null ?<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>  }<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span><a name="line.1244"></a>
-<span class="sourceLineNo">1245</span>  boolean isCleanerChoreEnabled() {<a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1246"></a>
+<span class="sourceLineNo">1200</span>   * &lt;p&gt;<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>   * Will be overridden in tests.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>   * &lt;/p&gt;<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>   */<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>  @VisibleForTesting<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>    // w/ a mocked up ServerManager.<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span>    setupClusterConnection();<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>    return new ServerManager(master);<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>  }<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span><a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>      throws IOException, InterruptedException {<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>    this.serverManager.waitForRegionServers(status);<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>  }<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span><a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>  // Will be overridden in tests<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span>  @VisibleForTesting<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>    this.clusterSchemaService.startAsync();<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>    try {<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>    } catch (TimeoutException toe) {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>    }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>  }<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span><a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>  private void initQuotaManager() throws IOException {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>    quotaManager.start();<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>    this.quotaManager = quotaManager;<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>  }<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span><a name="line.1236"></a>
+<span class="sourceLineNo">1237</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>    return notifier;<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>  }<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span><a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>  boolean isCatalogJanitorEnabled() {<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span>    return catalogJanitorChore != null ?<a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>  }<a name="line.1246"></a>
 <span class="sourceLineNo">1247</span><a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>    if (hfileCleaner != null) {<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span>    }<a name="line.1250"></a>
-<span class="sourceLineNo">1251</span><a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>    if (logCleaner != null) {<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span>    }<a name="line.1254"></a>
-<span class="sourceLineNo">1255</span><a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>  }<a name="line.1257"></a>
+<span class="sourceLineNo">1248</span>  boolean isCleanerChoreEnabled() {<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1249"></a>
+<span class="sourceLineNo">1250</span><a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>    if (hfileCleaner != null) {<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span>    }<a name="line.1253"></a>
+<span class="sourceLineNo">1254</span><a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>    if (logCleaner != null) {<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span>    }<a name="line.1257"></a>
 <span class="sourceLineNo">1258</span><a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>  @Override<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>  public TableDescriptors getTableDescriptors() {<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>    return this.tableDescriptors;<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>  }<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span><a name="line.1263"></a>
-<span class="sourceLineNo">1264</span>  @Override<a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>  public ServerManager getServerManager() {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    return this.serverManager;<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>  }<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span><a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>  @Override<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>    return this.fileSystemManager;<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span>  }<a name="line.1272"></a>
-<span class="sourceLineNo">1273</span><a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>  @Override<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>  public MasterWalManager getMasterWalManager() {<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>    return this.walManager;<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>  }<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span><a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>  @Override<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>  public TableStateManager getTableStateManager() {<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>    return tableStateManager;<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span>  }<a name="line.1282"></a>
-<span class="sourceLineNo">1283</span><a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>  /*<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>   * then they just die with a logged message.  This should be fine because<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>   *  need to install an unexpected exception handler.<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>   */<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>  private void startServiceThreads() throws IOException{<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span>   // Start the executor service pools<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span><a name="line.1303"></a>
-<span class="sourceLineNo">1304</span>   // We depend on there being only one instance of this executor running<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>   // tables.<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   // AccessController#postCompletedCreateTableAction<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   startProcedureExecutor();<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span><a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>    // Initial cleaner chore<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>    CleanerChore.initChorePool(conf);<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>   // Start log cleaner thread<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>   this.logCleaner =<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>      new LogCleaner(cleanerInterval,<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>         getMasterWalManager().getOldLogDir());<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span><a name="line.1321"></a>
-<span class="sourceLineNo">1322</span>    // start the hfile archive cleaner thread<a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>    params.put(MASTER, this);<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>        .getFileSystem(), archiveDir, params);<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span><a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>      replicationPeerManager);<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1332"></a>
-<span class="sourceLineNo">1333</span><a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>    serviceStarted = true;<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>    if (LOG.isTraceEnabled()) {<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>      LOG.trace("Started service threads");<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>    }<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>  }<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span><a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>  @Override<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>  protected void stopServiceThreads() {<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>    if (masterJettyServer != null) {<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>      LOG.info("Stopping master jetty server");<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>      try {<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>        masterJettyServer.stop();<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>      } catch (Exception e) {<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>      }<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span>    }<a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>    stopChores();<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>    if (this.mobCompactThread != null) {<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>      this.mobCompactThread.close();<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>    }<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>    super.stopServiceThreads();<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>    CleanerChore.shutDownChorePool();<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span><a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>    LOG.debug("Stopping service threads");<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span><a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>    if (this.quotaManager != null) {<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>      this.quotaManager.stop();<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>    }<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span><a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>    if (this.activeMasterManager != null) {<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>      this.activeMasterManager.stop();<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>    }<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span>    if (this.serverManager != null) {<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      this.serverManager.stop();<a name="line.1367"></a>
+<span class="sourceLineNo">1259</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>  }<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span><a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>  @Override<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span>  public TableDescriptors getTableDescriptors() {<a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>    return this.tableDescriptors;<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>  }<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span><a name="line.1266"></a>
+<span class="sourceLineNo">1267</span>  @Override<a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>  public ServerManager getServerManager() {<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>    return this.serverManager;<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>  }<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span><a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>  @Override<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span>    return this.fileSystemManager;<a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>  }<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span><a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>  @Override<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>  public MasterWalManager getMasterWalManager() {<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    return this.walManager;<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>  }<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span><a name="line.1281"></a>
+<span class="sourceLineNo">1282</span>  @Override<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>  public TableStateManager getTableStateManager() {<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>    return tableStateManager;<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>  }<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span><a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>  /*<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>   * then they just die with a logged message.  This should be fine because<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span>   *  need to install an unexpected exception handler.<a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>   */<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>  private void startServiceThreads() throws IOException{<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   // Start the executor service pools<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span><a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   // We depend on there being only one instance of this executor running<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   // tables.<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>   // AccessController#postCompletedCreateTableAction<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>   startProcedureExecutor();<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span><a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>    // Initial cleaner chore<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>    CleanerChore.initChorePool(conf);<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>   // Start log cleaner thread<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>   this.logCleaner =<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>      new LogCleaner(cleanerInterval,<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>         getMasterWalManager().getOldLogDir());<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span><a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>    // start the hfile archive cleaner thread<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>    params.put(MASTER, this);<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>        .getFileSystem(), archiveDir, params);<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1331"></a>
+<span class="sourceLineNo">1332</span><a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>      replicationPeerManager);<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span><a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>    serviceStarted = true;<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>    if (LOG.isTraceEnabled()) {<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>      LOG.trace("Started service threads");<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>    }<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>  }<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span><a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>  @Override<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>  protected void stopServiceThreads() {<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>    if (masterJettyServer != null) {<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>      LOG.info("Stopping master jetty server");<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>      try {<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span>        masterJettyServer.stop();<a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>      } catch (Exception e) {<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>      }<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>    }<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span>    stopChores();<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>    if (this.mobCompactThread != null) {<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>      this.mobCompactThread.close();<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span>    }<a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>    super.stopServiceThreads();<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>    CleanerChore.shutDownChorePool();<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span><a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>    LOG.debug("Stopping service threads");<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span><a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>    if (this.quotaManager != null) {<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>      this.quotaManager.stop();<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>    }<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span><a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>    if (this.activeMasterManager != null) {<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>      this.activeMasterManager.stop();<a name="line.1367"></a>
 <span class="sourceLineNo">1368</span>    }<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>    if (this.assignmentManager != null) {<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>      this.assignmentManager.stop();<a name="line.1370"></a>
+<span class="sourceLineNo">1369</span>    if (this.serverManager != null) {<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      this.serverManager.stop();<a name="line.1370"></a>
 <span class="sourceLineNo">1371</span>    }<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span><a name="line.1372"></a>
-<span class="sourceLineNo">1373</span>    stopProcedureExecutor();<a name="line.1373"></a>
-<span class="sourceLineNo">1374</span><a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>    if (this.walManager != null) {<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>      this.walManager.stop();<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>    }<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>    if (this.fileSystemManager != null) {<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>      this.fileSystemManager.stop();<a name="line.1379"></a>
+<span class="sourceLineNo">1372</span>    if (this.assignmentManager != null) {<a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>      this.assignmentManager.stop();<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>    }<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span><a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>    stopProcedureExecutor();<a name="line.1376"></a>
+<span class="sour

<TRUNCATED>

[10/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html
index 681e263..7cdf584 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html
@@ -182,7 +182,7 @@
 <span class="sourceLineNo">174</span>      }<a name="line.174"></a>
 <span class="sourceLineNo">175</span>      FileStatus[] logfiles = FSUtils.listStatus(fs, logDir, filter);<a name="line.175"></a>
 <span class="sourceLineNo">176</span>      if (logfiles == null || logfiles.length == 0) {<a name="line.176"></a>
-<span class="sourceLineNo">177</span>        LOG.info(logDir + " is empty dir, no logs to split");<a name="line.177"></a>
+<span class="sourceLineNo">177</span>        LOG.info("{} dir is empty, no logs to split.", logDir);<a name="line.177"></a>
 <span class="sourceLineNo">178</span>      } else {<a name="line.178"></a>
 <span class="sourceLineNo">179</span>        Collections.addAll(fileStatus, logfiles);<a name="line.179"></a>
 <span class="sourceLineNo">180</span>      }<a name="line.180"></a>
@@ -243,426 +243,430 @@
 <span class="sourceLineNo">235</span>      PathFilter filter) throws IOException {<a name="line.235"></a>
 <span class="sourceLineNo">236</span>    MonitoredTask status = TaskMonitor.get().createStatus("Doing distributed log split in " +<a name="line.236"></a>
 <span class="sourceLineNo">237</span>      logDirs + " for serverName=" + serverNames);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    status.setStatus("Checking directory contents...");<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.240"></a>
-<span class="sourceLineNo">241</span>    LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.241"></a>
-<span class="sourceLineNo">242</span>      " for " + serverNames);<a name="line.242"></a>
-<span class="sourceLineNo">243</span>    long t = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
-<span class="sourceLineNo">244</span>    long totalSize = 0;<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    TaskBatch batch = new TaskBatch();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    for (FileStatus lf : logfiles) {<a name="line.246"></a>
-<span class="sourceLineNo">247</span>      // TODO If the log file is still being written to - which is most likely<a name="line.247"></a>
-<span class="sourceLineNo">248</span>      // the case for the last log file - then its length will show up here<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      // as zero. The size of such a file can only be retrieved after<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      // recover-lease is done. totalSize will be under in most cases and the<a name="line.250"></a>
-<span class="sourceLineNo">251</span>      // metrics that it drives will also be under-reported.<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      totalSize += lf.getLen();<a name="line.252"></a>
-<span class="sourceLineNo">253</span>      String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.253"></a>
-<span class="sourceLineNo">254</span>      if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.254"></a>
-<span class="sourceLineNo">255</span>        throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.255"></a>
-<span class="sourceLineNo">256</span>      }<a name="line.256"></a>
-<span class="sourceLineNo">257</span>    }<a name="line.257"></a>
-<span class="sourceLineNo">258</span>    waitForSplittingCompletion(batch, status);<a name="line.258"></a>
-<span class="sourceLineNo">259</span><a name="line.259"></a>
-<span class="sourceLineNo">260</span>    if (batch.done != batch.installed) {<a name="line.260"></a>
-<span class="sourceLineNo">261</span>      batch.isDead = true;<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.262"></a>
-<span class="sourceLineNo">263</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.263"></a>
-<span class="sourceLineNo">264</span>          + " but only " + batch.done + " done");<a name="line.264"></a>
-<span class="sourceLineNo">265</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      status.abort(msg);<a name="line.266"></a>
-<span class="sourceLineNo">267</span>      throw new IOException(msg);<a name="line.267"></a>
-<span class="sourceLineNo">268</span>    }<a name="line.268"></a>
-<span class="sourceLineNo">269</span>    for (Path logDir : logDirs) {<a name="line.269"></a>
-<span class="sourceLineNo">270</span>      status.setStatus("Cleaning up log directory...");<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      try {<a name="line.272"></a>
-<span class="sourceLineNo">273</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.273"></a>
-<span class="sourceLineNo">274</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.274"></a>
-<span class="sourceLineNo">275</span>        }<a name="line.275"></a>
-<span class="sourceLineNo">276</span>      } catch (IOException ioe) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.277"></a>
-<span class="sourceLineNo">278</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.278"></a>
-<span class="sourceLineNo">279</span>          LOG.warn("Returning success without actually splitting and "<a name="line.279"></a>
-<span class="sourceLineNo">280</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.280"></a>
-<span class="sourceLineNo">281</span>              + Arrays.toString(files), ioe);<a name="line.281"></a>
-<span class="sourceLineNo">282</span>        } else {<a name="line.282"></a>
-<span class="sourceLineNo">283</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.283"></a>
-<span class="sourceLineNo">284</span>        }<a name="line.284"></a>
-<span class="sourceLineNo">285</span>      }<a name="line.285"></a>
-<span class="sourceLineNo">286</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.286"></a>
-<span class="sourceLineNo">287</span>    }<a name="line.287"></a>
-<span class="sourceLineNo">288</span>    String msg =<a name="line.288"></a>
-<span class="sourceLineNo">289</span>        "finished splitting (more than or equal to) " + totalSize + " bytes in " + batch.installed<a name="line.289"></a>
-<span class="sourceLineNo">290</span>            + " log files in " + logDirs + " in "<a name="line.290"></a>
-<span class="sourceLineNo">291</span>            + (EnvironmentEdgeManager.currentTime() - t) + "ms";<a name="line.291"></a>
-<span class="sourceLineNo">292</span>    status.markComplete(msg);<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    LOG.info(msg);<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    return totalSize;<a name="line.294"></a>
-<span class="sourceLineNo">295</span>  }<a name="line.295"></a>
-<span class="sourceLineNo">296</span><a name="line.296"></a>
-<span class="sourceLineNo">297</span>  /**<a name="line.297"></a>
-<span class="sourceLineNo">298</span>   * Add a task entry to coordination if it is not already there.<a name="line.298"></a>
-<span class="sourceLineNo">299</span>   * @param taskname the path of the log to be split<a name="line.299"></a>
-<span class="sourceLineNo">300</span>   * @param batch the batch this task belongs to<a name="line.300"></a>
-<span class="sourceLineNo">301</span>   * @return true if a new entry is created, false if it is already there.<a name="line.301"></a>
-<span class="sourceLineNo">302</span>   */<a name="line.302"></a>
-<span class="sourceLineNo">303</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.303"></a>
-<span class="sourceLineNo">304</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.304"></a>
-<span class="sourceLineNo">305</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.305"></a>
-<span class="sourceLineNo">306</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.306"></a>
-<span class="sourceLineNo">307</span>    if (oldtask == null) {<a name="line.307"></a>
-<span class="sourceLineNo">308</span>      // publish the task in the coordination engine<a name="line.308"></a>
-<span class="sourceLineNo">309</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.309"></a>
-<span class="sourceLineNo">310</span>      return true;<a name="line.310"></a>
-<span class="sourceLineNo">311</span>    }<a name="line.311"></a>
-<span class="sourceLineNo">312</span>    return false;<a name="line.312"></a>
-<span class="sourceLineNo">313</span>  }<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.315"></a>
-<span class="sourceLineNo">316</span>    synchronized (batch) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.317"></a>
-<span class="sourceLineNo">318</span>        try {<a name="line.318"></a>
-<span class="sourceLineNo">319</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.319"></a>
-<span class="sourceLineNo">320</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.320"></a>
-<span class="sourceLineNo">321</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.321"></a>
-<span class="sourceLineNo">322</span>          int actual = activeTasks(batch);<a name="line.322"></a>
-<span class="sourceLineNo">323</span>          if (remaining != actual) {<a name="line.323"></a>
-<span class="sourceLineNo">324</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.324"></a>
-<span class="sourceLineNo">325</span>          }<a name="line.325"></a>
-<span class="sourceLineNo">326</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.326"></a>
-<span class="sourceLineNo">327</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.327"></a>
-<span class="sourceLineNo">328</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.328"></a>
-<span class="sourceLineNo">329</span>                + remainingTasks);<a name="line.329"></a>
-<span class="sourceLineNo">330</span>          }<a name="line.330"></a>
-<span class="sourceLineNo">331</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span>            LOG.warn("No more task remaining, splitting "<a name="line.332"></a>
-<span class="sourceLineNo">333</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.333"></a>
-<span class="sourceLineNo">334</span>                + ", active tasks in map " + actual);<a name="line.334"></a>
-<span class="sourceLineNo">335</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.335"></a>
-<span class="sourceLineNo">336</span>              return;<a name="line.336"></a>
-<span class="sourceLineNo">337</span>            }<a name="line.337"></a>
-<span class="sourceLineNo">338</span>          }<a name="line.338"></a>
-<span class="sourceLineNo">339</span>          batch.wait(100);<a name="line.339"></a>
-<span class="sourceLineNo">340</span>          if (server.isStopped()) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.341"></a>
-<span class="sourceLineNo">342</span>            return;<a name="line.342"></a>
-<span class="sourceLineNo">343</span>          }<a name="line.343"></a>
-<span class="sourceLineNo">344</span>        } catch (InterruptedException e) {<a name="line.344"></a>
-<span class="sourceLineNo">345</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.345"></a>
-<span class="sourceLineNo">346</span>          Thread.currentThread().interrupt();<a name="line.346"></a>
-<span class="sourceLineNo">347</span>          return;<a name="line.347"></a>
-<span class="sourceLineNo">348</span>        }<a name="line.348"></a>
-<span class="sourceLineNo">349</span>      }<a name="line.349"></a>
-<span class="sourceLineNo">350</span>    }<a name="line.350"></a>
-<span class="sourceLineNo">351</span>  }<a name="line.351"></a>
-<span class="sourceLineNo">352</span><a name="line.352"></a>
-<span class="sourceLineNo">353</span>  @VisibleForTesting<a name="line.353"></a>
-<span class="sourceLineNo">354</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.354"></a>
-<span class="sourceLineNo">355</span>    return tasks;<a name="line.355"></a>
-<span class="sourceLineNo">356</span>  }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span>  private int activeTasks(final TaskBatch batch) {<a name="line.358"></a>
-<span class="sourceLineNo">359</span>    int count = 0;<a name="line.359"></a>
-<span class="sourceLineNo">360</span>    for (Task t : tasks.values()) {<a name="line.360"></a>
-<span class="sourceLineNo">361</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.361"></a>
-<span class="sourceLineNo">362</span>        count++;<a name="line.362"></a>
-<span class="sourceLineNo">363</span>      }<a name="line.363"></a>
-<span class="sourceLineNo">364</span>    }<a name="line.364"></a>
-<span class="sourceLineNo">365</span>    return count;<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span>  }<a name="line.367"></a>
-<span class="sourceLineNo">368</span><a name="line.368"></a>
-<span class="sourceLineNo">369</span>  /**<a name="line.369"></a>
-<span class="sourceLineNo">370</span>   * @param path<a name="line.370"></a>
-<span class="sourceLineNo">371</span>   * @param batch<a name="line.371"></a>
-<span class="sourceLineNo">372</span>   * @return null on success, existing task on error<a name="line.372"></a>
-<span class="sourceLineNo">373</span>   */<a name="line.373"></a>
-<span class="sourceLineNo">374</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span>    Task oldtask;<a name="line.375"></a>
-<span class="sourceLineNo">376</span>    // batch.installed is only changed via this function and<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    // a single thread touches batch.installed.<a name="line.377"></a>
-<span class="sourceLineNo">378</span>    Task newtask = new Task();<a name="line.378"></a>
-<span class="sourceLineNo">379</span>    newtask.batch = batch;<a name="line.379"></a>
-<span class="sourceLineNo">380</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    if (oldtask == null) {<a name="line.381"></a>
-<span class="sourceLineNo">382</span>      batch.installed++;<a name="line.382"></a>
-<span class="sourceLineNo">383</span>      return null;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    }<a name="line.384"></a>
-<span class="sourceLineNo">385</span>    // new task was not used.<a name="line.385"></a>
-<span class="sourceLineNo">386</span>    synchronized (oldtask) {<a name="line.386"></a>
-<span class="sourceLineNo">387</span>      if (oldtask.isOrphan()) {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>        if (oldtask.status == SUCCESS) {<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          // The task is already done. Do not install the batch for this<a name="line.389"></a>
-<span class="sourceLineNo">390</span>          // task because it might be too late for setDone() to update<a name="line.390"></a>
-<span class="sourceLineNo">391</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.391"></a>
-<span class="sourceLineNo">392</span>          // this task to complete.<a name="line.392"></a>
-<span class="sourceLineNo">393</span>          return (null);<a name="line.393"></a>
-<span class="sourceLineNo">394</span>        }<a name="line.394"></a>
-<span class="sourceLineNo">395</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.395"></a>
-<span class="sourceLineNo">396</span>          oldtask.batch = batch;<a name="line.396"></a>
-<span class="sourceLineNo">397</span>          batch.installed++;<a name="line.397"></a>
-<span class="sourceLineNo">398</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.398"></a>
-<span class="sourceLineNo">399</span>          return null;<a name="line.399"></a>
-<span class="sourceLineNo">400</span>        }<a name="line.400"></a>
-<span class="sourceLineNo">401</span>        while (oldtask.status == FAILURE) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.402"></a>
-<span class="sourceLineNo">403</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.403"></a>
-<span class="sourceLineNo">404</span>          try {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>            oldtask.wait();<a name="line.405"></a>
-<span class="sourceLineNo">406</span>          } catch (InterruptedException e) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span>            Thread.currentThread().interrupt();<a name="line.407"></a>
-<span class="sourceLineNo">408</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.408"></a>
-<span class="sourceLineNo">409</span>            // fall through to return failure<a name="line.409"></a>
-<span class="sourceLineNo">410</span>            break;<a name="line.410"></a>
-<span class="sourceLineNo">411</span>          }<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        }<a name="line.412"></a>
-<span class="sourceLineNo">413</span>        if (oldtask.status != DELETED) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span>          LOG.warn("Failure because previously failed task"<a name="line.414"></a>
-<span class="sourceLineNo">415</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.415"></a>
-<span class="sourceLineNo">416</span>          return oldtask;<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        }<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        // reinsert the newTask and it must succeed this time<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        if (t == null) {<a name="line.420"></a>
-<span class="sourceLineNo">421</span>          batch.installed++;<a name="line.421"></a>
-<span class="sourceLineNo">422</span>          return null;<a name="line.422"></a>
-<span class="sourceLineNo">423</span>        }<a name="line.423"></a>
-<span class="sourceLineNo">424</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        assert false : "Deleted task still present in tasks map";<a name="line.425"></a>
-<span class="sourceLineNo">426</span>        return t;<a name="line.426"></a>
-<span class="sourceLineNo">427</span>      }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.428"></a>
-<span class="sourceLineNo">429</span>      return oldtask;<a name="line.429"></a>
-<span class="sourceLineNo">430</span>    }<a name="line.430"></a>
-<span class="sourceLineNo">431</span>  }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span>  public void stop() {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (choreService != null) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      choreService.shutdown();<a name="line.435"></a>
-<span class="sourceLineNo">436</span>    }<a name="line.436"></a>
-<span class="sourceLineNo">437</span>    if (timeoutMonitor != null) {<a name="line.437"></a>
-<span class="sourceLineNo">438</span>      timeoutMonitor.cancel(true);<a name="line.438"></a>
-<span class="sourceLineNo">439</span>    }<a name="line.439"></a>
-<span class="sourceLineNo">440</span>  }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>  void handleDeadWorker(ServerName workerName) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.443"></a>
-<span class="sourceLineNo">444</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.444"></a>
-<span class="sourceLineNo">445</span>    synchronized (deadWorkersLock) {<a name="line.445"></a>
-<span class="sourceLineNo">446</span>      if (deadWorkers == null) {<a name="line.446"></a>
-<span class="sourceLineNo">447</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>      }<a name="line.448"></a>
-<span class="sourceLineNo">449</span>      deadWorkers.add(workerName);<a name="line.449"></a>
-<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    LOG.info("dead splitlog worker " + workerName);<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  }<a name="line.452"></a>
-<span class="sourceLineNo">453</span><a name="line.453"></a>
-<span class="sourceLineNo">454</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.454"></a>
-<span class="sourceLineNo">455</span>    synchronized (deadWorkersLock) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span>      if (deadWorkers == null) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      }<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      deadWorkers.addAll(serverNames);<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.461"></a>
-<span class="sourceLineNo">462</span>  }<a name="line.462"></a>
-<span class="sourceLineNo">463</span><a name="line.463"></a>
-<span class="sourceLineNo">464</span>  /**<a name="line.464"></a>
-<span class="sourceLineNo">465</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.465"></a>
-<span class="sourceLineNo">466</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.466"></a>
-<span class="sourceLineNo">467</span>   * &lt;p&gt;<a name="line.467"></a>
-<span class="sourceLineNo">468</span>   * All access is synchronized.<a name="line.468"></a>
-<span class="sourceLineNo">469</span>   */<a name="line.469"></a>
-<span class="sourceLineNo">470</span>  @InterfaceAudience.Private<a name="line.470"></a>
-<span class="sourceLineNo">471</span>  public static class TaskBatch {<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    public int installed = 0;<a name="line.472"></a>
-<span class="sourceLineNo">473</span>    public int done = 0;<a name="line.473"></a>
-<span class="sourceLineNo">474</span>    public int error = 0;<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    public volatile boolean isDead = false;<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span>    @Override<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    public String toString() {<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.479"></a>
-<span class="sourceLineNo">480</span>    }<a name="line.480"></a>
-<span class="sourceLineNo">481</span>  }<a name="line.481"></a>
-<span class="sourceLineNo">482</span><a name="line.482"></a>
-<span class="sourceLineNo">483</span>  /**<a name="line.483"></a>
-<span class="sourceLineNo">484</span>   * in memory state of an active task.<a name="line.484"></a>
-<span class="sourceLineNo">485</span>   */<a name="line.485"></a>
-<span class="sourceLineNo">486</span>  @InterfaceAudience.Private<a name="line.486"></a>
-<span class="sourceLineNo">487</span>  public static class Task {<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    public volatile long last_update;<a name="line.488"></a>
-<span class="sourceLineNo">489</span>    public volatile int last_version;<a name="line.489"></a>
-<span class="sourceLineNo">490</span>    public volatile ServerName cur_worker_name;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>    public volatile TaskBatch batch;<a name="line.491"></a>
-<span class="sourceLineNo">492</span>    public volatile TerminationStatus status;<a name="line.492"></a>
-<span class="sourceLineNo">493</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.493"></a>
-<span class="sourceLineNo">494</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    public volatile boolean resubmitThresholdReached;<a name="line.495"></a>
-<span class="sourceLineNo">496</span><a name="line.496"></a>
-<span class="sourceLineNo">497</span>    @Override<a name="line.497"></a>
-<span class="sourceLineNo">498</span>    public String toString() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.499"></a>
-<span class="sourceLineNo">500</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.500"></a>
-<span class="sourceLineNo">501</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.501"></a>
-<span class="sourceLineNo">502</span>    }<a name="line.502"></a>
-<span class="sourceLineNo">503</span><a name="line.503"></a>
-<span class="sourceLineNo">504</span>    public Task() {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      last_version = -1;<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      status = IN_PROGRESS;<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      setUnassigned();<a name="line.507"></a>
-<span class="sourceLineNo">508</span>    }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span>    public boolean isOrphan() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      return (batch == null || batch.isDead);<a name="line.511"></a>
+<span class="sourceLineNo">238</span>    long totalSize = 0;<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    TaskBatch batch = null;<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    long startTime = 0;<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.241"></a>
+<span class="sourceLineNo">242</span>    if (logfiles.length != 0) {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>      status.setStatus("Checking directory contents...");<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.245"></a>
+<span class="sourceLineNo">246</span>          " for " + serverNames);<a name="line.246"></a>
+<span class="sourceLineNo">247</span>      startTime = EnvironmentEdgeManager.currentTime();<a name="line.247"></a>
+<span class="sourceLineNo">248</span>      batch = new TaskBatch();<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      for (FileStatus lf : logfiles) {<a name="line.249"></a>
+<span class="sourceLineNo">250</span>        // TODO If the log file is still being written to - which is most likely<a name="line.250"></a>
+<span class="sourceLineNo">251</span>        // the case for the last log file - then its length will show up here<a name="line.251"></a>
+<span class="sourceLineNo">252</span>        // as zero. The size of such a file can only be retrieved after<a name="line.252"></a>
+<span class="sourceLineNo">253</span>        // recover-lease is done. totalSize will be under in most cases and the<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        // metrics that it drives will also be under-reported.<a name="line.254"></a>
+<span class="sourceLineNo">255</span>        totalSize += lf.getLen();<a name="line.255"></a>
+<span class="sourceLineNo">256</span>        String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.257"></a>
+<span class="sourceLineNo">258</span>          throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.258"></a>
+<span class="sourceLineNo">259</span>        }<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      }<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      waitForSplittingCompletion(batch, status);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    if (batch != null &amp;&amp; batch.done != batch.installed) {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      batch.isDead = true;<a name="line.265"></a>
+<span class="sourceLineNo">266</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.267"></a>
+<span class="sourceLineNo">268</span>          + " but only " + batch.done + " done");<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.269"></a>
+<span class="sourceLineNo">270</span>      status.abort(msg);<a name="line.270"></a>
+<span class="sourceLineNo">271</span>      throw new IOException(msg);<a name="line.271"></a>
+<span class="sourceLineNo">272</span>    }<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    for (Path logDir : logDirs) {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      status.setStatus("Cleaning up log directory...");<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.275"></a>
+<span class="sourceLineNo">276</span>      try {<a name="line.276"></a>
+<span class="sourceLineNo">277</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.277"></a>
+<span class="sourceLineNo">278</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.278"></a>
+<span class="sourceLineNo">279</span>        }<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      } catch (IOException ioe) {<a name="line.280"></a>
+<span class="sourceLineNo">281</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.281"></a>
+<span class="sourceLineNo">282</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.282"></a>
+<span class="sourceLineNo">283</span>          LOG.warn("Returning success without actually splitting and "<a name="line.283"></a>
+<span class="sourceLineNo">284</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.284"></a>
+<span class="sourceLineNo">285</span>              + Arrays.toString(files), ioe);<a name="line.285"></a>
+<span class="sourceLineNo">286</span>        } else {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.287"></a>
+<span class="sourceLineNo">288</span>        }<a name="line.288"></a>
+<span class="sourceLineNo">289</span>      }<a name="line.289"></a>
+<span class="sourceLineNo">290</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.290"></a>
+<span class="sourceLineNo">291</span>    }<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    String msg = "Finished splitting (more than or equal to) " + totalSize +<a name="line.292"></a>
+<span class="sourceLineNo">293</span>        " bytes in " + ((batch == null)? 0: batch.installed) +<a name="line.293"></a>
+<span class="sourceLineNo">294</span>        " log files in " + logDirs + " in " +<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        ((startTime == 0)? startTime: (EnvironmentEdgeManager.currentTime() - startTime)) + "ms";<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    status.markComplete(msg);<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    LOG.info(msg);<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    return totalSize;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
+<span class="sourceLineNo">300</span><a name="line.300"></a>
+<span class="sourceLineNo">301</span>  /**<a name="line.301"></a>
+<span class="sourceLineNo">302</span>   * Add a task entry to coordination if it is not already there.<a name="line.302"></a>
+<span class="sourceLineNo">303</span>   * @param taskname the path of the log to be split<a name="line.303"></a>
+<span class="sourceLineNo">304</span>   * @param batch the batch this task belongs to<a name="line.304"></a>
+<span class="sourceLineNo">305</span>   * @return true if a new entry is created, false if it is already there.<a name="line.305"></a>
+<span class="sourceLineNo">306</span>   */<a name="line.306"></a>
+<span class="sourceLineNo">307</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.307"></a>
+<span class="sourceLineNo">308</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.309"></a>
+<span class="sourceLineNo">310</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.310"></a>
+<span class="sourceLineNo">311</span>    if (oldtask == null) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span>      // publish the task in the coordination engine<a name="line.312"></a>
+<span class="sourceLineNo">313</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.313"></a>
+<span class="sourceLineNo">314</span>      return true;<a name="line.314"></a>
+<span class="sourceLineNo">315</span>    }<a name="line.315"></a>
+<span class="sourceLineNo">316</span>    return false;<a name="line.316"></a>
+<span class="sourceLineNo">317</span>  }<a name="line.317"></a>
+<span class="sourceLineNo">318</span><a name="line.318"></a>
+<span class="sourceLineNo">319</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    synchronized (batch) {<a name="line.320"></a>
+<span class="sourceLineNo">321</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.321"></a>
+<span class="sourceLineNo">322</span>        try {<a name="line.322"></a>
+<span class="sourceLineNo">323</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.323"></a>
+<span class="sourceLineNo">324</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.324"></a>
+<span class="sourceLineNo">325</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.325"></a>
+<span class="sourceLineNo">326</span>          int actual = activeTasks(batch);<a name="line.326"></a>
+<span class="sourceLineNo">327</span>          if (remaining != actual) {<a name="line.327"></a>
+<span class="sourceLineNo">328</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.328"></a>
+<span class="sourceLineNo">329</span>          }<a name="line.329"></a>
+<span class="sourceLineNo">330</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.330"></a>
+<span class="sourceLineNo">331</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.331"></a>
+<span class="sourceLineNo">332</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.332"></a>
+<span class="sourceLineNo">333</span>                + remainingTasks);<a name="line.333"></a>
+<span class="sourceLineNo">334</span>          }<a name="line.334"></a>
+<span class="sourceLineNo">335</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.335"></a>
+<span class="sourceLineNo">336</span>            LOG.warn("No more task remaining, splitting "<a name="line.336"></a>
+<span class="sourceLineNo">337</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.337"></a>
+<span class="sourceLineNo">338</span>                + ", active tasks in map " + actual);<a name="line.338"></a>
+<span class="sourceLineNo">339</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.339"></a>
+<span class="sourceLineNo">340</span>              return;<a name="line.340"></a>
+<span class="sourceLineNo">341</span>            }<a name="line.341"></a>
+<span class="sourceLineNo">342</span>          }<a name="line.342"></a>
+<span class="sourceLineNo">343</span>          batch.wait(100);<a name="line.343"></a>
+<span class="sourceLineNo">344</span>          if (server.isStopped()) {<a name="line.344"></a>
+<span class="sourceLineNo">345</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.345"></a>
+<span class="sourceLineNo">346</span>            return;<a name="line.346"></a>
+<span class="sourceLineNo">347</span>          }<a name="line.347"></a>
+<span class="sourceLineNo">348</span>        } catch (InterruptedException e) {<a name="line.348"></a>
+<span class="sourceLineNo">349</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.349"></a>
+<span class="sourceLineNo">350</span>          Thread.currentThread().interrupt();<a name="line.350"></a>
+<span class="sourceLineNo">351</span>          return;<a name="line.351"></a>
+<span class="sourceLineNo">352</span>        }<a name="line.352"></a>
+<span class="sourceLineNo">353</span>      }<a name="line.353"></a>
+<span class="sourceLineNo">354</span>    }<a name="line.354"></a>
+<span class="sourceLineNo">355</span>  }<a name="line.355"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span>  @VisibleForTesting<a name="line.357"></a>
+<span class="sourceLineNo">358</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.358"></a>
+<span class="sourceLineNo">359</span>    return tasks;<a name="line.359"></a>
+<span class="sourceLineNo">360</span>  }<a name="line.360"></a>
+<span class="sourceLineNo">361</span><a name="line.361"></a>
+<span class="sourceLineNo">362</span>  private int activeTasks(final TaskBatch batch) {<a name="line.362"></a>
+<span class="sourceLineNo">363</span>    int count = 0;<a name="line.363"></a>
+<span class="sourceLineNo">364</span>    for (Task t : tasks.values()) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.365"></a>
+<span class="sourceLineNo">366</span>        count++;<a name="line.366"></a>
+<span class="sourceLineNo">367</span>      }<a name="line.367"></a>
+<span class="sourceLineNo">368</span>    }<a name="line.368"></a>
+<span class="sourceLineNo">369</span>    return count;<a name="line.369"></a>
+<span class="sourceLineNo">370</span><a name="line.370"></a>
+<span class="sourceLineNo">371</span>  }<a name="line.371"></a>
+<span class="sourceLineNo">372</span><a name="line.372"></a>
+<span class="sourceLineNo">373</span>  /**<a name="line.373"></a>
+<span class="sourceLineNo">374</span>   * @param path<a name="line.374"></a>
+<span class="sourceLineNo">375</span>   * @param batch<a name="line.375"></a>
+<span class="sourceLineNo">376</span>   * @return null on success, existing task on error<a name="line.376"></a>
+<span class="sourceLineNo">377</span>   */<a name="line.377"></a>
+<span class="sourceLineNo">378</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.378"></a>
+<span class="sourceLineNo">379</span>    Task oldtask;<a name="line.379"></a>
+<span class="sourceLineNo">380</span>    // batch.installed is only changed via this function and<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    // a single thread touches batch.installed.<a name="line.381"></a>
+<span class="sourceLineNo">382</span>    Task newtask = new Task();<a name="line.382"></a>
+<span class="sourceLineNo">383</span>    newtask.batch = batch;<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.384"></a>
+<span class="sourceLineNo">385</span>    if (oldtask == null) {<a name="line.385"></a>
+<span class="sourceLineNo">386</span>      batch.installed++;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      return null;<a name="line.387"></a>
+<span class="sourceLineNo">388</span>    }<a name="line.388"></a>
+<span class="sourceLineNo">389</span>    // new task was not used.<a name="line.389"></a>
+<span class="sourceLineNo">390</span>    synchronized (oldtask) {<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      if (oldtask.isOrphan()) {<a name="line.391"></a>
+<span class="sourceLineNo">392</span>        if (oldtask.status == SUCCESS) {<a name="line.392"></a>
+<span class="sourceLineNo">393</span>          // The task is already done. Do not install the batch for this<a name="line.393"></a>
+<span class="sourceLineNo">394</span>          // task because it might be too late for setDone() to update<a name="line.394"></a>
+<span class="sourceLineNo">395</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.395"></a>
+<span class="sourceLineNo">396</span>          // this task to complete.<a name="line.396"></a>
+<span class="sourceLineNo">397</span>          return (null);<a name="line.397"></a>
+<span class="sourceLineNo">398</span>        }<a name="line.398"></a>
+<span class="sourceLineNo">399</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.399"></a>
+<span class="sourceLineNo">400</span>          oldtask.batch = batch;<a name="line.400"></a>
+<span class="sourceLineNo">401</span>          batch.installed++;<a name="line.401"></a>
+<span class="sourceLineNo">402</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.402"></a>
+<span class="sourceLineNo">403</span>          return null;<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        }<a name="line.404"></a>
+<span class="sourceLineNo">405</span>        while (oldtask.status == FAILURE) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.406"></a>
+<span class="sourceLineNo">407</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.407"></a>
+<span class="sourceLineNo">408</span>          try {<a name="line.408"></a>
+<span class="sourceLineNo">409</span>            oldtask.wait();<a name="line.409"></a>
+<span class="sourceLineNo">410</span>          } catch (InterruptedException e) {<a name="line.410"></a>
+<span class="sourceLineNo">411</span>            Thread.currentThread().interrupt();<a name="line.411"></a>
+<span class="sourceLineNo">412</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.412"></a>
+<span class="sourceLineNo">413</span>            // fall through to return failure<a name="line.413"></a>
+<span class="sourceLineNo">414</span>            break;<a name="line.414"></a>
+<span class="sourceLineNo">415</span>          }<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        }<a name="line.416"></a>
+<span class="sourceLineNo">417</span>        if (oldtask.status != DELETED) {<a name="line.417"></a>
+<span class="sourceLineNo">418</span>          LOG.warn("Failure because previously failed task"<a name="line.418"></a>
+<span class="sourceLineNo">419</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.419"></a>
+<span class="sourceLineNo">420</span>          return oldtask;<a name="line.420"></a>
+<span class="sourceLineNo">421</span>        }<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        // reinsert the newTask and it must succeed this time<a name="line.422"></a>
+<span class="sourceLineNo">423</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        if (t == null) {<a name="line.424"></a>
+<span class="sourceLineNo">425</span>          batch.installed++;<a name="line.425"></a>
+<span class="sourceLineNo">426</span>          return null;<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        }<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.428"></a>
+<span class="sourceLineNo">429</span>        assert false : "Deleted task still present in tasks map";<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        return t;<a name="line.430"></a>
+<span class="sourceLineNo">431</span>      }<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.432"></a>
+<span class="sourceLineNo">433</span>      return oldtask;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>    }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>  }<a name="line.435"></a>
+<span class="sourceLineNo">436</span><a name="line.436"></a>
+<span class="sourceLineNo">437</span>  public void stop() {<a name="line.437"></a>
+<span class="sourceLineNo">438</span>    if (choreService != null) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span>      choreService.shutdown();<a name="line.439"></a>
+<span class="sourceLineNo">440</span>    }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    if (timeoutMonitor != null) {<a name="line.441"></a>
+<span class="sourceLineNo">442</span>      timeoutMonitor.cancel(true);<a name="line.442"></a>
+<span class="sourceLineNo">443</span>    }<a name="line.443"></a>
+<span class="sourceLineNo">444</span>  }<a name="line.444"></a>
+<span class="sourceLineNo">445</span><a name="line.445"></a>
+<span class="sourceLineNo">446</span>  void handleDeadWorker(ServerName workerName) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.447"></a>
+<span class="sourceLineNo">448</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.448"></a>
+<span class="sourceLineNo">449</span>    synchronized (deadWorkersLock) {<a name="line.449"></a>
+<span class="sourceLineNo">450</span>      if (deadWorkers == null) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      }<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      deadWorkers.add(workerName);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    }<a name="line.454"></a>
+<span class="sourceLineNo">455</span>    LOG.info("Dead splitlog worker {}", workerName);<a name="line.455"></a>
+<span class="sourceLineNo">456</span>  }<a name="line.456"></a>
+<span class="sourceLineNo">457</span><a name="line.457"></a>
+<span class="sourceLineNo">458</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>    synchronized (deadWorkersLock) {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>      if (deadWorkers == null) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.461"></a>
+<span class="sourceLineNo">462</span>      }<a name="line.462"></a>
+<span class="sourceLineNo">463</span>      deadWorkers.addAll(serverNames);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    }<a name="line.464"></a>
+<span class="sourceLineNo">465</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.465"></a>
+<span class="sourceLineNo">466</span>  }<a name="line.466"></a>
+<span class="sourceLineNo">467</span><a name="line.467"></a>
+<span class="sourceLineNo">468</span>  /**<a name="line.468"></a>
+<span class="sourceLineNo">469</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.469"></a>
+<span class="sourceLineNo">470</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.470"></a>
+<span class="sourceLineNo">471</span>   * &lt;p&gt;<a name="line.471"></a>
+<span class="sourceLineNo">472</span>   * All access is synchronized.<a name="line.472"></a>
+<span class="sourceLineNo">473</span>   */<a name="line.473"></a>
+<span class="sourceLineNo">474</span>  @InterfaceAudience.Private<a name="line.474"></a>
+<span class="sourceLineNo">475</span>  public static class TaskBatch {<a name="line.475"></a>
+<span class="sourceLineNo">476</span>    public int installed = 0;<a name="line.476"></a>
+<span class="sourceLineNo">477</span>    public int done = 0;<a name="line.477"></a>
+<span class="sourceLineNo">478</span>    public int error = 0;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    public volatile boolean isDead = false;<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span>    @Override<a name="line.481"></a>
+<span class="sourceLineNo">482</span>    public String toString() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    }<a name="line.484"></a>
+<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
+<span class="sourceLineNo">486</span><a name="line.486"></a>
+<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
+<span class="sourceLineNo">488</span>   * in memory state of an active task.<a name="line.488"></a>
+<span class="sourceLineNo">489</span>   */<a name="line.489"></a>
+<span class="sourceLineNo">490</span>  @InterfaceAudience.Private<a name="line.490"></a>
+<span class="sourceLineNo">491</span>  public static class Task {<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    public volatile long last_update;<a name="line.492"></a>
+<span class="sourceLineNo">493</span>    public volatile int last_version;<a name="line.493"></a>
+<span class="sourceLineNo">494</span>    public volatile ServerName cur_worker_name;<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    public volatile TaskBatch batch;<a name="line.495"></a>
+<span class="sourceLineNo">496</span>    public volatile TerminationStatus status;<a name="line.496"></a>
+<span class="sourceLineNo">497</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.497"></a>
+<span class="sourceLineNo">498</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.498"></a>
+<span class="sourceLineNo">499</span>    public volatile boolean resubmitThresholdReached;<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span>    @Override<a name="line.501"></a>
+<span class="sourceLineNo">502</span>    public String toString() {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.503"></a>
+<span class="sourceLineNo">504</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.504"></a>
+<span class="sourceLineNo">505</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.505"></a>
+<span class="sourceLineNo">506</span>    }<a name="line.506"></a>
+<span class="sourceLineNo">507</span><a name="line.507"></a>
+<span class="sourceLineNo">508</span>    public Task() {<a name="line.508"></a>
+<span class="sourceLineNo">509</span>      last_version = -1;<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      status = IN_PROGRESS;<a name="line.510"></a>
+<span class="sourceLineNo">511</span>      setUnassigned();<a name="line.511"></a>
 <span class="sourceLineNo">512</span>    }<a name="line.512"></a>
 <span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span>    public boolean isUnassigned() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span>      return (cur_worker_name == null);<a name="line.515"></a>
+<span class="sourceLineNo">514</span>    public boolean isOrphan() {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>      return (batch == null || batch.isDead);<a name="line.515"></a>
 <span class="sourceLineNo">516</span>    }<a name="line.516"></a>
 <span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span>    public void heartbeatNoDetails(long time) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span>      last_update = time;<a name="line.519"></a>
+<span class="sourceLineNo">518</span>    public boolean isUnassigned() {<a name="line.518"></a>
+<span class="sourceLineNo">519</span>      return (cur_worker_name == null);<a name="line.519"></a>
 <span class="sourceLineNo">520</span>    }<a name="line.520"></a>
 <span class="sourceLineNo">521</span><a name="line.521"></a>
-<span class="sourceLineNo">522</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      last_version = version;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      last_update = time;<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      cur_worker_name = worker;<a name="line.525"></a>
-<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span>    public void setUnassigned() {<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      cur_worker_name = null;<a name="line.529"></a>
-<span class="sourceLineNo">530</span>      last_update = -1;<a name="line.530"></a>
-<span class="sourceLineNo">531</span>    }<a name="line.531"></a>
-<span class="sourceLineNo">532</span>  }<a name="line.532"></a>
-<span class="sourceLineNo">533</span><a name="line.533"></a>
-<span class="sourceLineNo">534</span>  /**<a name="line.534"></a>
-<span class="sourceLineNo">535</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.535"></a>
-<span class="sourceLineNo">536</span>   */<a name="line.536"></a>
-<span class="sourceLineNo">537</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.537"></a>
-<span class="sourceLineNo">538</span>    private long lastLog = 0;<a name="line.538"></a>
-<span class="sourceLineNo">539</span><a name="line.539"></a>
-<span class="sourceLineNo">540</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.540"></a>
-<span class="sourceLineNo">541</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.541"></a>
-<span class="sourceLineNo">542</span>    }<a name="line.542"></a>
+<span class="sourceLineNo">522</span>    public void heartbeatNoDetails(long time) {<a name="line.522"></a>
+<span class="sourceLineNo">523</span>      last_update = time;<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    }<a name="line.524"></a>
+<span class="sourceLineNo">525</span><a name="line.525"></a>
+<span class="sourceLineNo">526</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span>      last_version = version;<a name="line.527"></a>
+<span class="sourceLineNo">528</span>      last_update = time;<a name="line.528"></a>
+<span class="sourceLineNo">529</span>      cur_worker_name = worker;<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    }<a name="line.530"></a>
+<span class="sourceLineNo">531</span><a name="line.531"></a>
+<span class="sourceLineNo">532</span>    public void setUnassigned() {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>      cur_worker_name = null;<a name="line.533"></a>
+<span class="sourceLineNo">534</span>      last_update = -1;<a name="line.534"></a>
+<span class="sourceLineNo">535</span>    }<a name="line.535"></a>
+<span class="sourceLineNo">536</span>  }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>  /**<a name="line.538"></a>
+<span class="sourceLineNo">539</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.539"></a>
+<span class="sourceLineNo">540</span>   */<a name="line.540"></a>
+<span class="sourceLineNo">541</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.541"></a>
+<span class="sourceLineNo">542</span>    private long lastLog = 0;<a name="line.542"></a>
 <span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>    @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    protected void chore() {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.546"></a>
+<span class="sourceLineNo">544</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    }<a name="line.546"></a>
 <span class="sourceLineNo">547</span><a name="line.547"></a>
-<span class="sourceLineNo">548</span>      int resubmitted = 0;<a name="line.548"></a>
-<span class="sourceLineNo">549</span>      int unassigned = 0;<a name="line.549"></a>
-<span class="sourceLineNo">550</span>      int tot = 0;<a name="line.550"></a>
-<span class="sourceLineNo">551</span>      boolean found_assigned_task = false;<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.552"></a>
-<span class="sourceLineNo">553</span><a name="line.553"></a>
-<span class="sourceLineNo">554</span>      synchronized (deadWorkersLock) {<a name="line.554"></a>
-<span class="sourceLineNo">555</span>        localDeadWorkers = deadWorkers;<a name="line.555"></a>
-<span class="sourceLineNo">556</span>        deadWorkers = null;<a name="line.556"></a>
-<span class="sourceLineNo">557</span>      }<a name="line.557"></a>
-<span class="sourceLineNo">558</span><a name="line.558"></a>
-<span class="sourceLineNo">559</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>        String path = e.getKey();<a name="line.560"></a>
-<span class="sourceLineNo">561</span>        Task task = e.getValue();<a name="line.561"></a>
-<span class="sourceLineNo">562</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.562"></a>
-<span class="sourceLineNo">563</span>        tot++;<a name="line.563"></a>
-<span class="sourceLineNo">564</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.564"></a>
-<span class="sourceLineNo">565</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.565"></a>
-<span class="sourceLineNo">566</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.566"></a>
-<span class="sourceLineNo">567</span>        // time. If we want progress when there are no region servers then we<a name="line.567"></a>
-<span class="sourceLineNo">568</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.568"></a>
-<span class="sourceLineNo">569</span>        if (task.isUnassigned()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>          unassigned++;<a name="line.570"></a>
-<span class="sourceLineNo">571</span>          continue;<a name="line.571"></a>
-<span class="sourceLineNo">572</span>        }<a name="line.572"></a>
-<span class="sourceLineNo">573</span>        found_assigned_task = true;<a name="line.573"></a>
-<span class="sourceLineNo">574</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.575"></a>
-<span class="sourceLineNo">576</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.576"></a>
-<span class="sourceLineNo">577</span>            resubmitted++;<a name="line.577"></a>
-<span class="sourceLineNo">578</span>          } else {<a name="line.578"></a>
-<span class="sourceLineNo">579</span>            handleDeadWorker(cur_worker);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.580"></a>
-<span class="sourceLineNo">581</span>                + ", will retry.");<a name="line.581"></a>
-<span class="sourceLineNo">582</span>          }<a name="line.582"></a>
-<span class="sourceLineNo">583</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.583"></a>
-<span class="sourceLineNo">584</span>          resubmitted++;<a name="line.584"></a>
-<span class="sourceLineNo">585</span>        }<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      }<a name="line.586"></a>
-<span class="sourceLineNo">587</span>      if (tot &gt; 0) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.588"></a>
-<span class="sourceLineNo">589</span>        if (now &gt; lastLog + 5000) {<a name="line.589"></a>
-<span class="sourceLineNo">590</span>          lastLog = now;<a name="line.590"></a>
-<span class="sourceLineNo">591</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.591"></a>
-<span class="sourceLineNo">592</span>        }<a name="line.592"></a>
-<span class="sourceLineNo">593</span>      }<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      if (resubmitted &gt; 0) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.595"></a>
-<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
-<span class="sourceLineNo">597</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.598"></a>
-<span class="sourceLineNo">599</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // because a. it is very unlikely that every worker had a<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      // transient error when trying to grab the task b. if there are no<a name="line.601"></a>
-<span class="sourceLineNo">602</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.604"></a>
-<span class="sourceLineNo">605</span>      // that there is always one worker in the system<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      if (tot &gt; 0<a name="line.606"></a>
-<span class="sourceLineNo">607</span>          &amp;&amp; !found_assigned_task<a name="line.607"></a>
-<span class="sourceLineNo">608</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.608"></a>
-<span class="sourceLineNo">609</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.609"></a>
-<span class="sourceLineNo">610</span>          String key = e.getKey();<a name="line.610"></a>
-<span class="sourceLineNo">611</span>          Task task = e.getValue();<a name="line.611"></a>
-<span class="sourceLineNo">612</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.612"></a>
-<span class="sourceLineNo">613</span>          // have been asynchronously assigned. There is no locking required<a name="line.613"></a>
-<span class="sourceLineNo">614</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.614"></a>
-<span class="sourceLineNo">615</span>          // called unnecessarily for a taskpath<a name="line.615"></a>
-<span class="sourceLineNo">616</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>            // We just touch the znode to make sure its still there<a name="line.617"></a>
-<span class="sourceLineNo">618</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.618"></a>
-<span class="sourceLineNo">619</span>          }<a name="line.619"></a>
-<span class="sourceLineNo">620</span>        }<a name="line.620"></a>
-<span class="sourceLineNo">621</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.621"></a>
-<span class="sourceLineNo">622</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.623"></a>
-<span class="sourceLineNo">624</span>      }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      Set&lt;String&gt; failedDeletions =<a name="line.625"></a>
-<span class="sourceLineNo">626</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      // Retry previously failed deletes<a name="line.627"></a>
-<span class="sourceLineNo">628</span>      if (failedDeletions.size() &gt; 0) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.629"></a>
-<span class="sourceLineNo">630</span>        for (String tmpPath : tmpPaths) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>          // deleteNode is an async call<a name="line.631"></a>
-<span class="sourceLineNo">632</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.632"></a>
-<span class="sourceLineNo">633</span>        }<a name="line.633"></a>
-<span class="sourceLineNo">634</span>        failedDeletions.removeAll(tmpPaths);<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      }<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span>  }<a name="line.637"></a>
-<span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>  public enum ResubmitDirective {<a name="line.639"></a>
-<span class="sourceLineNo">640</span>    CHECK(), FORCE()<a name="line.640"></a>
+<span class="sourceLineNo">548</span>    @Override<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    protected void chore() {<a name="line.549"></a>
+<span class="sourceLineNo">550</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>      int resubmitted = 0;<a name="line.552"></a>
+<span class="sourceLineNo">553</span>      int unassigned = 0;<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      int tot = 0;<a name="line.554"></a>
+<span class="sourceLineNo">555</span>      boolean found_assigned_task = false;<a name="line.555"></a>
+<span class="sourceLineNo">556</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>      synchronized (deadWorkersLock) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>        localDeadWorkers = deadWorkers;<a name="line.559"></a>
+<span class="sourceLineNo">560</span>        deadWorkers = null;<a name="line.560"></a>
+<span class="sourceLineNo">561</span>      }<a name="line.561"></a>
+<span class="sourceLineNo">562</span><a name="line.562"></a>
+<span class="sourceLineNo">563</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.563"></a>
+<span class="sourceLineNo">564</span>        String path = e.getKey();<a name="line.564"></a>
+<span class="sourceLineNo">565</span>        Task task = e.getValue();<a name="line.565"></a>
+<span class="sourceLineNo">566</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.566"></a>
+<span class="sourceLineNo">567</span>        tot++;<a name="line.567"></a>
+<span class="sourceLineNo">568</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.568"></a>
+<span class="sourceLineNo">569</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.569"></a>
+<span class="sourceLineNo">570</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.570"></a>
+<span class="sourceLineNo">571</span>        // time. If we want progress when there are no region servers then we<a name="line.571"></a>
+<span class="sourceLineNo">572</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.572"></a>
+<span class="sourceLineNo">573</span>        if (task.isUnassigned()) {<a name="line.573"></a>
+<span class="sourceLineNo">574</span>          unassigned++;<a name="line.574"></a>
+<span class="sourceLineNo">575</span>          continue;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>        }<a name="line.576"></a>
+<span class="sourceLineNo">577</span>        found_assigned_task = true;<a name="line.577"></a>
+<span class="sourceLineNo">578</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.578"></a>
+<span class="sourceLineNo">579</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.579"></a>
+<span class="sourceLineNo">580</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.580"></a>
+<span class="sourceLineNo">581</span>            resubmitted++;<a name="line.581"></a>
+<span class="sourceLineNo">582</span>          } else {<a name="line.582"></a>
+<span class="sourceLineNo">583</span>            handleDeadWorker(cur_worker);<a name="line.583"></a>
+<span class="sourceLineNo">584</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.584"></a>
+<span class="sourceLineNo">585</span>                + ", will retry.");<a name="line.585"></a>
+<span class="sourceLineNo">586</span>          }<a name="line.586"></a>
+<span class="sourceLineNo">587</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>          resubmitted++;<a name="line.588"></a>
+<span class="sourceLineNo">589</span>        }<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      }<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      if (tot &gt; 0) {<a name="line.591"></a>
+<span class="sourceLineNo">592</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.592"></a>
+<span class="sourceLineNo">593</span>        if (now &gt; lastLog + 5000) {<a name="line.593"></a>
+<span class="sourceLineNo">594</span>          lastLog = now;<a name="line.594"></a>
+<span class="sourceLineNo">595</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.595"></a>
+<span class="sourceLineNo">596</span>        }<a name="line.596"></a>
+<span class="sourceLineNo">597</span>      }<a name="line.597"></a>
+<span class="sourceLineNo">598</span>      if (resubmitted &gt; 0) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.599"></a>
+<span class="sourceLineNo">600</span>      }<a name="line.600"></a>
+<span class="sourceLineNo">601</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.602"></a>
+<span class="sourceLineNo">603</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.603"></a>
+<span class="sourceLineNo">604</span>      // because a. it is very unlikely that every worker had a<a name="line.604"></a>
+<span class="sourceLineNo">605</span>      // transient error when trying to grab the task b. if there are no<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.608"></a>
+<span class="sourceLineNo">609</span>      // that there is always one worker in the system<a name="line.609"></a>
+<span class="sourceLineNo">610</span>      if (tot &gt; 0<a name="line.610"></a>
+<span class="sourceLineNo">611</span>          &amp;&amp; !found_assigned_task<a name="line.611"></a>
+<span class="sourceLineNo">612</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>          String key = e.getKey();<a name="line.614"></a>
+<span class="sourceLineNo">615</span>          Task task = e.getValue();<a name="line.615"></a>
+<span class="sourceLineNo">616</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.616"></a>
+<span class="sourceLineNo">617</span>          // have been asynchronously assigned. There is no locking required<a name="line.617"></a>
+<span class="sourceLineNo">618</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.618"></a>
+<span class="sourceLineNo">619</span>          // called unnecessarily for a taskpath<a name="line.619"></a>
+<span class="sourceLineNo">620</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.620"></a>
+<span class="sourceLineNo">621</span>            // We just touch the znode to make sure its still there<a name="line.621"></a>
+<span class="sourceLineNo">622</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.622"></a>
+<span class="sourceLineNo">623</span>          }<a name="line.623"></a>
+<span class="sourceLineNo">624</span>        }<a name="line.624"></a>
+<span class="sourceLineNo">625</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.625"></a>
+<span class="sourceLineNo">626</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.626"></a>
+<span class="sourceLineNo">627</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      }<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      Set&lt;String&gt; failedDeletions =<a name="line.629"></a>
+<span class="sourceLineNo">630</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.630"></a>
+<span class="sourceLineNo">631</span>      // Retry previously failed deletes<a name="line.631"></a>
+<span class="sourceLineNo">632</span>      if (failedDeletions.size() &gt; 0) {<a name="line.632"></a>
+<span class="sourceLineNo">633</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.633"></a>
+<span class="sourceLineNo">634</span>        for (String tmpPath : tmpPaths) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>          // deleteNode is an async call<a name="line.635"></a>
+<span class="sourceLineNo">636</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.636"></a>
+<span class="sourceLineNo">637</span>        }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>        failedDeletions.removeAll(tmpPaths);<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      }<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    }<a name="line.640"></a>
 <span class="sourceLineNo">641</span>  }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>  public enum TerminationStatus {<a name="line.643"></a>
-<span class="sourceLineNo">644</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.644"></a>
-<span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    final String statusMsg;<a name="line.646"></a>
-<span class="sourceLineNo">647</span><a name="line.647"></a>
-<span class="sourceLineNo">648</span>    TerminationStatus(String msg) {<a name="line.648"></a>
-<span class="sourceLineNo">649</span>      statusMsg = msg;<a name="line.649"></a>
-<span class="sourceLineNo">650</span>    }<a name="line.650"></a>
+<span class="sourceLineNo">643</span>  public enum ResubmitDirective {<a name="line.643"></a>
+<span class="sourceLineNo">644</span>    CHECK(), FORCE()<a name="line.644"></a>
+<span class="sourceLineNo">645</span>  }<a name="line.645"></a>
+<span class="sourceLineNo">646</span><a name="line.646"></a>
+<span class="sourceLineNo">647</span>  public enum TerminationStatus {<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.648"></a>
+<span class="sourceLineNo">649</span><a name="line.649"></a>
+<span class="sourceLineNo">650</span>    final String statusMsg;<a name="line.650"></a>
 <span class="sourceLineNo">651</span><a name="line.651"></a>
-<span class="sourceLineNo">652</span>    @Override<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    public String toString() {<a name="line.653"></a>
-<span class="sourceLineNo">654</span>      return statusMsg;<a name="line.654"></a>
-<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
-<span class="sourceLineNo">657</span>}<a name="line.657"></a>
+<span class="sourceLineNo">652</span>    TerminationStatus(String msg) {<a name="line.652"></a>
+<span class="sourceLineNo">653</span>      statusMsg = msg;<a name="line.653"></a>
+<span class="sourceLineNo">654</span>    }<a name="line.654"></a>
+<span class="sourceLineNo">655</span><a name="line.655"></a>
+<span class="sourceLineNo">656</span>    @Override<a name="line.656"></a>
+<span class="sourceLineNo">657</span>    public String toString() {<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      return statusMsg;<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    }<a name="line.659"></a>
+<span class="sourceLineNo">660</span>  }<a name="line.660"></a>
+<span class="sourceLineNo">661</span>}<a name="line.661"></a>
 
 
 


[06/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
index 4a42bda..5815be5 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
@@ -91,177 +91,178 @@
 <span class="sourceLineNo">083</span>      // Ignore the last log which is current active log.<a name="line.83"></a>
 <span class="sourceLineNo">084</span>      while (logs.hasNext()) {<a name="line.84"></a>
 <span class="sourceLineNo">085</span>        ProcedureWALFile log = logs.next();<a name="line.85"></a>
-<span class="sourceLineNo">086</span>        log.open();<a name="line.86"></a>
-<span class="sourceLineNo">087</span>        try {<a name="line.87"></a>
-<span class="sourceLineNo">088</span>          reader.read(log);<a name="line.88"></a>
-<span class="sourceLineNo">089</span>        } finally {<a name="line.89"></a>
-<span class="sourceLineNo">090</span>          log.close();<a name="line.90"></a>
-<span class="sourceLineNo">091</span>        }<a name="line.91"></a>
-<span class="sourceLineNo">092</span>      }<a name="line.92"></a>
-<span class="sourceLineNo">093</span>      reader.finish();<a name="line.93"></a>
-<span class="sourceLineNo">094</span><a name="line.94"></a>
-<span class="sourceLineNo">095</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.95"></a>
-<span class="sourceLineNo">096</span>      tracker.setPartialFlag(false);<a name="line.96"></a>
-<span class="sourceLineNo">097</span>      tracker.resetUpdates();<a name="line.97"></a>
-<span class="sourceLineNo">098</span>    } finally {<a name="line.98"></a>
-<span class="sourceLineNo">099</span>      tracker.setKeepDeletes(false);<a name="line.99"></a>
-<span class="sourceLineNo">100</span>    }<a name="line.100"></a>
-<span class="sourceLineNo">101</span>  }<a name="line.101"></a>
-<span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.103"></a>
-<span class="sourceLineNo">104</span>      throws IOException {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>    header.writeDelimitedTo(stream);<a name="line.105"></a>
-<span class="sourceLineNo">106</span>  }<a name="line.106"></a>
-<span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span>  /*<a name="line.108"></a>
-<span class="sourceLineNo">109</span>   * +-----------------+<a name="line.109"></a>
-<span class="sourceLineNo">110</span>   * | END OF WAL DATA | &lt;---+<a name="line.110"></a>
-<span class="sourceLineNo">111</span>   * +-----------------+     |<a name="line.111"></a>
-<span class="sourceLineNo">112</span>   * |                 |     |<a name="line.112"></a>
-<span class="sourceLineNo">113</span>   * |     Tracker     |     |<a name="line.113"></a>
-<span class="sourceLineNo">114</span>   * |                 |     |<a name="line.114"></a>
-<span class="sourceLineNo">115</span>   * +-----------------+     |<a name="line.115"></a>
-<span class="sourceLineNo">116</span>   * |     version     |     |<a name="line.116"></a>
-<span class="sourceLineNo">117</span>   * +-----------------+     |<a name="line.117"></a>
-<span class="sourceLineNo">118</span>   * |  TRAILER_MAGIC  |     |<a name="line.118"></a>
-<span class="sourceLineNo">119</span>   * +-----------------+     |<a name="line.119"></a>
-<span class="sourceLineNo">120</span>   * |      offset     |-----+<a name="line.120"></a>
-<span class="sourceLineNo">121</span>   * +-----------------+<a name="line.121"></a>
-<span class="sourceLineNo">122</span>   */<a name="line.122"></a>
-<span class="sourceLineNo">123</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.123"></a>
-<span class="sourceLineNo">124</span>      throws IOException {<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    long offset = stream.getPos();<a name="line.125"></a>
-<span class="sourceLineNo">126</span><a name="line.126"></a>
-<span class="sourceLineNo">127</span>    // Write EOF Entry<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    ProcedureWALEntry.newBuilder()<a name="line.128"></a>
-<span class="sourceLineNo">129</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.129"></a>
-<span class="sourceLineNo">130</span>      .build().writeDelimitedTo(stream);<a name="line.130"></a>
-<span class="sourceLineNo">131</span><a name="line.131"></a>
-<span class="sourceLineNo">132</span>    // Write Tracker<a name="line.132"></a>
-<span class="sourceLineNo">133</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.133"></a>
-<span class="sourceLineNo">134</span><a name="line.134"></a>
-<span class="sourceLineNo">135</span>    stream.write(TRAILER_VERSION);<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    StreamUtils.writeLong(stream, offset);<a name="line.137"></a>
-<span class="sourceLineNo">138</span>    return stream.getPos() - offset;<a name="line.138"></a>
-<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
-<span class="sourceLineNo">140</span><a name="line.140"></a>
-<span class="sourceLineNo">141</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.141"></a>
-<span class="sourceLineNo">142</span>      throws IOException {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>    ProcedureWALHeader header;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    try {<a name="line.144"></a>
-<span class="sourceLineNo">145</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.145"></a>
-<span class="sourceLineNo">146</span>    } catch (InvalidProtocolBufferException e) {<a name="line.146"></a>
-<span class="sourceLineNo">147</span>      throw new InvalidWALDataException(e);<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    }<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span>    if (header == null) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    }<a name="line.152"></a>
-<span class="sourceLineNo">153</span><a name="line.153"></a>
-<span class="sourceLineNo">154</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.155"></a>
-<span class="sourceLineNo">156</span>          " expected " + HEADER_VERSION);<a name="line.156"></a>
-<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
-<span class="sourceLineNo">158</span><a name="line.158"></a>
-<span class="sourceLineNo">159</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    }<a name="line.161"></a>
-<span class="sourceLineNo">162</span><a name="line.162"></a>
-<span class="sourceLineNo">163</span>    return header;<a name="line.163"></a>
-<span class="sourceLineNo">164</span>  }<a name="line.164"></a>
-<span class="sourceLineNo">165</span><a name="line.165"></a>
-<span class="sourceLineNo">166</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.166"></a>
-<span class="sourceLineNo">167</span>      throws IOException {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    long trailerPos = size - 17;<a name="line.169"></a>
-<span class="sourceLineNo">170</span><a name="line.170"></a>
-<span class="sourceLineNo">171</span>    if (trailerPos &lt; startPos) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.172"></a>
-<span class="sourceLineNo">173</span>    }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>    stream.seek(trailerPos);<a name="line.175"></a>
-<span class="sourceLineNo">176</span>    int version = stream.read();<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    if (version != TRAILER_VERSION) {<a name="line.177"></a>
-<span class="sourceLineNo">178</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.178"></a>
-<span class="sourceLineNo">179</span>          " expected " + TRAILER_VERSION);<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    }<a name="line.180"></a>
-<span class="sourceLineNo">181</span><a name="line.181"></a>
-<span class="sourceLineNo">182</span>    long magic = StreamUtils.readLong(stream);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    if (magic != TRAILER_MAGIC) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.184"></a>
-<span class="sourceLineNo">185</span>          " expected " + TRAILER_MAGIC);<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    stream.seek(trailerOffset);<a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    }<a name="line.194"></a>
-<span class="sourceLineNo">195</span><a name="line.195"></a>
-<span class="sourceLineNo">196</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      .setVersion(version)<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      .setTrackerPos(stream.getPos())<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      .build();<a name="line.199"></a>
-<span class="sourceLineNo">200</span>    return trailer;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>  }<a name="line.201"></a>
-<span class="sourceLineNo">202</span><a name="line.202"></a>
-<span class="sourceLineNo">203</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.203"></a>
-<span class="sourceLineNo">204</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.204"></a>
-<span class="sourceLineNo">205</span>  }<a name="line.205"></a>
-<span class="sourceLineNo">206</span><a name="line.206"></a>
-<span class="sourceLineNo">207</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      Procedure proc, Procedure[] subprocs) throws IOException {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.209"></a>
-<span class="sourceLineNo">210</span>    builder.setType(type);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.211"></a>
-<span class="sourceLineNo">212</span>    if (subprocs != null) {<a name="line.212"></a>
-<span class="sourceLineNo">213</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      }<a name="line.215"></a>
-<span class="sourceLineNo">216</span>    }<a name="line.216"></a>
-<span class="sourceLineNo">217</span>    builder.build().writeDelimitedTo(slot);<a name="line.217"></a>
-<span class="sourceLineNo">218</span>  }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span>  public static void writeInsert(ByteSlot slot, Procedure proc)<a name="line.220"></a>
-<span class="sourceLineNo">221</span>      throws IOException {<a name="line.221"></a>
-<span class="sourceLineNo">222</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.222"></a>
-<span class="sourceLineNo">223</span>  }<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>  public static void writeInsert(ByteSlot slot, Procedure proc, Procedure[] subprocs)<a name="line.225"></a>
-<span class="sourceLineNo">226</span>      throws IOException {<a name="line.226"></a>
-<span class="sourceLineNo">227</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.227"></a>
-<span class="sourceLineNo">228</span>  }<a name="line.228"></a>
-<span class="sourceLineNo">229</span><a name="line.229"></a>
-<span class="sourceLineNo">230</span>  public static void writeUpdate(ByteSlot slot, Procedure proc)<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      throws IOException {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>  }<a name="line.233"></a>
-<span class="sourceLineNo">234</span><a name="line.234"></a>
-<span class="sourceLineNo">235</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      throws IOException {<a name="line.236"></a>
-<span class="sourceLineNo">237</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    builder.setProcId(procId);<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    builder.build().writeDelimitedTo(slot);<a name="line.240"></a>
-<span class="sourceLineNo">241</span>  }<a name="line.241"></a>
-<span class="sourceLineNo">242</span><a name="line.242"></a>
-<span class="sourceLineNo">243</span>  public static void writeDelete(ByteSlot slot, Procedure proc, long[] subprocs)<a name="line.243"></a>
-<span class="sourceLineNo">244</span>      throws IOException {<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    builder.setProcId(proc.getProcId());<a name="line.247"></a>
-<span class="sourceLineNo">248</span>    if (subprocs != null) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        builder.addChildId(subprocs[i]);<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span>    builder.build().writeDelimitedTo(slot);<a name="line.254"></a>
-<span class="sourceLineNo">255</span>  }<a name="line.255"></a>
-<span class="sourceLineNo">256</span>}<a name="line.256"></a>
+<span class="sourceLineNo">086</span>        LOG.debug("Loading WAL id={}", log.getLogId());<a name="line.86"></a>
+<span class="sourceLineNo">087</span>        log.open();<a name="line.87"></a>
+<span class="sourceLineNo">088</span>        try {<a name="line.88"></a>
+<span class="sourceLineNo">089</span>          reader.read(log);<a name="line.89"></a>
+<span class="sourceLineNo">090</span>        } finally {<a name="line.90"></a>
+<span class="sourceLineNo">091</span>          log.close();<a name="line.91"></a>
+<span class="sourceLineNo">092</span>        }<a name="line.92"></a>
+<span class="sourceLineNo">093</span>      }<a name="line.93"></a>
+<span class="sourceLineNo">094</span>      reader.finish();<a name="line.94"></a>
+<span class="sourceLineNo">095</span><a name="line.95"></a>
+<span class="sourceLineNo">096</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.96"></a>
+<span class="sourceLineNo">097</span>      tracker.setPartialFlag(false);<a name="line.97"></a>
+<span class="sourceLineNo">098</span>      tracker.resetUpdates();<a name="line.98"></a>
+<span class="sourceLineNo">099</span>    } finally {<a name="line.99"></a>
+<span class="sourceLineNo">100</span>      tracker.setKeepDeletes(false);<a name="line.100"></a>
+<span class="sourceLineNo">101</span>    }<a name="line.101"></a>
+<span class="sourceLineNo">102</span>  }<a name="line.102"></a>
+<span class="sourceLineNo">103</span><a name="line.103"></a>
+<span class="sourceLineNo">104</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.104"></a>
+<span class="sourceLineNo">105</span>      throws IOException {<a name="line.105"></a>
+<span class="sourceLineNo">106</span>    header.writeDelimitedTo(stream);<a name="line.106"></a>
+<span class="sourceLineNo">107</span>  }<a name="line.107"></a>
+<span class="sourceLineNo">108</span><a name="line.108"></a>
+<span class="sourceLineNo">109</span>  /*<a name="line.109"></a>
+<span class="sourceLineNo">110</span>   * +-----------------+<a name="line.110"></a>
+<span class="sourceLineNo">111</span>   * | END OF WAL DATA | &lt;---+<a name="line.111"></a>
+<span class="sourceLineNo">112</span>   * +-----------------+     |<a name="line.112"></a>
+<span class="sourceLineNo">113</span>   * |                 |     |<a name="line.113"></a>
+<span class="sourceLineNo">114</span>   * |     Tracker     |     |<a name="line.114"></a>
+<span class="sourceLineNo">115</span>   * |                 |     |<a name="line.115"></a>
+<span class="sourceLineNo">116</span>   * +-----------------+     |<a name="line.116"></a>
+<span class="sourceLineNo">117</span>   * |     version     |     |<a name="line.117"></a>
+<span class="sourceLineNo">118</span>   * +-----------------+     |<a name="line.118"></a>
+<span class="sourceLineNo">119</span>   * |  TRAILER_MAGIC  |     |<a name="line.119"></a>
+<span class="sourceLineNo">120</span>   * +-----------------+     |<a name="line.120"></a>
+<span class="sourceLineNo">121</span>   * |      offset     |-----+<a name="line.121"></a>
+<span class="sourceLineNo">122</span>   * +-----------------+<a name="line.122"></a>
+<span class="sourceLineNo">123</span>   */<a name="line.123"></a>
+<span class="sourceLineNo">124</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.124"></a>
+<span class="sourceLineNo">125</span>      throws IOException {<a name="line.125"></a>
+<span class="sourceLineNo">126</span>    long offset = stream.getPos();<a name="line.126"></a>
+<span class="sourceLineNo">127</span><a name="line.127"></a>
+<span class="sourceLineNo">128</span>    // Write EOF Entry<a name="line.128"></a>
+<span class="sourceLineNo">129</span>    ProcedureWALEntry.newBuilder()<a name="line.129"></a>
+<span class="sourceLineNo">130</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.130"></a>
+<span class="sourceLineNo">131</span>      .build().writeDelimitedTo(stream);<a name="line.131"></a>
+<span class="sourceLineNo">132</span><a name="line.132"></a>
+<span class="sourceLineNo">133</span>    // Write Tracker<a name="line.133"></a>
+<span class="sourceLineNo">134</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span>    stream.write(TRAILER_VERSION);<a name="line.136"></a>
+<span class="sourceLineNo">137</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    StreamUtils.writeLong(stream, offset);<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    return stream.getPos() - offset;<a name="line.139"></a>
+<span class="sourceLineNo">140</span>  }<a name="line.140"></a>
+<span class="sourceLineNo">141</span><a name="line.141"></a>
+<span class="sourceLineNo">142</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.142"></a>
+<span class="sourceLineNo">143</span>      throws IOException {<a name="line.143"></a>
+<span class="sourceLineNo">144</span>    ProcedureWALHeader header;<a name="line.144"></a>
+<span class="sourceLineNo">145</span>    try {<a name="line.145"></a>
+<span class="sourceLineNo">146</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    } catch (InvalidProtocolBufferException e) {<a name="line.147"></a>
+<span class="sourceLineNo">148</span>      throw new InvalidWALDataException(e);<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    }<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span>    if (header == null) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.152"></a>
+<span class="sourceLineNo">153</span>    }<a name="line.153"></a>
+<span class="sourceLineNo">154</span><a name="line.154"></a>
+<span class="sourceLineNo">155</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.156"></a>
+<span class="sourceLineNo">157</span>          " expected " + HEADER_VERSION);<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.160"></a>
+<span class="sourceLineNo">161</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.161"></a>
+<span class="sourceLineNo">162</span>    }<a name="line.162"></a>
+<span class="sourceLineNo">163</span><a name="line.163"></a>
+<span class="sourceLineNo">164</span>    return header;<a name="line.164"></a>
+<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
+<span class="sourceLineNo">166</span><a name="line.166"></a>
+<span class="sourceLineNo">167</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.167"></a>
+<span class="sourceLineNo">168</span>      throws IOException {<a name="line.168"></a>
+<span class="sourceLineNo">169</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.169"></a>
+<span class="sourceLineNo">170</span>    long trailerPos = size - 17;<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span>    if (trailerPos &lt; startPos) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.173"></a>
+<span class="sourceLineNo">174</span>    }<a name="line.174"></a>
+<span class="sourceLineNo">175</span><a name="line.175"></a>
+<span class="sourceLineNo">176</span>    stream.seek(trailerPos);<a name="line.176"></a>
+<span class="sourceLineNo">177</span>    int version = stream.read();<a name="line.177"></a>
+<span class="sourceLineNo">178</span>    if (version != TRAILER_VERSION) {<a name="line.178"></a>
+<span class="sourceLineNo">179</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.179"></a>
+<span class="sourceLineNo">180</span>          " expected " + TRAILER_VERSION);<a name="line.180"></a>
+<span class="sourceLineNo">181</span>    }<a name="line.181"></a>
+<span class="sourceLineNo">182</span><a name="line.182"></a>
+<span class="sourceLineNo">183</span>    long magic = StreamUtils.readLong(stream);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    if (magic != TRAILER_MAGIC) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.185"></a>
+<span class="sourceLineNo">186</span>          " expected " + TRAILER_MAGIC);<a name="line.186"></a>
+<span class="sourceLineNo">187</span>    }<a name="line.187"></a>
+<span class="sourceLineNo">188</span><a name="line.188"></a>
+<span class="sourceLineNo">189</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.189"></a>
+<span class="sourceLineNo">190</span>    stream.seek(trailerOffset);<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.193"></a>
+<span class="sourceLineNo">194</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    }<a name="line.195"></a>
+<span class="sourceLineNo">196</span><a name="line.196"></a>
+<span class="sourceLineNo">197</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.197"></a>
+<span class="sourceLineNo">198</span>      .setVersion(version)<a name="line.198"></a>
+<span class="sourceLineNo">199</span>      .setTrackerPos(stream.getPos())<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      .build();<a name="line.200"></a>
+<span class="sourceLineNo">201</span>    return trailer;<a name="line.201"></a>
+<span class="sourceLineNo">202</span>  }<a name="line.202"></a>
+<span class="sourceLineNo">203</span><a name="line.203"></a>
+<span class="sourceLineNo">204</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.205"></a>
+<span class="sourceLineNo">206</span>  }<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      Procedure proc, Procedure[] subprocs) throws IOException {<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.210"></a>
+<span class="sourceLineNo">211</span>    builder.setType(type);<a name="line.211"></a>
+<span class="sourceLineNo">212</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.212"></a>
+<span class="sourceLineNo">213</span>    if (subprocs != null) {<a name="line.213"></a>
+<span class="sourceLineNo">214</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.214"></a>
+<span class="sourceLineNo">215</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.215"></a>
+<span class="sourceLineNo">216</span>      }<a name="line.216"></a>
+<span class="sourceLineNo">217</span>    }<a name="line.217"></a>
+<span class="sourceLineNo">218</span>    builder.build().writeDelimitedTo(slot);<a name="line.218"></a>
+<span class="sourceLineNo">219</span>  }<a name="line.219"></a>
+<span class="sourceLineNo">220</span><a name="line.220"></a>
+<span class="sourceLineNo">221</span>  public static void writeInsert(ByteSlot slot, Procedure proc)<a name="line.221"></a>
+<span class="sourceLineNo">222</span>      throws IOException {<a name="line.222"></a>
+<span class="sourceLineNo">223</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.223"></a>
+<span class="sourceLineNo">224</span>  }<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span>  public static void writeInsert(ByteSlot slot, Procedure proc, Procedure[] subprocs)<a name="line.226"></a>
+<span class="sourceLineNo">227</span>      throws IOException {<a name="line.227"></a>
+<span class="sourceLineNo">228</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.228"></a>
+<span class="sourceLineNo">229</span>  }<a name="line.229"></a>
+<span class="sourceLineNo">230</span><a name="line.230"></a>
+<span class="sourceLineNo">231</span>  public static void writeUpdate(ByteSlot slot, Procedure proc)<a name="line.231"></a>
+<span class="sourceLineNo">232</span>      throws IOException {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.233"></a>
+<span class="sourceLineNo">234</span>  }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.236"></a>
+<span class="sourceLineNo">237</span>      throws IOException {<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    builder.setProcId(procId);<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    builder.build().writeDelimitedTo(slot);<a name="line.241"></a>
+<span class="sourceLineNo">242</span>  }<a name="line.242"></a>
+<span class="sourceLineNo">243</span><a name="line.243"></a>
+<span class="sourceLineNo">244</span>  public static void writeDelete(ByteSlot slot, Procedure proc, long[] subprocs)<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      throws IOException {<a name="line.245"></a>
+<span class="sourceLineNo">246</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.246"></a>
+<span class="sourceLineNo">247</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    builder.setProcId(proc.getProcId());<a name="line.248"></a>
+<span class="sourceLineNo">249</span>    if (subprocs != null) {<a name="line.249"></a>
+<span class="sourceLineNo">250</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.250"></a>
+<span class="sourceLineNo">251</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.251"></a>
+<span class="sourceLineNo">252</span>        builder.addChildId(subprocs[i]);<a name="line.252"></a>
+<span class="sourceLineNo">253</span>      }<a name="line.253"></a>
+<span class="sourceLineNo">254</span>    }<a name="line.254"></a>
+<span class="sourceLineNo">255</span>    builder.build().writeDelimitedTo(slot);<a name="line.255"></a>
+<span class="sourceLineNo">256</span>  }<a name="line.256"></a>
+<span class="sourceLineNo">257</span>}<a name="line.257"></a>
 
 
 


[09/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html
index 681e263..7cdf584 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.TimeoutMonitor.html
@@ -182,7 +182,7 @@
 <span class="sourceLineNo">174</span>      }<a name="line.174"></a>
 <span class="sourceLineNo">175</span>      FileStatus[] logfiles = FSUtils.listStatus(fs, logDir, filter);<a name="line.175"></a>
 <span class="sourceLineNo">176</span>      if (logfiles == null || logfiles.length == 0) {<a name="line.176"></a>
-<span class="sourceLineNo">177</span>        LOG.info(logDir + " is empty dir, no logs to split");<a name="line.177"></a>
+<span class="sourceLineNo">177</span>        LOG.info("{} dir is empty, no logs to split.", logDir);<a name="line.177"></a>
 <span class="sourceLineNo">178</span>      } else {<a name="line.178"></a>
 <span class="sourceLineNo">179</span>        Collections.addAll(fileStatus, logfiles);<a name="line.179"></a>
 <span class="sourceLineNo">180</span>      }<a name="line.180"></a>
@@ -243,426 +243,430 @@
 <span class="sourceLineNo">235</span>      PathFilter filter) throws IOException {<a name="line.235"></a>
 <span class="sourceLineNo">236</span>    MonitoredTask status = TaskMonitor.get().createStatus("Doing distributed log split in " +<a name="line.236"></a>
 <span class="sourceLineNo">237</span>      logDirs + " for serverName=" + serverNames);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    status.setStatus("Checking directory contents...");<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.240"></a>
-<span class="sourceLineNo">241</span>    LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.241"></a>
-<span class="sourceLineNo">242</span>      " for " + serverNames);<a name="line.242"></a>
-<span class="sourceLineNo">243</span>    long t = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
-<span class="sourceLineNo">244</span>    long totalSize = 0;<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    TaskBatch batch = new TaskBatch();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    for (FileStatus lf : logfiles) {<a name="line.246"></a>
-<span class="sourceLineNo">247</span>      // TODO If the log file is still being written to - which is most likely<a name="line.247"></a>
-<span class="sourceLineNo">248</span>      // the case for the last log file - then its length will show up here<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      // as zero. The size of such a file can only be retrieved after<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      // recover-lease is done. totalSize will be under in most cases and the<a name="line.250"></a>
-<span class="sourceLineNo">251</span>      // metrics that it drives will also be under-reported.<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      totalSize += lf.getLen();<a name="line.252"></a>
-<span class="sourceLineNo">253</span>      String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.253"></a>
-<span class="sourceLineNo">254</span>      if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.254"></a>
-<span class="sourceLineNo">255</span>        throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.255"></a>
-<span class="sourceLineNo">256</span>      }<a name="line.256"></a>
-<span class="sourceLineNo">257</span>    }<a name="line.257"></a>
-<span class="sourceLineNo">258</span>    waitForSplittingCompletion(batch, status);<a name="line.258"></a>
-<span class="sourceLineNo">259</span><a name="line.259"></a>
-<span class="sourceLineNo">260</span>    if (batch.done != batch.installed) {<a name="line.260"></a>
-<span class="sourceLineNo">261</span>      batch.isDead = true;<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.262"></a>
-<span class="sourceLineNo">263</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.263"></a>
-<span class="sourceLineNo">264</span>          + " but only " + batch.done + " done");<a name="line.264"></a>
-<span class="sourceLineNo">265</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      status.abort(msg);<a name="line.266"></a>
-<span class="sourceLineNo">267</span>      throw new IOException(msg);<a name="line.267"></a>
-<span class="sourceLineNo">268</span>    }<a name="line.268"></a>
-<span class="sourceLineNo">269</span>    for (Path logDir : logDirs) {<a name="line.269"></a>
-<span class="sourceLineNo">270</span>      status.setStatus("Cleaning up log directory...");<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      try {<a name="line.272"></a>
-<span class="sourceLineNo">273</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.273"></a>
-<span class="sourceLineNo">274</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.274"></a>
-<span class="sourceLineNo">275</span>        }<a name="line.275"></a>
-<span class="sourceLineNo">276</span>      } catch (IOException ioe) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.277"></a>
-<span class="sourceLineNo">278</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.278"></a>
-<span class="sourceLineNo">279</span>          LOG.warn("Returning success without actually splitting and "<a name="line.279"></a>
-<span class="sourceLineNo">280</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.280"></a>
-<span class="sourceLineNo">281</span>              + Arrays.toString(files), ioe);<a name="line.281"></a>
-<span class="sourceLineNo">282</span>        } else {<a name="line.282"></a>
-<span class="sourceLineNo">283</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.283"></a>
-<span class="sourceLineNo">284</span>        }<a name="line.284"></a>
-<span class="sourceLineNo">285</span>      }<a name="line.285"></a>
-<span class="sourceLineNo">286</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.286"></a>
-<span class="sourceLineNo">287</span>    }<a name="line.287"></a>
-<span class="sourceLineNo">288</span>    String msg =<a name="line.288"></a>
-<span class="sourceLineNo">289</span>        "finished splitting (more than or equal to) " + totalSize + " bytes in " + batch.installed<a name="line.289"></a>
-<span class="sourceLineNo">290</span>            + " log files in " + logDirs + " in "<a name="line.290"></a>
-<span class="sourceLineNo">291</span>            + (EnvironmentEdgeManager.currentTime() - t) + "ms";<a name="line.291"></a>
-<span class="sourceLineNo">292</span>    status.markComplete(msg);<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    LOG.info(msg);<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    return totalSize;<a name="line.294"></a>
-<span class="sourceLineNo">295</span>  }<a name="line.295"></a>
-<span class="sourceLineNo">296</span><a name="line.296"></a>
-<span class="sourceLineNo">297</span>  /**<a name="line.297"></a>
-<span class="sourceLineNo">298</span>   * Add a task entry to coordination if it is not already there.<a name="line.298"></a>
-<span class="sourceLineNo">299</span>   * @param taskname the path of the log to be split<a name="line.299"></a>
-<span class="sourceLineNo">300</span>   * @param batch the batch this task belongs to<a name="line.300"></a>
-<span class="sourceLineNo">301</span>   * @return true if a new entry is created, false if it is already there.<a name="line.301"></a>
-<span class="sourceLineNo">302</span>   */<a name="line.302"></a>
-<span class="sourceLineNo">303</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.303"></a>
-<span class="sourceLineNo">304</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.304"></a>
-<span class="sourceLineNo">305</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.305"></a>
-<span class="sourceLineNo">306</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.306"></a>
-<span class="sourceLineNo">307</span>    if (oldtask == null) {<a name="line.307"></a>
-<span class="sourceLineNo">308</span>      // publish the task in the coordination engine<a name="line.308"></a>
-<span class="sourceLineNo">309</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.309"></a>
-<span class="sourceLineNo">310</span>      return true;<a name="line.310"></a>
-<span class="sourceLineNo">311</span>    }<a name="line.311"></a>
-<span class="sourceLineNo">312</span>    return false;<a name="line.312"></a>
-<span class="sourceLineNo">313</span>  }<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.315"></a>
-<span class="sourceLineNo">316</span>    synchronized (batch) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.317"></a>
-<span class="sourceLineNo">318</span>        try {<a name="line.318"></a>
-<span class="sourceLineNo">319</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.319"></a>
-<span class="sourceLineNo">320</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.320"></a>
-<span class="sourceLineNo">321</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.321"></a>
-<span class="sourceLineNo">322</span>          int actual = activeTasks(batch);<a name="line.322"></a>
-<span class="sourceLineNo">323</span>          if (remaining != actual) {<a name="line.323"></a>
-<span class="sourceLineNo">324</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.324"></a>
-<span class="sourceLineNo">325</span>          }<a name="line.325"></a>
-<span class="sourceLineNo">326</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.326"></a>
-<span class="sourceLineNo">327</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.327"></a>
-<span class="sourceLineNo">328</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.328"></a>
-<span class="sourceLineNo">329</span>                + remainingTasks);<a name="line.329"></a>
-<span class="sourceLineNo">330</span>          }<a name="line.330"></a>
-<span class="sourceLineNo">331</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span>            LOG.warn("No more task remaining, splitting "<a name="line.332"></a>
-<span class="sourceLineNo">333</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.333"></a>
-<span class="sourceLineNo">334</span>                + ", active tasks in map " + actual);<a name="line.334"></a>
-<span class="sourceLineNo">335</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.335"></a>
-<span class="sourceLineNo">336</span>              return;<a name="line.336"></a>
-<span class="sourceLineNo">337</span>            }<a name="line.337"></a>
-<span class="sourceLineNo">338</span>          }<a name="line.338"></a>
-<span class="sourceLineNo">339</span>          batch.wait(100);<a name="line.339"></a>
-<span class="sourceLineNo">340</span>          if (server.isStopped()) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.341"></a>
-<span class="sourceLineNo">342</span>            return;<a name="line.342"></a>
-<span class="sourceLineNo">343</span>          }<a name="line.343"></a>
-<span class="sourceLineNo">344</span>        } catch (InterruptedException e) {<a name="line.344"></a>
-<span class="sourceLineNo">345</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.345"></a>
-<span class="sourceLineNo">346</span>          Thread.currentThread().interrupt();<a name="line.346"></a>
-<span class="sourceLineNo">347</span>          return;<a name="line.347"></a>
-<span class="sourceLineNo">348</span>        }<a name="line.348"></a>
-<span class="sourceLineNo">349</span>      }<a name="line.349"></a>
-<span class="sourceLineNo">350</span>    }<a name="line.350"></a>
-<span class="sourceLineNo">351</span>  }<a name="line.351"></a>
-<span class="sourceLineNo">352</span><a name="line.352"></a>
-<span class="sourceLineNo">353</span>  @VisibleForTesting<a name="line.353"></a>
-<span class="sourceLineNo">354</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.354"></a>
-<span class="sourceLineNo">355</span>    return tasks;<a name="line.355"></a>
-<span class="sourceLineNo">356</span>  }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span>  private int activeTasks(final TaskBatch batch) {<a name="line.358"></a>
-<span class="sourceLineNo">359</span>    int count = 0;<a name="line.359"></a>
-<span class="sourceLineNo">360</span>    for (Task t : tasks.values()) {<a name="line.360"></a>
-<span class="sourceLineNo">361</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.361"></a>
-<span class="sourceLineNo">362</span>        count++;<a name="line.362"></a>
-<span class="sourceLineNo">363</span>      }<a name="line.363"></a>
-<span class="sourceLineNo">364</span>    }<a name="line.364"></a>
-<span class="sourceLineNo">365</span>    return count;<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span>  }<a name="line.367"></a>
-<span class="sourceLineNo">368</span><a name="line.368"></a>
-<span class="sourceLineNo">369</span>  /**<a name="line.369"></a>
-<span class="sourceLineNo">370</span>   * @param path<a name="line.370"></a>
-<span class="sourceLineNo">371</span>   * @param batch<a name="line.371"></a>
-<span class="sourceLineNo">372</span>   * @return null on success, existing task on error<a name="line.372"></a>
-<span class="sourceLineNo">373</span>   */<a name="line.373"></a>
-<span class="sourceLineNo">374</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span>    Task oldtask;<a name="line.375"></a>
-<span class="sourceLineNo">376</span>    // batch.installed is only changed via this function and<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    // a single thread touches batch.installed.<a name="line.377"></a>
-<span class="sourceLineNo">378</span>    Task newtask = new Task();<a name="line.378"></a>
-<span class="sourceLineNo">379</span>    newtask.batch = batch;<a name="line.379"></a>
-<span class="sourceLineNo">380</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    if (oldtask == null) {<a name="line.381"></a>
-<span class="sourceLineNo">382</span>      batch.installed++;<a name="line.382"></a>
-<span class="sourceLineNo">383</span>      return null;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    }<a name="line.384"></a>
-<span class="sourceLineNo">385</span>    // new task was not used.<a name="line.385"></a>
-<span class="sourceLineNo">386</span>    synchronized (oldtask) {<a name="line.386"></a>
-<span class="sourceLineNo">387</span>      if (oldtask.isOrphan()) {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>        if (oldtask.status == SUCCESS) {<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          // The task is already done. Do not install the batch for this<a name="line.389"></a>
-<span class="sourceLineNo">390</span>          // task because it might be too late for setDone() to update<a name="line.390"></a>
-<span class="sourceLineNo">391</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.391"></a>
-<span class="sourceLineNo">392</span>          // this task to complete.<a name="line.392"></a>
-<span class="sourceLineNo">393</span>          return (null);<a name="line.393"></a>
-<span class="sourceLineNo">394</span>        }<a name="line.394"></a>
-<span class="sourceLineNo">395</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.395"></a>
-<span class="sourceLineNo">396</span>          oldtask.batch = batch;<a name="line.396"></a>
-<span class="sourceLineNo">397</span>          batch.installed++;<a name="line.397"></a>
-<span class="sourceLineNo">398</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.398"></a>
-<span class="sourceLineNo">399</span>          return null;<a name="line.399"></a>
-<span class="sourceLineNo">400</span>        }<a name="line.400"></a>
-<span class="sourceLineNo">401</span>        while (oldtask.status == FAILURE) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.402"></a>
-<span class="sourceLineNo">403</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.403"></a>
-<span class="sourceLineNo">404</span>          try {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>            oldtask.wait();<a name="line.405"></a>
-<span class="sourceLineNo">406</span>          } catch (InterruptedException e) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span>            Thread.currentThread().interrupt();<a name="line.407"></a>
-<span class="sourceLineNo">408</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.408"></a>
-<span class="sourceLineNo">409</span>            // fall through to return failure<a name="line.409"></a>
-<span class="sourceLineNo">410</span>            break;<a name="line.410"></a>
-<span class="sourceLineNo">411</span>          }<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        }<a name="line.412"></a>
-<span class="sourceLineNo">413</span>        if (oldtask.status != DELETED) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span>          LOG.warn("Failure because previously failed task"<a name="line.414"></a>
-<span class="sourceLineNo">415</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.415"></a>
-<span class="sourceLineNo">416</span>          return oldtask;<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        }<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        // reinsert the newTask and it must succeed this time<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        if (t == null) {<a name="line.420"></a>
-<span class="sourceLineNo">421</span>          batch.installed++;<a name="line.421"></a>
-<span class="sourceLineNo">422</span>          return null;<a name="line.422"></a>
-<span class="sourceLineNo">423</span>        }<a name="line.423"></a>
-<span class="sourceLineNo">424</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        assert false : "Deleted task still present in tasks map";<a name="line.425"></a>
-<span class="sourceLineNo">426</span>        return t;<a name="line.426"></a>
-<span class="sourceLineNo">427</span>      }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.428"></a>
-<span class="sourceLineNo">429</span>      return oldtask;<a name="line.429"></a>
-<span class="sourceLineNo">430</span>    }<a name="line.430"></a>
-<span class="sourceLineNo">431</span>  }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span>  public void stop() {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (choreService != null) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      choreService.shutdown();<a name="line.435"></a>
-<span class="sourceLineNo">436</span>    }<a name="line.436"></a>
-<span class="sourceLineNo">437</span>    if (timeoutMonitor != null) {<a name="line.437"></a>
-<span class="sourceLineNo">438</span>      timeoutMonitor.cancel(true);<a name="line.438"></a>
-<span class="sourceLineNo">439</span>    }<a name="line.439"></a>
-<span class="sourceLineNo">440</span>  }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>  void handleDeadWorker(ServerName workerName) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.443"></a>
-<span class="sourceLineNo">444</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.444"></a>
-<span class="sourceLineNo">445</span>    synchronized (deadWorkersLock) {<a name="line.445"></a>
-<span class="sourceLineNo">446</span>      if (deadWorkers == null) {<a name="line.446"></a>
-<span class="sourceLineNo">447</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>      }<a name="line.448"></a>
-<span class="sourceLineNo">449</span>      deadWorkers.add(workerName);<a name="line.449"></a>
-<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    LOG.info("dead splitlog worker " + workerName);<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  }<a name="line.452"></a>
-<span class="sourceLineNo">453</span><a name="line.453"></a>
-<span class="sourceLineNo">454</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.454"></a>
-<span class="sourceLineNo">455</span>    synchronized (deadWorkersLock) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span>      if (deadWorkers == null) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      }<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      deadWorkers.addAll(serverNames);<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.461"></a>
-<span class="sourceLineNo">462</span>  }<a name="line.462"></a>
-<span class="sourceLineNo">463</span><a name="line.463"></a>
-<span class="sourceLineNo">464</span>  /**<a name="line.464"></a>
-<span class="sourceLineNo">465</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.465"></a>
-<span class="sourceLineNo">466</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.466"></a>
-<span class="sourceLineNo">467</span>   * &lt;p&gt;<a name="line.467"></a>
-<span class="sourceLineNo">468</span>   * All access is synchronized.<a name="line.468"></a>
-<span class="sourceLineNo">469</span>   */<a name="line.469"></a>
-<span class="sourceLineNo">470</span>  @InterfaceAudience.Private<a name="line.470"></a>
-<span class="sourceLineNo">471</span>  public static class TaskBatch {<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    public int installed = 0;<a name="line.472"></a>
-<span class="sourceLineNo">473</span>    public int done = 0;<a name="line.473"></a>
-<span class="sourceLineNo">474</span>    public int error = 0;<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    public volatile boolean isDead = false;<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span>    @Override<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    public String toString() {<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.479"></a>
-<span class="sourceLineNo">480</span>    }<a name="line.480"></a>
-<span class="sourceLineNo">481</span>  }<a name="line.481"></a>
-<span class="sourceLineNo">482</span><a name="line.482"></a>
-<span class="sourceLineNo">483</span>  /**<a name="line.483"></a>
-<span class="sourceLineNo">484</span>   * in memory state of an active task.<a name="line.484"></a>
-<span class="sourceLineNo">485</span>   */<a name="line.485"></a>
-<span class="sourceLineNo">486</span>  @InterfaceAudience.Private<a name="line.486"></a>
-<span class="sourceLineNo">487</span>  public static class Task {<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    public volatile long last_update;<a name="line.488"></a>
-<span class="sourceLineNo">489</span>    public volatile int last_version;<a name="line.489"></a>
-<span class="sourceLineNo">490</span>    public volatile ServerName cur_worker_name;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>    public volatile TaskBatch batch;<a name="line.491"></a>
-<span class="sourceLineNo">492</span>    public volatile TerminationStatus status;<a name="line.492"></a>
-<span class="sourceLineNo">493</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.493"></a>
-<span class="sourceLineNo">494</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    public volatile boolean resubmitThresholdReached;<a name="line.495"></a>
-<span class="sourceLineNo">496</span><a name="line.496"></a>
-<span class="sourceLineNo">497</span>    @Override<a name="line.497"></a>
-<span class="sourceLineNo">498</span>    public String toString() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.499"></a>
-<span class="sourceLineNo">500</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.500"></a>
-<span class="sourceLineNo">501</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.501"></a>
-<span class="sourceLineNo">502</span>    }<a name="line.502"></a>
-<span class="sourceLineNo">503</span><a name="line.503"></a>
-<span class="sourceLineNo">504</span>    public Task() {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      last_version = -1;<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      status = IN_PROGRESS;<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      setUnassigned();<a name="line.507"></a>
-<span class="sourceLineNo">508</span>    }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span>    public boolean isOrphan() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      return (batch == null || batch.isDead);<a name="line.511"></a>
+<span class="sourceLineNo">238</span>    long totalSize = 0;<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    TaskBatch batch = null;<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    long startTime = 0;<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.241"></a>
+<span class="sourceLineNo">242</span>    if (logfiles.length != 0) {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>      status.setStatus("Checking directory contents...");<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.245"></a>
+<span class="sourceLineNo">246</span>          " for " + serverNames);<a name="line.246"></a>
+<span class="sourceLineNo">247</span>      startTime = EnvironmentEdgeManager.currentTime();<a name="line.247"></a>
+<span class="sourceLineNo">248</span>      batch = new TaskBatch();<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      for (FileStatus lf : logfiles) {<a name="line.249"></a>
+<span class="sourceLineNo">250</span>        // TODO If the log file is still being written to - which is most likely<a name="line.250"></a>
+<span class="sourceLineNo">251</span>        // the case for the last log file - then its length will show up here<a name="line.251"></a>
+<span class="sourceLineNo">252</span>        // as zero. The size of such a file can only be retrieved after<a name="line.252"></a>
+<span class="sourceLineNo">253</span>        // recover-lease is done. totalSize will be under in most cases and the<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        // metrics that it drives will also be under-reported.<a name="line.254"></a>
+<span class="sourceLineNo">255</span>        totalSize += lf.getLen();<a name="line.255"></a>
+<span class="sourceLineNo">256</span>        String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.257"></a>
+<span class="sourceLineNo">258</span>          throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.258"></a>
+<span class="sourceLineNo">259</span>        }<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      }<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      waitForSplittingCompletion(batch, status);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    if (batch != null &amp;&amp; batch.done != batch.installed) {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      batch.isDead = true;<a name="line.265"></a>
+<span class="sourceLineNo">266</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.267"></a>
+<span class="sourceLineNo">268</span>          + " but only " + batch.done + " done");<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.269"></a>
+<span class="sourceLineNo">270</span>      status.abort(msg);<a name="line.270"></a>
+<span class="sourceLineNo">271</span>      throw new IOException(msg);<a name="line.271"></a>
+<span class="sourceLineNo">272</span>    }<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    for (Path logDir : logDirs) {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      status.setStatus("Cleaning up log directory...");<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.275"></a>
+<span class="sourceLineNo">276</span>      try {<a name="line.276"></a>
+<span class="sourceLineNo">277</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.277"></a>
+<span class="sourceLineNo">278</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.278"></a>
+<span class="sourceLineNo">279</span>        }<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      } catch (IOException ioe) {<a name="line.280"></a>
+<span class="sourceLineNo">281</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.281"></a>
+<span class="sourceLineNo">282</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.282"></a>
+<span class="sourceLineNo">283</span>          LOG.warn("Returning success without actually splitting and "<a name="line.283"></a>
+<span class="sourceLineNo">284</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.284"></a>
+<span class="sourceLineNo">285</span>              + Arrays.toString(files), ioe);<a name="line.285"></a>
+<span class="sourceLineNo">286</span>        } else {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.287"></a>
+<span class="sourceLineNo">288</span>        }<a name="line.288"></a>
+<span class="sourceLineNo">289</span>      }<a name="line.289"></a>
+<span class="sourceLineNo">290</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.290"></a>
+<span class="sourceLineNo">291</span>    }<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    String msg = "Finished splitting (more than or equal to) " + totalSize +<a name="line.292"></a>
+<span class="sourceLineNo">293</span>        " bytes in " + ((batch == null)? 0: batch.installed) +<a name="line.293"></a>
+<span class="sourceLineNo">294</span>        " log files in " + logDirs + " in " +<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        ((startTime == 0)? startTime: (EnvironmentEdgeManager.currentTime() - startTime)) + "ms";<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    status.markComplete(msg);<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    LOG.info(msg);<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    return totalSize;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
+<span class="sourceLineNo">300</span><a name="line.300"></a>
+<span class="sourceLineNo">301</span>  /**<a name="line.301"></a>
+<span class="sourceLineNo">302</span>   * Add a task entry to coordination if it is not already there.<a name="line.302"></a>
+<span class="sourceLineNo">303</span>   * @param taskname the path of the log to be split<a name="line.303"></a>
+<span class="sourceLineNo">304</span>   * @param batch the batch this task belongs to<a name="line.304"></a>
+<span class="sourceLineNo">305</span>   * @return true if a new entry is created, false if it is already there.<a name="line.305"></a>
+<span class="sourceLineNo">306</span>   */<a name="line.306"></a>
+<span class="sourceLineNo">307</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.307"></a>
+<span class="sourceLineNo">308</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.309"></a>
+<span class="sourceLineNo">310</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.310"></a>
+<span class="sourceLineNo">311</span>    if (oldtask == null) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span>      // publish the task in the coordination engine<a name="line.312"></a>
+<span class="sourceLineNo">313</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.313"></a>
+<span class="sourceLineNo">314</span>      return true;<a name="line.314"></a>
+<span class="sourceLineNo">315</span>    }<a name="line.315"></a>
+<span class="sourceLineNo">316</span>    return false;<a name="line.316"></a>
+<span class="sourceLineNo">317</span>  }<a name="line.317"></a>
+<span class="sourceLineNo">318</span><a name="line.318"></a>
+<span class="sourceLineNo">319</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    synchronized (batch) {<a name="line.320"></a>
+<span class="sourceLineNo">321</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.321"></a>
+<span class="sourceLineNo">322</span>        try {<a name="line.322"></a>
+<span class="sourceLineNo">323</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.323"></a>
+<span class="sourceLineNo">324</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.324"></a>
+<span class="sourceLineNo">325</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.325"></a>
+<span class="sourceLineNo">326</span>          int actual = activeTasks(batch);<a name="line.326"></a>
+<span class="sourceLineNo">327</span>          if (remaining != actual) {<a name="line.327"></a>
+<span class="sourceLineNo">328</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.328"></a>
+<span class="sourceLineNo">329</span>          }<a name="line.329"></a>
+<span class="sourceLineNo">330</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.330"></a>
+<span class="sourceLineNo">331</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.331"></a>
+<span class="sourceLineNo">332</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.332"></a>
+<span class="sourceLineNo">333</span>                + remainingTasks);<a name="line.333"></a>
+<span class="sourceLineNo">334</span>          }<a name="line.334"></a>
+<span class="sourceLineNo">335</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.335"></a>
+<span class="sourceLineNo">336</span>            LOG.warn("No more task remaining, splitting "<a name="line.336"></a>
+<span class="sourceLineNo">337</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.337"></a>
+<span class="sourceLineNo">338</span>                + ", active tasks in map " + actual);<a name="line.338"></a>
+<span class="sourceLineNo">339</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.339"></a>
+<span class="sourceLineNo">340</span>              return;<a name="line.340"></a>
+<span class="sourceLineNo">341</span>            }<a name="line.341"></a>
+<span class="sourceLineNo">342</span>          }<a name="line.342"></a>
+<span class="sourceLineNo">343</span>          batch.wait(100);<a name="line.343"></a>
+<span class="sourceLineNo">344</span>          if (server.isStopped()) {<a name="line.344"></a>
+<span class="sourceLineNo">345</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.345"></a>
+<span class="sourceLineNo">346</span>            return;<a name="line.346"></a>
+<span class="sourceLineNo">347</span>          }<a name="line.347"></a>
+<span class="sourceLineNo">348</span>        } catch (InterruptedException e) {<a name="line.348"></a>
+<span class="sourceLineNo">349</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.349"></a>
+<span class="sourceLineNo">350</span>          Thread.currentThread().interrupt();<a name="line.350"></a>
+<span class="sourceLineNo">351</span>          return;<a name="line.351"></a>
+<span class="sourceLineNo">352</span>        }<a name="line.352"></a>
+<span class="sourceLineNo">353</span>      }<a name="line.353"></a>
+<span class="sourceLineNo">354</span>    }<a name="line.354"></a>
+<span class="sourceLineNo">355</span>  }<a name="line.355"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span>  @VisibleForTesting<a name="line.357"></a>
+<span class="sourceLineNo">358</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.358"></a>
+<span class="sourceLineNo">359</span>    return tasks;<a name="line.359"></a>
+<span class="sourceLineNo">360</span>  }<a name="line.360"></a>
+<span class="sourceLineNo">361</span><a name="line.361"></a>
+<span class="sourceLineNo">362</span>  private int activeTasks(final TaskBatch batch) {<a name="line.362"></a>
+<span class="sourceLineNo">363</span>    int count = 0;<a name="line.363"></a>
+<span class="sourceLineNo">364</span>    for (Task t : tasks.values()) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.365"></a>
+<span class="sourceLineNo">366</span>        count++;<a name="line.366"></a>
+<span class="sourceLineNo">367</span>      }<a name="line.367"></a>
+<span class="sourceLineNo">368</span>    }<a name="line.368"></a>
+<span class="sourceLineNo">369</span>    return count;<a name="line.369"></a>
+<span class="sourceLineNo">370</span><a name="line.370"></a>
+<span class="sourceLineNo">371</span>  }<a name="line.371"></a>
+<span class="sourceLineNo">372</span><a name="line.372"></a>
+<span class="sourceLineNo">373</span>  /**<a name="line.373"></a>
+<span class="sourceLineNo">374</span>   * @param path<a name="line.374"></a>
+<span class="sourceLineNo">375</span>   * @param batch<a name="line.375"></a>
+<span class="sourceLineNo">376</span>   * @return null on success, existing task on error<a name="line.376"></a>
+<span class="sourceLineNo">377</span>   */<a name="line.377"></a>
+<span class="sourceLineNo">378</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.378"></a>
+<span class="sourceLineNo">379</span>    Task oldtask;<a name="line.379"></a>
+<span class="sourceLineNo">380</span>    // batch.installed is only changed via this function and<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    // a single thread touches batch.installed.<a name="line.381"></a>
+<span class="sourceLineNo">382</span>    Task newtask = new Task();<a name="line.382"></a>
+<span class="sourceLineNo">383</span>    newtask.batch = batch;<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.384"></a>
+<span class="sourceLineNo">385</span>    if (oldtask == null) {<a name="line.385"></a>
+<span class="sourceLineNo">386</span>      batch.installed++;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      return null;<a name="line.387"></a>
+<span class="sourceLineNo">388</span>    }<a name="line.388"></a>
+<span class="sourceLineNo">389</span>    // new task was not used.<a name="line.389"></a>
+<span class="sourceLineNo">390</span>    synchronized (oldtask) {<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      if (oldtask.isOrphan()) {<a name="line.391"></a>
+<span class="sourceLineNo">392</span>        if (oldtask.status == SUCCESS) {<a name="line.392"></a>
+<span class="sourceLineNo">393</span>          // The task is already done. Do not install the batch for this<a name="line.393"></a>
+<span class="sourceLineNo">394</span>          // task because it might be too late for setDone() to update<a name="line.394"></a>
+<span class="sourceLineNo">395</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.395"></a>
+<span class="sourceLineNo">396</span>          // this task to complete.<a name="line.396"></a>
+<span class="sourceLineNo">397</span>          return (null);<a name="line.397"></a>
+<span class="sourceLineNo">398</span>        }<a name="line.398"></a>
+<span class="sourceLineNo">399</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.399"></a>
+<span class="sourceLineNo">400</span>          oldtask.batch = batch;<a name="line.400"></a>
+<span class="sourceLineNo">401</span>          batch.installed++;<a name="line.401"></a>
+<span class="sourceLineNo">402</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.402"></a>
+<span class="sourceLineNo">403</span>          return null;<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        }<a name="line.404"></a>
+<span class="sourceLineNo">405</span>        while (oldtask.status == FAILURE) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.406"></a>
+<span class="sourceLineNo">407</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.407"></a>
+<span class="sourceLineNo">408</span>          try {<a name="line.408"></a>
+<span class="sourceLineNo">409</span>            oldtask.wait();<a name="line.409"></a>
+<span class="sourceLineNo">410</span>          } catch (InterruptedException e) {<a name="line.410"></a>
+<span class="sourceLineNo">411</span>            Thread.currentThread().interrupt();<a name="line.411"></a>
+<span class="sourceLineNo">412</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.412"></a>
+<span class="sourceLineNo">413</span>            // fall through to return failure<a name="line.413"></a>
+<span class="sourceLineNo">414</span>            break;<a name="line.414"></a>
+<span class="sourceLineNo">415</span>          }<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        }<a name="line.416"></a>
+<span class="sourceLineNo">417</span>        if (oldtask.status != DELETED) {<a name="line.417"></a>
+<span class="sourceLineNo">418</span>          LOG.warn("Failure because previously failed task"<a name="line.418"></a>
+<span class="sourceLineNo">419</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.419"></a>
+<span class="sourceLineNo">420</span>          return oldtask;<a name="line.420"></a>
+<span class="sourceLineNo">421</span>        }<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        // reinsert the newTask and it must succeed this time<a name="line.422"></a>
+<span class="sourceLineNo">423</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        if (t == null) {<a name="line.424"></a>
+<span class="sourceLineNo">425</span>          batch.installed++;<a name="line.425"></a>
+<span class="sourceLineNo">426</span>          return null;<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        }<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.428"></a>
+<span class="sourceLineNo">429</span>        assert false : "Deleted task still present in tasks map";<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        return t;<a name="line.430"></a>
+<span class="sourceLineNo">431</span>      }<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.432"></a>
+<span class="sourceLineNo">433</span>      return oldtask;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>    }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>  }<a name="line.435"></a>
+<span class="sourceLineNo">436</span><a name="line.436"></a>
+<span class="sourceLineNo">437</span>  public void stop() {<a name="line.437"></a>
+<span class="sourceLineNo">438</span>    if (choreService != null) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span>      choreService.shutdown();<a name="line.439"></a>
+<span class="sourceLineNo">440</span>    }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    if (timeoutMonitor != null) {<a name="line.441"></a>
+<span class="sourceLineNo">442</span>      timeoutMonitor.cancel(true);<a name="line.442"></a>
+<span class="sourceLineNo">443</span>    }<a name="line.443"></a>
+<span class="sourceLineNo">444</span>  }<a name="line.444"></a>
+<span class="sourceLineNo">445</span><a name="line.445"></a>
+<span class="sourceLineNo">446</span>  void handleDeadWorker(ServerName workerName) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.447"></a>
+<span class="sourceLineNo">448</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.448"></a>
+<span class="sourceLineNo">449</span>    synchronized (deadWorkersLock) {<a name="line.449"></a>
+<span class="sourceLineNo">450</span>      if (deadWorkers == null) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      }<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      deadWorkers.add(workerName);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    }<a name="line.454"></a>
+<span class="sourceLineNo">455</span>    LOG.info("Dead splitlog worker {}", workerName);<a name="line.455"></a>
+<span class="sourceLineNo">456</span>  }<a name="line.456"></a>
+<span class="sourceLineNo">457</span><a name="line.457"></a>
+<span class="sourceLineNo">458</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>    synchronized (deadWorkersLock) {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>      if (deadWorkers == null) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.461"></a>
+<span class="sourceLineNo">462</span>      }<a name="line.462"></a>
+<span class="sourceLineNo">463</span>      deadWorkers.addAll(serverNames);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    }<a name="line.464"></a>
+<span class="sourceLineNo">465</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.465"></a>
+<span class="sourceLineNo">466</span>  }<a name="line.466"></a>
+<span class="sourceLineNo">467</span><a name="line.467"></a>
+<span class="sourceLineNo">468</span>  /**<a name="line.468"></a>
+<span class="sourceLineNo">469</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.469"></a>
+<span class="sourceLineNo">470</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.470"></a>
+<span class="sourceLineNo">471</span>   * &lt;p&gt;<a name="line.471"></a>
+<span class="sourceLineNo">472</span>   * All access is synchronized.<a name="line.472"></a>
+<span class="sourceLineNo">473</span>   */<a name="line.473"></a>
+<span class="sourceLineNo">474</span>  @InterfaceAudience.Private<a name="line.474"></a>
+<span class="sourceLineNo">475</span>  public static class TaskBatch {<a name="line.475"></a>
+<span class="sourceLineNo">476</span>    public int installed = 0;<a name="line.476"></a>
+<span class="sourceLineNo">477</span>    public int done = 0;<a name="line.477"></a>
+<span class="sourceLineNo">478</span>    public int error = 0;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    public volatile boolean isDead = false;<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span>    @Override<a name="line.481"></a>
+<span class="sourceLineNo">482</span>    public String toString() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    }<a name="line.484"></a>
+<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
+<span class="sourceLineNo">486</span><a name="line.486"></a>
+<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
+<span class="sourceLineNo">488</span>   * in memory state of an active task.<a name="line.488"></a>
+<span class="sourceLineNo">489</span>   */<a name="line.489"></a>
+<span class="sourceLineNo">490</span>  @InterfaceAudience.Private<a name="line.490"></a>
+<span class="sourceLineNo">491</span>  public static class Task {<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    public volatile long last_update;<a name="line.492"></a>
+<span class="sourceLineNo">493</span>    public volatile int last_version;<a name="line.493"></a>
+<span class="sourceLineNo">494</span>    public volatile ServerName cur_worker_name;<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    public volatile TaskBatch batch;<a name="line.495"></a>
+<span class="sourceLineNo">496</span>    public volatile TerminationStatus status;<a name="line.496"></a>
+<span class="sourceLineNo">497</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.497"></a>
+<span class="sourceLineNo">498</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.498"></a>
+<span class="sourceLineNo">499</span>    public volatile boolean resubmitThresholdReached;<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span>    @Override<a name="line.501"></a>
+<span class="sourceLineNo">502</span>    public String toString() {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.503"></a>
+<span class="sourceLineNo">504</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.504"></a>
+<span class="sourceLineNo">505</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.505"></a>
+<span class="sourceLineNo">506</span>    }<a name="line.506"></a>
+<span class="sourceLineNo">507</span><a name="line.507"></a>
+<span class="sourceLineNo">508</span>    public Task() {<a name="line.508"></a>
+<span class="sourceLineNo">509</span>      last_version = -1;<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      status = IN_PROGRESS;<a name="line.510"></a>
+<span class="sourceLineNo">511</span>      setUnassigned();<a name="line.511"></a>
 <span class="sourceLineNo">512</span>    }<a name="line.512"></a>
 <span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span>    public boolean isUnassigned() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span>      return (cur_worker_name == null);<a name="line.515"></a>
+<span class="sourceLineNo">514</span>    public boolean isOrphan() {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>      return (batch == null || batch.isDead);<a name="line.515"></a>
 <span class="sourceLineNo">516</span>    }<a name="line.516"></a>
 <span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span>    public void heartbeatNoDetails(long time) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span>      last_update = time;<a name="line.519"></a>
+<span class="sourceLineNo">518</span>    public boolean isUnassigned() {<a name="line.518"></a>
+<span class="sourceLineNo">519</span>      return (cur_worker_name == null);<a name="line.519"></a>
 <span class="sourceLineNo">520</span>    }<a name="line.520"></a>
 <span class="sourceLineNo">521</span><a name="line.521"></a>
-<span class="sourceLineNo">522</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      last_version = version;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      last_update = time;<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      cur_worker_name = worker;<a name="line.525"></a>
-<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span>    public void setUnassigned() {<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      cur_worker_name = null;<a name="line.529"></a>
-<span class="sourceLineNo">530</span>      last_update = -1;<a name="line.530"></a>
-<span class="sourceLineNo">531</span>    }<a name="line.531"></a>
-<span class="sourceLineNo">532</span>  }<a name="line.532"></a>
-<span class="sourceLineNo">533</span><a name="line.533"></a>
-<span class="sourceLineNo">534</span>  /**<a name="line.534"></a>
-<span class="sourceLineNo">535</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.535"></a>
-<span class="sourceLineNo">536</span>   */<a name="line.536"></a>
-<span class="sourceLineNo">537</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.537"></a>
-<span class="sourceLineNo">538</span>    private long lastLog = 0;<a name="line.538"></a>
-<span class="sourceLineNo">539</span><a name="line.539"></a>
-<span class="sourceLineNo">540</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.540"></a>
-<span class="sourceLineNo">541</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.541"></a>
-<span class="sourceLineNo">542</span>    }<a name="line.542"></a>
+<span class="sourceLineNo">522</span>    public void heartbeatNoDetails(long time) {<a name="line.522"></a>
+<span class="sourceLineNo">523</span>      last_update = time;<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    }<a name="line.524"></a>
+<span class="sourceLineNo">525</span><a name="line.525"></a>
+<span class="sourceLineNo">526</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span>      last_version = version;<a name="line.527"></a>
+<span class="sourceLineNo">528</span>      last_update = time;<a name="line.528"></a>
+<span class="sourceLineNo">529</span>      cur_worker_name = worker;<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    }<a name="line.530"></a>
+<span class="sourceLineNo">531</span><a name="line.531"></a>
+<span class="sourceLineNo">532</span>    public void setUnassigned() {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>      cur_worker_name = null;<a name="line.533"></a>
+<span class="sourceLineNo">534</span>      last_update = -1;<a name="line.534"></a>
+<span class="sourceLineNo">535</span>    }<a name="line.535"></a>
+<span class="sourceLineNo">536</span>  }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>  /**<a name="line.538"></a>
+<span class="sourceLineNo">539</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.539"></a>
+<span class="sourceLineNo">540</span>   */<a name="line.540"></a>
+<span class="sourceLineNo">541</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.541"></a>
+<span class="sourceLineNo">542</span>    private long lastLog = 0;<a name="line.542"></a>
 <span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>    @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    protected void chore() {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.546"></a>
+<span class="sourceLineNo">544</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    }<a name="line.546"></a>
 <span class="sourceLineNo">547</span><a name="line.547"></a>
-<span class="sourceLineNo">548</span>      int resubmitted = 0;<a name="line.548"></a>
-<span class="sourceLineNo">549</span>      int unassigned = 0;<a name="line.549"></a>
-<span class="sourceLineNo">550</span>      int tot = 0;<a name="line.550"></a>
-<span class="sourceLineNo">551</span>      boolean found_assigned_task = false;<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.552"></a>
-<span class="sourceLineNo">553</span><a name="line.553"></a>
-<span class="sourceLineNo">554</span>      synchronized (deadWorkersLock) {<a name="line.554"></a>
-<span class="sourceLineNo">555</span>        localDeadWorkers = deadWorkers;<a name="line.555"></a>
-<span class="sourceLineNo">556</span>        deadWorkers = null;<a name="line.556"></a>
-<span class="sourceLineNo">557</span>      }<a name="line.557"></a>
-<span class="sourceLineNo">558</span><a name="line.558"></a>
-<span class="sourceLineNo">559</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>        String path = e.getKey();<a name="line.560"></a>
-<span class="sourceLineNo">561</span>        Task task = e.getValue();<a name="line.561"></a>
-<span class="sourceLineNo">562</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.562"></a>
-<span class="sourceLineNo">563</span>        tot++;<a name="line.563"></a>
-<span class="sourceLineNo">564</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.564"></a>
-<span class="sourceLineNo">565</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.565"></a>
-<span class="sourceLineNo">566</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.566"></a>
-<span class="sourceLineNo">567</span>        // time. If we want progress when there are no region servers then we<a name="line.567"></a>
-<span class="sourceLineNo">568</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.568"></a>
-<span class="sourceLineNo">569</span>        if (task.isUnassigned()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>          unassigned++;<a name="line.570"></a>
-<span class="sourceLineNo">571</span>          continue;<a name="line.571"></a>
-<span class="sourceLineNo">572</span>        }<a name="line.572"></a>
-<span class="sourceLineNo">573</span>        found_assigned_task = true;<a name="line.573"></a>
-<span class="sourceLineNo">574</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.575"></a>
-<span class="sourceLineNo">576</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.576"></a>
-<span class="sourceLineNo">577</span>            resubmitted++;<a name="line.577"></a>
-<span class="sourceLineNo">578</span>          } else {<a name="line.578"></a>
-<span class="sourceLineNo">579</span>            handleDeadWorker(cur_worker);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.580"></a>
-<span class="sourceLineNo">581</span>                + ", will retry.");<a name="line.581"></a>
-<span class="sourceLineNo">582</span>          }<a name="line.582"></a>
-<span class="sourceLineNo">583</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.583"></a>
-<span class="sourceLineNo">584</span>          resubmitted++;<a name="line.584"></a>
-<span class="sourceLineNo">585</span>        }<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      }<a name="line.586"></a>
-<span class="sourceLineNo">587</span>      if (tot &gt; 0) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.588"></a>
-<span class="sourceLineNo">589</span>        if (now &gt; lastLog + 5000) {<a name="line.589"></a>
-<span class="sourceLineNo">590</span>          lastLog = now;<a name="line.590"></a>
-<span class="sourceLineNo">591</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.591"></a>
-<span class="sourceLineNo">592</span>        }<a name="line.592"></a>
-<span class="sourceLineNo">593</span>      }<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      if (resubmitted &gt; 0) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.595"></a>
-<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
-<span class="sourceLineNo">597</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.598"></a>
-<span class="sourceLineNo">599</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // because a. it is very unlikely that every worker had a<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      // transient error when trying to grab the task b. if there are no<a name="line.601"></a>
-<span class="sourceLineNo">602</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.604"></a>
-<span class="sourceLineNo">605</span>      // that there is always one worker in the system<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      if (tot &gt; 0<a name="line.606"></a>
-<span class="sourceLineNo">607</span>          &amp;&amp; !found_assigned_task<a name="line.607"></a>
-<span class="sourceLineNo">608</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.608"></a>
-<span class="sourceLineNo">609</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.609"></a>
-<span class="sourceLineNo">610</span>          String key = e.getKey();<a name="line.610"></a>
-<span class="sourceLineNo">611</span>          Task task = e.getValue();<a name="line.611"></a>
-<span class="sourceLineNo">612</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.612"></a>
-<span class="sourceLineNo">613</span>          // have been asynchronously assigned. There is no locking required<a name="line.613"></a>
-<span class="sourceLineNo">614</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.614"></a>
-<span class="sourceLineNo">615</span>          // called unnecessarily for a taskpath<a name="line.615"></a>
-<span class="sourceLineNo">616</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>            // We just touch the znode to make sure its still there<a name="line.617"></a>
-<span class="sourceLineNo">618</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.618"></a>
-<span class="sourceLineNo">619</span>          }<a name="line.619"></a>
-<span class="sourceLineNo">620</span>        }<a name="line.620"></a>
-<span class="sourceLineNo">621</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.621"></a>
-<span class="sourceLineNo">622</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.623"></a>
-<span class="sourceLineNo">624</span>      }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      Set&lt;String&gt; failedDeletions =<a name="line.625"></a>
-<span class="sourceLineNo">626</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      // Retry previously failed deletes<a name="line.627"></a>
-<span class="sourceLineNo">628</span>      if (failedDeletions.size() &gt; 0) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.629"></a>
-<span class="sourceLineNo">630</span>        for (String tmpPath : tmpPaths) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>          // deleteNode is an async call<a name="line.631"></a>
-<span class="sourceLineNo">632</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.632"></a>
-<span class="sourceLineNo">633</span>        }<a name="line.633"></a>
-<span class="sourceLineNo">634</span>        failedDeletions.removeAll(tmpPaths);<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      }<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span>  }<a name="line.637"></a>
-<span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>  public enum ResubmitDirective {<a name="line.639"></a>
-<span class="sourceLineNo">640</span>    CHECK(), FORCE()<a name="line.640"></a>
+<span class="sourceLineNo">548</span>    @Override<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    protected void chore() {<a name="line.549"></a>
+<span class="sourceLineNo">550</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>      int resubmitted = 0;<a name="line.552"></a>
+<span class="sourceLineNo">553</span>      int unassigned = 0;<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      int tot = 0;<a name="line.554"></a>
+<span class="sourceLineNo">555</span>      boolean found_assigned_task = false;<a name="line.555"></a>
+<span class="sourceLineNo">556</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>      synchronized (deadWorkersLock) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>        localDeadWorkers = deadWorkers;<a name="line.559"></a>
+<span class="sourceLineNo">560</span>        deadWorkers = null;<a name="line.560"></a>
+<span class="sourceLineNo">561</span>      }<a name="line.561"></a>
+<span class="sourceLineNo">562</span><a name="line.562"></a>
+<span class="sourceLineNo">563</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.563"></a>
+<span class="sourceLineNo">564</span>        String path = e.getKey();<a name="line.564"></a>
+<span class="sourceLineNo">565</span>        Task task = e.getValue();<a name="line.565"></a>
+<span class="sourceLineNo">566</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.566"></a>
+<span class="sourceLineNo">567</span>        tot++;<a name="line.567"></a>
+<span class="sourceLineNo">568</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.568"></a>
+<span class="sourceLineNo">569</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.569"></a>
+<span class="sourceLineNo">570</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.570"></a>
+<span class="sourceLineNo">571</span>        // time. If we want progress when there are no region servers then we<a name="line.571"></a>
+<span class="sourceLineNo">572</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.572"></a>
+<span class="sourceLineNo">573</span>        if (task.isUnassigned()) {<a name="line.573"></a>
+<span class="sourceLineNo">574</span>          unassigned++;<a name="line.574"></a>
+<span class="sourceLineNo">575</span>          continue;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>        }<a name="line.576"></a>
+<span class="sourceLineNo">577</span>        found_assigned_task = true;<a name="line.577"></a>
+<span class="sourceLineNo">578</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.578"></a>
+<span class="sourceLineNo">579</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.579"></a>
+<span class="sourceLineNo">580</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.580"></a>
+<span class="sourceLineNo">581</span>            resubmitted++;<a name="line.581"></a>
+<span class="sourceLineNo">582</span>          } else {<a name="line.582"></a>
+<span class="sourceLineNo">583</span>            handleDeadWorker(cur_worker);<a name="line.583"></a>
+<span class="sourceLineNo">584</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.584"></a>
+<span class="sourceLineNo">585</span>                + ", will retry.");<a name="line.585"></a>
+<span class="sourceLineNo">586</span>          }<a name="line.586"></a>
+<span class="sourceLineNo">587</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>          resubmitted++;<a name="line.588"></a>
+<span class="sourceLineNo">589</span>        }<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      }<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      if (tot &gt; 0) {<a name="line.591"></a>
+<span class="sourceLineNo">592</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.592"></a>
+<span class="sourceLineNo">593</span>        if (now &gt; lastLog + 5000) {<a name="line.593"></a>
+<span class="sourceLineNo">594</span>          lastLog = now;<a name="line.594"></a>
+<span class="sourceLineNo">595</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.595"></a>
+<span class="sourceLineNo">596</span>        }<a name="line.596"></a>
+<span class="sourceLineNo">597</span>      }<a name="line.597"></a>
+<span class="sourceLineNo">598</span>      if (resubmitted &gt; 0) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.599"></a>
+<span class="sourceLineNo">600</span>      }<a name="line.600"></a>
+<span class="sourceLineNo">601</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.602"></a>
+<span class="sourceLineNo">603</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.603"></a>
+<span class="sourceLineNo">604</span>      // because a. it is very unlikely that every worker had a<a name="line.604"></a>
+<span class="sourceLineNo">605</span>      // transient error when trying to grab the task b. if there are no<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.608"></a>
+<span class="sourceLineNo">609</span>      // that there is always one worker in the system<a name="line.609"></a>
+<span class="sourceLineNo">610</span>      if (tot &gt; 0<a name="line.610"></a>
+<span class="sourceLineNo">611</span>          &amp;&amp; !found_assigned_task<a name="line.611"></a>
+<span class="sourceLineNo">612</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>          String key = e.getKey();<a name="line.614"></a>
+<span class="sourceLineNo">615</span>          Task task = e.getValue();<a name="line.615"></a>
+<span class="sourceLineNo">616</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.616"></a>
+<span class="sourceLineNo">617</span>          // have been asynchronously assigned. There is no locking required<a name="line.617"></a>
+<span class="sourceLineNo">618</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.618"></a>
+<span class="sourceLineNo">619</span>          // called unnecessarily for a taskpath<a name="line.619"></a>
+<span class="sourceLineNo">620</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.620"></a>
+<span class="sourceLineNo">621</span>            // We just touch the znode to make sure its still there<a name="line.621"></a>
+<span class="sourceLineNo">622</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.622"></a>
+<span class="sourceLineNo">623</span>          }<a name="line.623"></a>
+<span class="sourceLineNo">624</span>        }<a name="line.624"></a>
+<span class="sourceLineNo">625</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.625"></a>
+<span class="sourceLineNo">626</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.626"></a>
+<span class="sourceLineNo">627</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      }<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      Set&lt;String&gt; failedDeletions =<a name="line.629"></a>
+<span class="sourceLineNo">630</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.630"></a>
+<span class="sourceLineNo">631</span>      // Retry previously failed deletes<a name="line.631"></a>
+<span class="sourceLineNo">632</span>      if (failedDeletions.size() &gt; 0) {<a name="line.632"></a>
+<span class="sourceLineNo">633</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.633"></a>
+<span class="sourceLineNo">634</span>        for (String tmpPath : tmpPaths) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>          // deleteNode is an async call<a name="line.635"></a>
+<span class="sourceLineNo">636</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.636"></a>
+<span class="sourceLineNo">637</span>        }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>        failedDeletions.removeAll(tmpPaths);<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      }<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    }<a name="line.640"></a>
 <span class="sourceLineNo">641</span>  }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>  public enum TerminationStatus {<a name="line.643"></a>
-<span class="sourceLineNo">644</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.644"></a>
-<span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    final String statusMsg;<a name="line.646"></a>
-<span class="sourceLineNo">647</span><a name="line.647"></a>
-<span class="sourceLineNo">648</span>    TerminationStatus(String msg) {<a name="line.648"></a>
-<span class="sourceLineNo">649</span>      statusMsg = msg;<a name="line.649"></a>
-<span class="sourceLineNo">650</span>    }<a name="line.650"></a>
+<span class="sourceLineNo">643</span>  public enum ResubmitDirective {<a name="line.643"></a>
+<span class="sourceLineNo">644</span>    CHECK(), FORCE()<a name="line.644"></a>
+<span class="sourceLineNo">645</span>  }<a name="line.645"></a>
+<span class="sourceLineNo">646</span><a name="line.646"></a>
+<span class="sourceLineNo">647</span>  public enum TerminationStatus {<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.648"></a>
+<span class="sourceLineNo">649</span><a name="line.649"></a>
+<span class="sourceLineNo">650</span>    final String statusMsg;<a name="line.650"></a>
 <span class="sourceLineNo">651</span><a name="line.651"></a>
-<span class="sourceLineNo">652</span>    @Override<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    public String toString() {<a name="line.653"></a>
-<span class="sourceLineNo">654</span>      return statusMsg;<a name="line.654"></a>
-<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
-<span class="sourceLineNo">657</span>}<a name="line.657"></a>
+<span class="sourceLineNo">652</span>    TerminationStatus(String msg) {<a name="line.652"></a>
+<span class="sourceLineNo">653</span>      statusMsg = msg;<a name="line.653"></a>
+<span class="sourceLineNo">654</span>    }<a name="line.654"></a>
+<span class="sourceLineNo">655</span><a name="line.655"></a>
+<span class="sourceLineNo">656</span>    @Override<a name="line.656"></a>
+<span class="sourceLineNo">657</span>    public String toString() {<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      return statusMsg;<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    }<a name="line.659"></a>
+<span class="sourceLineNo">660</span>  }<a name="line.660"></a>
+<span class="sourceLineNo">661</span>}<a name="line.661"></a>
 
 
 


[20/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
index 8cce792..498bb60 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
@@ -1428,7 +1428,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>WALS_PATH_FILTER</h4>
-<pre>private static final&nbsp;org.apache.hadoop.fs.PathFilter <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1201">WALS_PATH_FILTER</a></pre>
+<pre>private static final&nbsp;org.apache.hadoop.fs.PathFilter <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1208">WALS_PATH_FILTER</a></pre>
 </li>
 </ul>
 <a name="FILE_STATUS_ID_COMPARATOR">
@@ -1437,7 +1437,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockListLast">
 <li class="blockList">
 <h4>FILE_STATUS_ID_COMPARATOR</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparator.html?is-external=true" title="class or interface in java.util">Comparator</a>&lt;org.apache.hadoop.fs.FileStatus&gt; <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1209">FILE_STATUS_ID_COMPARATOR</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparator.html?is-external=true" title="class or interface in java.util">Comparator</a>&lt;org.apache.hadoop.fs.FileStatus&gt; <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1216">FILE_STATUS_ID_COMPARATOR</a></pre>
 </li>
 </ul>
 </li>
@@ -1624,7 +1624,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>tryCleanupLogsOnLoad</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.463">tryCleanupLogsOnLoad</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.470">tryCleanupLogsOnLoad</a>()</pre>
 </li>
 </ul>
 <a name="insert-org.apache.hadoop.hbase.procedure2.Procedure-org.apache.hadoop.hbase.procedure2.Procedure:A-">
@@ -1633,7 +1633,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>insert</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.482">insert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.489">insert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc,
                    <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>[]&nbsp;subprocs)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html#insert-org.apache.hadoop.hbase.procedure2.Procedure-org.apache.hadoop.hbase.procedure2.Procedure:A-">ProcedureStore</a></code></span></div>
 <div class="block">When a procedure is submitted to the executor insert(proc, null) will be called.
@@ -1655,7 +1655,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>insert</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.516">insert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>[]&nbsp;procs)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.523">insert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>[]&nbsp;procs)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html#insert-org.apache.hadoop.hbase.procedure2.Procedure:A-">ProcedureStore</a></code></span></div>
 <div class="block">Serialize a set of new procedures.
  These procedures are freshly submitted to the executor and each procedure
@@ -1672,7 +1672,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>update</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.545">update</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.552">update</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html#update-org.apache.hadoop.hbase.procedure2.Procedure-">ProcedureStore</a></code></span></div>
 <div class="block">The specified procedure was executed,
  and the new state should be written to the store.</div>
@@ -1688,7 +1688,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>delete</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.568">delete</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.575">delete</a>(long&nbsp;procId)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html#delete-long-">ProcedureStore</a></code></span></div>
 <div class="block">The specified procId was removed from the executor,
  due to completion, abort or failure.
@@ -1705,7 +1705,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>delete</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.591">delete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.598">delete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&nbsp;proc,
                    long[]&nbsp;subProcIds)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html#delete-org.apache.hadoop.hbase.procedure2.Procedure-long:A-">ProcedureStore</a></code></span></div>
 <div class="block">The parent procedure completed.
@@ -1723,7 +1723,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>delete</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.616">delete</a>(long[]&nbsp;procIds,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.623">delete</a>(long[]&nbsp;procIds,
                    int&nbsp;offset,
                    int&nbsp;count)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html#delete-long:A-int-int-">ProcedureStore</a></code></span></div>
@@ -1744,7 +1744,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>delete</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.627">delete</a>(long[]&nbsp;procIds)</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.634">delete</a>(long[]&nbsp;procIds)</pre>
 </li>
 </ul>
 <a name="acquireSlot--">
@@ -1753,7 +1753,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>acquireSlot</h4>
-<pre>private&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.651">acquireSlot</a>()</pre>
+<pre>private&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.658">acquireSlot</a>()</pre>
 </li>
 </ul>
 <a name="releaseSlot-org.apache.hadoop.hbase.procedure2.util.ByteSlot-">
@@ -1762,7 +1762,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>releaseSlot</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.656">releaseSlot</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot)</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.663">releaseSlot</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot)</pre>
 </li>
 </ul>
 <a name="pushData-org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.PushType-org.apache.hadoop.hbase.procedure2.util.ByteSlot-long-long:A-">
@@ -1771,7 +1771,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>pushData</h4>
-<pre>private&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.663">pushData</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a>&nbsp;type,
+<pre>private&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.670">pushData</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a>&nbsp;type,
                       <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                       long&nbsp;procId,
                       long[]&nbsp;subProcIds)</pre>
@@ -1783,7 +1783,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>updateStoreTracker</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.723">updateStoreTracker</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a>&nbsp;type,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.730">updateStoreTracker</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html" title="enum in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.PushType</a>&nbsp;type,
                                 long&nbsp;procId,
                                 long[]&nbsp;subProcIds)</pre>
 </li>
@@ -1794,7 +1794,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>isSyncAborted</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.754">isSyncAborted</a>()</pre>
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.761">isSyncAborted</a>()</pre>
 </li>
 </ul>
 <a name="syncLoop--">
@@ -1803,7 +1803,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>syncLoop</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.758">syncLoop</a>()
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.765">syncLoop</a>()
                throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -1817,7 +1817,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getSyncMetrics</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.SyncMetrics</a>&gt;&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.838">getSyncMetrics</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore.SyncMetrics</a>&gt;&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.845">getSyncMetrics</a>()</pre>
 </li>
 </ul>
 <a name="syncSlots--">
@@ -1826,7 +1826,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>syncSlots</h4>
-<pre>private&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.847">syncSlots</a>()
+<pre>private&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.854">syncSlots</a>()
                 throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -1840,7 +1840,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>syncSlots</h4>
-<pre>protected&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.875">syncSlots</a>(org.apache.hadoop.fs.FSDataOutputStream&nbsp;stream,
+<pre>protected&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.882">syncSlots</a>(org.apache.hadoop.fs.FSDataOutputStream&nbsp;stream,
                          <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>[]&nbsp;slots,
                          int&nbsp;offset,
                          int&nbsp;count)
@@ -1857,7 +1857,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>syncStream</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.894">syncStream</a>(org.apache.hadoop.fs.FSDataOutputStream&nbsp;stream)
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.901">syncStream</a>(org.apache.hadoop.fs.FSDataOutputStream&nbsp;stream)
                    throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -1871,7 +1871,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>rollWriterWithRetries</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.902">rollWriterWithRetries</a>()</pre>
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.909">rollWriterWithRetries</a>()</pre>
 </li>
 </ul>
 <a name="tryRollWriter--">
@@ -1880,7 +1880,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>tryRollWriter</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.918">tryRollWriter</a>()</pre>
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.925">tryRollWriter</a>()</pre>
 </li>
 </ul>
 <a name="getMillisToNextPeriodicRoll--">
@@ -1889,7 +1889,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getMillisToNextPeriodicRoll</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.927">getMillisToNextPeriodicRoll</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.934">getMillisToNextPeriodicRoll</a>()</pre>
 </li>
 </ul>
 <a name="getMillisFromLastRoll--">
@@ -1898,7 +1898,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getMillisFromLastRoll</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.934">getMillisFromLastRoll</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.941">getMillisFromLastRoll</a>()</pre>
 </li>
 </ul>
 <a name="periodicRollForTesting--">
@@ -1907,7 +1907,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>periodicRollForTesting</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.939">periodicRollForTesting</a>()
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.946">periodicRollForTesting</a>()
                                throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -1921,7 +1921,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>rollWriterForTesting</h4>
-<pre>protected&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.949">rollWriterForTesting</a>()
+<pre>protected&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.956">rollWriterForTesting</a>()
                                 throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -1935,7 +1935,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>removeInactiveLogsForTesting</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.959">removeInactiveLogsForTesting</a>()
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.966">removeInactiveLogsForTesting</a>()
                                      throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -1949,7 +1949,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>periodicRoll</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.968">periodicRoll</a>()
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.975">periodicRoll</a>()
                    throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -1963,7 +1963,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>rollWriter</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.993">rollWriter</a>()
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1000">rollWriter</a>()
                     throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -1977,7 +1977,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>rollWriter</h4>
-<pre>boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1015">rollWriter</a>(long&nbsp;logId)
+<pre>boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1022">rollWriter</a>(long&nbsp;logId)
             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -1991,7 +1991,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>closeCurrentLogStream</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1082">closeCurrentLogStream</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1089">closeCurrentLogStream</a>()</pre>
 </li>
 </ul>
 <a name="removeInactiveLogs--">
@@ -2000,7 +2000,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>removeInactiveLogs</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1107">removeInactiveLogs</a>()
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1114">removeInactiveLogs</a>()
                          throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2014,7 +2014,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>buildHoldingCleanupTracker</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1119">buildHoldingCleanupTracker</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1126">buildHoldingCleanupTracker</a>()</pre>
 </li>
 </ul>
 <a name="removeAllLogs-long-">
@@ -2023,7 +2023,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>removeAllLogs</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1140">removeAllLogs</a>(long&nbsp;lastLogId)</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1147">removeAllLogs</a>(long&nbsp;lastLogId)</pre>
 <div class="block">Remove all logs with logId <= <code>lastLogId</code>.</div>
 </li>
 </ul>
@@ -2033,7 +2033,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>removeLogFile</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1162">removeLogFile</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&nbsp;log,
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1169">removeLogFile</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&nbsp;log,
                               org.apache.hadoop.fs.Path&nbsp;walArchiveDir)</pre>
 </li>
 </ul>
@@ -2043,7 +2043,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getWALDir</h4>
-<pre>public&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1183">getWALDir</a>()</pre>
+<pre>public&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1190">getWALDir</a>()</pre>
 </li>
 </ul>
 <a name="getFileSystem--">
@@ -2052,7 +2052,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getFileSystem</h4>
-<pre>public&nbsp;org.apache.hadoop.fs.FileSystem&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1187">getFileSystem</a>()</pre>
+<pre>public&nbsp;org.apache.hadoop.fs.FileSystem&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1194">getFileSystem</a>()</pre>
 </li>
 </ul>
 <a name="getLogFilePath-long-">
@@ -2061,7 +2061,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getLogFilePath</h4>
-<pre>protected&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1191">getLogFilePath</a>(long&nbsp;logId)
+<pre>protected&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1198">getLogFilePath</a>(long&nbsp;logId)
                                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2075,7 +2075,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getLogIdFromName</h4>
-<pre>private static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1195">getLogIdFromName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
+<pre>private static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1202">getLogIdFromName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
 </li>
 </ul>
 <a name="getLogFiles--">
@@ -2084,7 +2084,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getLogFiles</h4>
-<pre>private&nbsp;org.apache.hadoop.fs.FileStatus[]&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1219">getLogFiles</a>()
+<pre>private&nbsp;org.apache.hadoop.fs.FileStatus[]&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1226">getLogFiles</a>()
                                                throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2098,7 +2098,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getMaxLogId</h4>
-<pre>private static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1230">getMaxLogId</a>(org.apache.hadoop.fs.FileStatus[]&nbsp;logFiles)</pre>
+<pre>private static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1237">getMaxLogId</a>(org.apache.hadoop.fs.FileStatus[]&nbsp;logFiles)</pre>
 </li>
 </ul>
 <a name="initOldLogs-org.apache.hadoop.fs.FileStatus:A-">
@@ -2107,7 +2107,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>initOldLogs</h4>
-<pre>private&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1243">initOldLogs</a>(org.apache.hadoop.fs.FileStatus[]&nbsp;logFiles)
+<pre>private&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1250">initOldLogs</a>(org.apache.hadoop.fs.FileStatus[]&nbsp;logFiles)
                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
@@ -2123,7 +2123,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>initTrackerFromOldLogs</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1272">initTrackerFromOldLogs</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1279">initTrackerFromOldLogs</a>()</pre>
 <div class="block">If last log's tracker is not null, use it as <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#storeTracker"><code>storeTracker</code></a>.
  Otherwise, set storeTracker as partial, and let <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal"><code>ProcedureWALFormatReader</code></a> rebuild
  it using entries in the log.</div>
@@ -2135,7 +2135,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockListLast">
 <li class="blockList">
 <h4>initOldLog</h4>
-<pre>private&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1286">initOldLog</a>(org.apache.hadoop.fs.FileStatus&nbsp;logFile,
+<pre>private&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1293">initOldLog</a>(org.apache.hadoop.fs.FileStatus&nbsp;logFile,
                                     org.apache.hadoop.fs.Path&nbsp;walArchiveDir)
                              throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Loads given log file and it's tracker.</div>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html
index 937666b..08f9e8b 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html
@@ -260,7 +260,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>values</h4>
-<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html" title="enum in org.apache.hadoop.hbase.regionserver">HRegion.FlushResult.Result</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html#line.575">values</a>()</pre>
+<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html" title="enum in org.apache.hadoop.hbase.regionserver">HRegion.FlushResult.Result</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html#line.580">values</a>()</pre>
 <div class="block">Returns an array containing the constants of this enum type, in
 the order they are declared.  This method may be used to iterate
 over the constants as follows:
@@ -280,7 +280,7 @@ for (HRegion.FlushResult.Result c : HRegion.FlushResult.Result.values())
 <ul class="blockListLast">
 <li class="blockList">
 <h4>valueOf</h4>
-<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html" title="enum in org.apache.hadoop.hbase.regionserver">HRegion.FlushResult.Result</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html#line.575">valueOf</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
+<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html" title="enum in org.apache.hadoop.hbase.regionserver">HRegion.FlushResult.Result</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html#line.580">valueOf</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
 <div class="block">Returns the enum constant of this type with the specified name.
 The string must match <i>exactly</i> an identifier used to declare an
 enum constant in this type.  (Extraneous whitespace characters are 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
index a7c8ce8..b4bf156 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
@@ -16,11 +16,11 @@
 <span class="sourceLineNo">008</span>@InterfaceAudience.Private<a name="line.8"></a>
 <span class="sourceLineNo">009</span>public class Version {<a name="line.9"></a>
 <span class="sourceLineNo">010</span>  public static final String version = "3.0.0-SNAPSHOT";<a name="line.10"></a>
-<span class="sourceLineNo">011</span>  public static final String revision = "f8b12805bbff18fc94e2ac6894f92276b1453350";<a name="line.11"></a>
+<span class="sourceLineNo">011</span>  public static final String revision = "24f2893059e745aa6691cdd6cb37fc248880b3b9";<a name="line.11"></a>
 <span class="sourceLineNo">012</span>  public static final String user = "jenkins";<a name="line.12"></a>
-<span class="sourceLineNo">013</span>  public static final String date = "Thu Sep  6 14:44:09 UTC 2018";<a name="line.13"></a>
+<span class="sourceLineNo">013</span>  public static final String date = "Fri Sep  7 14:43:58 UTC 2018";<a name="line.13"></a>
 <span class="sourceLineNo">014</span>  public static final String url = "git://jenkins-websites1.apache.org/home/jenkins/jenkins-slave/workspace/hbase_generate_website/hbase";<a name="line.14"></a>
-<span class="sourceLineNo">015</span>  public static final String srcChecksum = "e6959349aa0db5aaa6ff1d2a92b23943";<a name="line.15"></a>
+<span class="sourceLineNo">015</span>  public static final String srcChecksum = "7b31fbf46eb4d4d18049c8be8d853a38";<a name="line.15"></a>
 <span class="sourceLineNo">016</span>}<a name="line.16"></a>
 
 


[16/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
index a990153..d2da8f4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
@@ -593,7 +593,7 @@
 <span class="sourceLineNo">585</span>              abort(error, t);<a name="line.585"></a>
 <span class="sourceLineNo">586</span>            }<a name="line.586"></a>
 <span class="sourceLineNo">587</span>          }<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        }));<a name="line.588"></a>
+<span class="sourceLineNo">588</span>        }), getName() + ":becomeActiveMaster");<a name="line.588"></a>
 <span class="sourceLineNo">589</span>      }<a name="line.589"></a>
 <span class="sourceLineNo">590</span>      // Fall in here even if we have been aborted. Need to run the shutdown services and<a name="line.590"></a>
 <span class="sourceLineNo">591</span>      // the super run call will do this for us.<a name="line.591"></a>
@@ -932,7 +932,7 @@
 <span class="sourceLineNo">924</span>          HBaseFsck.createLockRetryCounterFactory(this.conf).create());<a name="line.924"></a>
 <span class="sourceLineNo">925</span>    }<a name="line.925"></a>
 <span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span>    status.setStatus("Initialze ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
+<span class="sourceLineNo">927</span>    status.setStatus("Initialize ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
 <span class="sourceLineNo">928</span>    this.serverManager = createServerManager(this);<a name="line.928"></a>
 <span class="sourceLineNo">929</span>    createProcedureExecutor();<a name="line.929"></a>
 <span class="sourceLineNo">930</span>    @SuppressWarnings("rawtypes")<a name="line.930"></a>
@@ -954,2890 +954,2893 @@
 <span class="sourceLineNo">946</span>        .collect(Collectors.toList());<a name="line.946"></a>
 <span class="sourceLineNo">947</span>    this.assignmentManager.setupRIT(ritList);<a name="line.947"></a>
 <span class="sourceLineNo">948</span><a name="line.948"></a>
-<span class="sourceLineNo">949</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.949"></a>
-<span class="sourceLineNo">950</span>    this.regionServerTracker.start(<a name="line.950"></a>
-<span class="sourceLineNo">951</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.951"></a>
-<span class="sourceLineNo">952</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.952"></a>
-<span class="sourceLineNo">953</span>      walManager.getLiveServersFromWALDir());<a name="line.953"></a>
-<span class="sourceLineNo">954</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.954"></a>
-<span class="sourceLineNo">955</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.955"></a>
-<span class="sourceLineNo">956</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.956"></a>
-<span class="sourceLineNo">957</span>    this.tableStateManager =<a name="line.957"></a>
-<span class="sourceLineNo">958</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.958"></a>
-<span class="sourceLineNo">959</span>        ?<a name="line.959"></a>
-<span class="sourceLineNo">960</span>        new MirroringTableStateManager(this):<a name="line.960"></a>
-<span class="sourceLineNo">961</span>        new TableStateManager(this);<a name="line.961"></a>
-<span class="sourceLineNo">962</span><a name="line.962"></a>
-<span class="sourceLineNo">963</span>    status.setStatus("Initializing ZK system trackers");<a name="line.963"></a>
-<span class="sourceLineNo">964</span>    initializeZKBasedSystemTrackers();<a name="line.964"></a>
-<span class="sourceLineNo">965</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.965"></a>
-<span class="sourceLineNo">966</span>    try {<a name="line.966"></a>
-<span class="sourceLineNo">967</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.967"></a>
-<span class="sourceLineNo">968</span>    } catch (IOException e) {<a name="line.968"></a>
-<span class="sourceLineNo">969</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.969"></a>
-<span class="sourceLineNo">970</span>          + " from file system", e);<a name="line.970"></a>
-<span class="sourceLineNo">971</span>    }<a name="line.971"></a>
-<span class="sourceLineNo">972</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.972"></a>
-<span class="sourceLineNo">973</span>    this.activeMaster = true;<a name="line.973"></a>
-<span class="sourceLineNo">974</span><a name="line.974"></a>
-<span class="sourceLineNo">975</span>    // This is for backwards compatibility<a name="line.975"></a>
-<span class="sourceLineNo">976</span>    // See HBASE-11393<a name="line.976"></a>
-<span class="sourceLineNo">977</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.977"></a>
-<span class="sourceLineNo">978</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.978"></a>
-<span class="sourceLineNo">979</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.979"></a>
-<span class="sourceLineNo">980</span>    tableCFsUpdater.copyTableCFs();<a name="line.980"></a>
-<span class="sourceLineNo">981</span><a name="line.981"></a>
-<span class="sourceLineNo">982</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.982"></a>
-<span class="sourceLineNo">983</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.983"></a>
-<span class="sourceLineNo">984</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.984"></a>
-<span class="sourceLineNo">985</span>      updateConfigurationForQuotasObserver(conf);<a name="line.985"></a>
-<span class="sourceLineNo">986</span>    }<a name="line.986"></a>
-<span class="sourceLineNo">987</span>    // initialize master side coprocessors before we start handling requests<a name="line.987"></a>
-<span class="sourceLineNo">988</span>    status.setStatus("Initializing master coprocessors");<a name="line.988"></a>
-<span class="sourceLineNo">989</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.989"></a>
-<span class="sourceLineNo">990</span><a name="line.990"></a>
-<span class="sourceLineNo">991</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.991"></a>
-<span class="sourceLineNo">992</span>    InitMetaProcedure initMetaProc = null;<a name="line.992"></a>
-<span class="sourceLineNo">993</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.993"></a>
-<span class="sourceLineNo">994</span>      .isOffline()) {<a name="line.994"></a>
-<span class="sourceLineNo">995</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.995"></a>
-<span class="sourceLineNo">996</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.996"></a>
-<span class="sourceLineNo">997</span>      if (optProc.isPresent()) {<a name="line.997"></a>
-<span class="sourceLineNo">998</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.998"></a>
-<span class="sourceLineNo">999</span>      } else {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span>        initMetaProc = new InitMetaProcedure();<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span>      }<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span>    }<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1006"></a>
+<span class="sourceLineNo">949</span>    // Start RegionServerTracker with listing of servers found with exiting SCPs -- these should<a name="line.949"></a>
+<span class="sourceLineNo">950</span>    // be registered in the deadServers set -- and with the list of servernames out on the<a name="line.950"></a>
+<span class="sourceLineNo">951</span>    // filesystem that COULD BE 'alive' (we'll schedule SCPs for each and let SCP figure it out).<a name="line.951"></a>
+<span class="sourceLineNo">952</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.952"></a>
+<span class="sourceLineNo">953</span>    this.regionServerTracker.start(<a name="line.953"></a>
+<span class="sourceLineNo">954</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.954"></a>
+<span class="sourceLineNo">955</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.955"></a>
+<span class="sourceLineNo">956</span>      walManager.getLiveServersFromWALDir());<a name="line.956"></a>
+<span class="sourceLineNo">957</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.957"></a>
+<span class="sourceLineNo">958</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.958"></a>
+<span class="sourceLineNo">959</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.959"></a>
+<span class="sourceLineNo">960</span>    this.tableStateManager =<a name="line.960"></a>
+<span class="sourceLineNo">961</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.961"></a>
+<span class="sourceLineNo">962</span>        ?<a name="line.962"></a>
+<span class="sourceLineNo">963</span>        new MirroringTableStateManager(this):<a name="line.963"></a>
+<span class="sourceLineNo">964</span>        new TableStateManager(this);<a name="line.964"></a>
+<span class="sourceLineNo">965</span><a name="line.965"></a>
+<span class="sourceLineNo">966</span>    status.setStatus("Initializing ZK system trackers");<a name="line.966"></a>
+<span class="sourceLineNo">967</span>    initializeZKBasedSystemTrackers();<a name="line.967"></a>
+<span class="sourceLineNo">968</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.968"></a>
+<span class="sourceLineNo">969</span>    try {<a name="line.969"></a>
+<span class="sourceLineNo">970</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.970"></a>
+<span class="sourceLineNo">971</span>    } catch (IOException e) {<a name="line.971"></a>
+<span class="sourceLineNo">972</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.972"></a>
+<span class="sourceLineNo">973</span>          + " from file system", e);<a name="line.973"></a>
+<span class="sourceLineNo">974</span>    }<a name="line.974"></a>
+<span class="sourceLineNo">975</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.975"></a>
+<span class="sourceLineNo">976</span>    this.activeMaster = true;<a name="line.976"></a>
+<span class="sourceLineNo">977</span><a name="line.977"></a>
+<span class="sourceLineNo">978</span>    // This is for backwards compatibility<a name="line.978"></a>
+<span class="sourceLineNo">979</span>    // See HBASE-11393<a name="line.979"></a>
+<span class="sourceLineNo">980</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.980"></a>
+<span class="sourceLineNo">981</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.981"></a>
+<span class="sourceLineNo">982</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.982"></a>
+<span class="sourceLineNo">983</span>    tableCFsUpdater.copyTableCFs();<a name="line.983"></a>
+<span class="sourceLineNo">984</span><a name="line.984"></a>
+<span class="sourceLineNo">985</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.985"></a>
+<span class="sourceLineNo">986</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.986"></a>
+<span class="sourceLineNo">987</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.987"></a>
+<span class="sourceLineNo">988</span>      updateConfigurationForQuotasObserver(conf);<a name="line.988"></a>
+<span class="sourceLineNo">989</span>    }<a name="line.989"></a>
+<span class="sourceLineNo">990</span>    // initialize master side coprocessors before we start handling requests<a name="line.990"></a>
+<span class="sourceLineNo">991</span>    status.setStatus("Initializing master coprocessors");<a name="line.991"></a>
+<span class="sourceLineNo">992</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.992"></a>
+<span class="sourceLineNo">993</span><a name="line.993"></a>
+<span class="sourceLineNo">994</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.994"></a>
+<span class="sourceLineNo">995</span>    InitMetaProcedure initMetaProc = null;<a name="line.995"></a>
+<span class="sourceLineNo">996</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.996"></a>
+<span class="sourceLineNo">997</span>      .isOffline()) {<a name="line.997"></a>
+<span class="sourceLineNo">998</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.998"></a>
+<span class="sourceLineNo">999</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.999"></a>
+<span class="sourceLineNo">1000</span>      if (optProc.isPresent()) {<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span>      } else {<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span>        initMetaProc = new InitMetaProcedure();<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span>      }<a name="line.1006"></a>
 <span class="sourceLineNo">1007</span>    }<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span><a name="line.1008"></a>
-<span class="sourceLineNo">1009</span>    // initialize load balancer<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span>    this.balancer.setMasterServices(this);<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span>    this.balancer.initialize();<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span><a name="line.1013"></a>
-<span class="sourceLineNo">1014</span>    // start up all service threads.<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span>    status.setStatus("Initializing master service threads");<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span>    startServiceThreads();<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span>    // wait meta to be initialized after we start procedure executor<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span>    if (initMetaProc != null) {<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span>      initMetaProc.await();<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span>    }<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span>    tableStateManager.start();<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span>    // Wake up this server to check in<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span>    sleeper.skipSleepCycle();<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span>    // Wait for region servers to report in.<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span>    String statusStr = "Wait for region servers to report in";<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span>    status.setStatus(statusStr);<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span>    LOG.info(Objects.toString(status));<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span>    waitForRegionServers(status);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span><a name="line.1032"></a>
-<span class="sourceLineNo">1033</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span>    if (isStopped()) {<a name="line.1034"></a>
-<span class="sourceLineNo">1035</span>      return;<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span>    }<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span><a name="line.1037"></a>
-<span class="sourceLineNo">1038</span>    //Initialize after meta as it scans meta<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span>    if (favoredNodesManager != null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span>      snapshotOfRegionAssignment.initialize();<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span>    }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span><a name="line.1045"></a>
-<span class="sourceLineNo">1046</span>    // Fix up assignment manager status<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span>    status.setStatus("Starting assignment manager");<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span>    this.assignmentManager.joinCluster();<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span><a name="line.1049"></a>
-<span class="sourceLineNo">1050</span>    // set cluster status again after user regions are assigned<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1051"></a>
+<span class="sourceLineNo">1008</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span>    }<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span><a name="line.1011"></a>
+<span class="sourceLineNo">1012</span>    // initialize load balancer<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span>    this.balancer.setMasterServices(this);<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span>    this.balancer.initialize();<a name="line.1015"></a>
+<span class="sourceLineNo">1016</span><a name="line.1016"></a>
+<span class="sourceLineNo">1017</span>    // start up all service threads.<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span>    status.setStatus("Initializing master service threads");<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span>    startServiceThreads();<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span>    // wait meta to be initialized after we start procedure executor<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span>    if (initMetaProc != null) {<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span>      initMetaProc.await();<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span>    }<a name="line.1023"></a>
+<span class="sourceLineNo">1024</span>    tableStateManager.start();<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span>    // Wake up this server to check in<a name="line.1025"></a>
+<span class="sourceLineNo">1026</span>    sleeper.skipSleepCycle();<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span><a name="line.1027"></a>
+<span class="sourceLineNo">1028</span>    // Wait for region servers to report in.<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span>    String statusStr = "Wait for region servers to report in";<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span>    status.setStatus(statusStr);<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span>    LOG.info(Objects.toString(status));<a name="line.1033"></a>
+<span class="sourceLineNo">1034</span>    waitForRegionServers(status);<a name="line.1034"></a>
+<span class="sourceLineNo">1035</span><a name="line.1035"></a>
+<span class="sourceLineNo">1036</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1036"></a>
+<span class="sourceLineNo">1037</span>    if (isStopped()) {<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span>      return;<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span>    }<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span><a name="line.1040"></a>
+<span class="sourceLineNo">1041</span>    //Initialize after meta as it scans meta<a name="line.1041"></a>
+<span class="sourceLineNo">1042</span>    if (favoredNodesManager != null) {<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span>      snapshotOfRegionAssignment.initialize();<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span>    }<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span><a name="line.1048"></a>
+<span class="sourceLineNo">1049</span>    // Fix up assignment manager status<a name="line.1049"></a>
+<span class="sourceLineNo">1050</span>    status.setStatus("Starting assignment manager");<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span>    this.assignmentManager.joinCluster();<a name="line.1051"></a>
 <span class="sourceLineNo">1052</span><a name="line.1052"></a>
-<span class="sourceLineNo">1053</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span>    this.balancerChore = new BalancerChore(this);<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span>    this.serverManager.startChore();<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span>    status.setStatus("Starting cluster schema service");<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span>    initClusterSchemaService();<a name="line.1066"></a>
+<span class="sourceLineNo">1053</span>    // set cluster status again after user regions are assigned<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span><a name="line.1055"></a>
+<span class="sourceLineNo">1056</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1057"></a>
+<span class="sourceLineNo">1058</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span>    this.balancerChore = new BalancerChore(this);<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1065"></a>
+<span class="sourceLineNo">1066</span>    this.serverManager.startChore();<a name="line.1066"></a>
 <span class="sourceLineNo">1067</span><a name="line.1067"></a>
-<span class="sourceLineNo">1068</span>    if (this.cpHost != null) {<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span>      try {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span>        this.cpHost.preMasterInitialization();<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span>      } catch (IOException e) {<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span>      }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span>    }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span><a name="line.1075"></a>
-<span class="sourceLineNo">1076</span>    status.markComplete("Initialization successful");<a name="line.1076"></a>
-<span class="sourceLineNo">1077</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span>    configurationManager.registerObserver(this.balancer);<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1082"></a>
-<span class="sourceLineNo">1083</span>    // Set master as 'initialized'.<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span>    setInitialized(true);<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span>    status.setStatus("Assign meta replicas");<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span>    metaBootstrap.assignMetaReplicas();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span>    status.setStatus("Starting quota manager");<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span>    initQuotaManager();<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span>      // Create the quota snapshot notifier<a name="line.1092"></a>
-<span class="sourceLineNo">1093</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span><a name="line.1098"></a>
-<span class="sourceLineNo">1099</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1100"></a>
-<span class="sourceLineNo">1101</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span>    }<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span><a name="line.1103"></a>
-<span class="sourceLineNo">1104</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span>    // master initialization. See HBASE-5916.<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span><a name="line.1108"></a>
-<span class="sourceLineNo">1109</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span><a name="line.1112"></a>
-<span class="sourceLineNo">1113</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span>    initMobCleaner();<a name="line.1114"></a>
+<span class="sourceLineNo">1068</span>    status.setStatus("Starting cluster schema service");<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span>    initClusterSchemaService();<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span><a name="line.1070"></a>
+<span class="sourceLineNo">1071</span>    if (this.cpHost != null) {<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span>      try {<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span>        this.cpHost.preMasterInitialization();<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span>      } catch (IOException e) {<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1075"></a>
+<span class="sourceLineNo">1076</span>      }<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span>    }<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span><a name="line.1078"></a>
+<span class="sourceLineNo">1079</span>    status.markComplete("Initialization successful");<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1081"></a>
+<span class="sourceLineNo">1082</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span>    configurationManager.registerObserver(this.balancer);<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1085"></a>
+<span class="sourceLineNo">1086</span>    // Set master as 'initialized'.<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span>    setInitialized(true);<a name="line.1087"></a>
+<span class="sourceLineNo">1088</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span>    status.setStatus("Assign meta replicas");<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span>    metaBootstrap.assignMetaReplicas();<a name="line.1091"></a>
+<span class="sourceLineNo">1092</span>    status.setStatus("Starting quota manager");<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span>    initQuotaManager();<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span>      // Create the quota snapshot notifier<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1097"></a>
+<span class="sourceLineNo">1098</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1099"></a>
+<span class="sourceLineNo">1100</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span><a name="line.1101"></a>
+<span class="sourceLineNo">1102</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span>    }<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span><a name="line.1106"></a>
+<span class="sourceLineNo">1107</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1107"></a>
+<span class="sourceLineNo">1108</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span>    // master initialization. See HBASE-5916.<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span><a name="line.1111"></a>
+<span class="sourceLineNo">1112</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1112"></a>
+<span class="sourceLineNo">1113</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1114"></a>
 <span class="sourceLineNo">1115</span><a name="line.1115"></a>
-<span class="sourceLineNo">1116</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1116"></a>
-<span class="sourceLineNo">1117</span>    if (this.cpHost != null) {<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span>      // don't let cp initialization errors kill the master<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span>      try {<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span>        this.cpHost.postStartMaster();<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span>      } catch (IOException ioe) {<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span>      }<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span>    }<a name="line.1124"></a>
-<span class="sourceLineNo">1125</span><a name="line.1125"></a>
-<span class="sourceLineNo">1126</span>    zombieDetector.interrupt();<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span><a name="line.1127"></a>
-<span class="sourceLineNo">1128</span>    /*<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span>     * in activeMasterManager thread, it should be fine.<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span>     */<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span>    long start = System.currentTimeMillis();<a name="line.1132"></a>
-<span class="sourceLineNo">1133</span>    this.balancer.postMasterStartupInitialize();<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span>    if (LOG.isDebugEnabled()) {<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span>    }<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span>  }<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span><a name="line.1139"></a>
-<span class="sourceLineNo">1140</span>  /**<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span>   */<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span>  @VisibleForTesting<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span>    if (!conf.getBoolean(<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1148"></a>
-<span class="sourceLineNo">1149</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span>      return;<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span>    }<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1152"></a>
-<span class="sourceLineNo">1153</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span>    if (length &gt; 0) {<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span>    }<a name="line.1157"></a>
-<span class="sourceLineNo">1158</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1159"></a>
-<span class="sourceLineNo">1160</span>  }<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span><a name="line.1161"></a>
-<span class="sourceLineNo">1162</span>  private void initMobCleaner() {<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1163"></a>
-<span class="sourceLineNo">1164</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span><a name="line.1165"></a>
-<span class="sourceLineNo">1166</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1167"></a>
-<span class="sourceLineNo">1168</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1168"></a>
-<span class="sourceLineNo">1169</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span>    } else {<a name="line.1171"></a>
-<span class="sourceLineNo">1172</span>      LOG<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span>    }<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>  }<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span><a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>  /**<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>   * &lt;p&gt;<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>   * &lt;/p&gt;<a name="line.1181"></a>
+<span class="sourceLineNo">1116</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span>    initMobCleaner();<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span><a name="line.1118"></a>
+<span class="sourceLineNo">1119</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span>    if (this.cpHost != null) {<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span>      // don't let cp initialization errors kill the master<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span>      try {<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span>        this.cpHost.postStartMaster();<a name="line.1123"></a>
+<span class="sourceLineNo">1124</span>      } catch (IOException ioe) {<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span>      }<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span>    }<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span><a name="line.1128"></a>
+<span class="sourceLineNo">1129</span>    zombieDetector.interrupt();<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span><a name="line.1130"></a>
+<span class="sourceLineNo">1131</span>    /*<a name="line.1131"></a>
+<span class="sourceLineNo">1132</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span>     * in activeMasterManager thread, it should be fine.<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span>     */<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span>    long start = System.currentTimeMillis();<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span>    this.balancer.postMasterStartupInitialize();<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span>    if (LOG.isDebugEnabled()) {<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span>    }<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span>  }<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span><a name="line.1142"></a>
+<span class="sourceLineNo">1143</span>  /**<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span>   */<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span>  @VisibleForTesting<a name="line.1147"></a>
+<span class="sourceLineNo">1148</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span>    if (!conf.getBoolean(<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1151"></a>
+<span class="sourceLineNo">1152</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span>      return;<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span>    }<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1155"></a>
+<span class="sourceLineNo">1156</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span>    if (length &gt; 0) {<a name="line.1158"></a>
+<span class="sourceLineNo">1159</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span>    }<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1162"></a>
+<span class="sourceLineNo">1163</span>  }<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span><a name="line.1164"></a>
+<span class="sourceLineNo">1165</span>  private void initMobCleaner() {<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1166"></a>
+<span class="sourceLineNo">1167</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span><a name="line.1168"></a>
+<span class="sourceLineNo">1169</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1170"></a>
+<span class="sourceLineNo">1171</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span>    } else {<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span>      LOG<a name="line.1175"></a>
+<span class="sourceLineNo">1176</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>    }<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>  }<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span><a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>  /**<a name="line.1181"></a>
 <span class="sourceLineNo">1182</span>   * &lt;p&gt;<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>   * Will be overridden in tests.<a name="line.1183"></a>
+<span class="sourceLineNo">1183</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1183"></a>
 <span class="sourceLineNo">1184</span>   * &lt;/p&gt;<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>   */<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>  @VisibleForTesting<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>    return new MasterMetaBootstrap(this);<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>  }<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span><a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>  /**<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>   * &lt;p&gt;<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>   * Create a {@link ServerManager} instance.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>   * &lt;/p&gt;<a name="line.1196"></a>
+<span class="sourceLineNo">1185</span>   * &lt;p&gt;<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>   * Will be overridden in tests.<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>   * &lt;/p&gt;<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>   */<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>  @VisibleForTesting<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span>    return new MasterMetaBootstrap(this);<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>  }<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span><a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>  /**<a name="line.1196"></a>
 <span class="sourceLineNo">1197</span>   * &lt;p&gt;<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>   * Will be overridden in tests.<a name="line.1198"></a>
+<span class="sourceLineNo">1198</span>   * Create a {@link ServerManager} instance.<a name="line.1198"></a>
 <span class="sourceLineNo">1199</span>   * &lt;/p&gt;<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>   */<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>  @VisibleForTesting<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>    // w/ a mocked up ServerManager.<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>    setupClusterConnection();<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    return new ServerManager(master);<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span>  }<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span><a name="line.1208"></a>
-<span class="sourceLineNo">1209</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>      throws IOException, InterruptedException {<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span>    this.serverManager.waitForRegionServers(status);<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>  }<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span><a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>  // Will be overridden in tests<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>  @VisibleForTesting<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    this.clusterSchemaService.startAsync();<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span>    try {<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span>    } catch (TimeoutException toe) {<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>    }<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>  }<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span><a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>  private void initQuotaManager() throws IOException {<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>    quotaManager.start();<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>    this.quotaManager = quotaManager;<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>  }<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span><a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>    return notifier;<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span>  }<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span><a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>  boolean isCatalogJanitorEnabled() {<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>    return catalogJanitorChore != null ?<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>  }<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span><a name="line.1244"></a>
-<span class="sourceLineNo">1245</span>  boolean isCleanerChoreEnabled() {<a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1246"></a>
+<span class="sourceLineNo">1200</span>   * &lt;p&gt;<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>   * Will be overridden in tests.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>   * &lt;/p&gt;<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>   */<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>  @VisibleForTesting<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>    // w/ a mocked up ServerManager.<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span>    setupClusterConnection();<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>    return new ServerManager(master);<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>  }<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span><a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>      throws IOException, InterruptedException {<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>    this.serverManager.waitForRegionServers(status);<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>  }<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span><a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>  // Will be overridden in tests<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span>  @VisibleForTesting<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>    this.clusterSchemaService.startAsync();<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>    try {<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>    } catch (TimeoutException toe) {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>    }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>  }<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span><a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>  private void initQuotaManager() throws IOException {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>    quotaManager.start();<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>    this.quotaManager = quotaManager;<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>  }<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span><a name="line.1236"></a>
+<span class="sourceLineNo">1237</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>    return notifier;<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>  }<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span><a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>  boolean isCatalogJanitorEnabled() {<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span>    return catalogJanitorChore != null ?<a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>  }<a name="line.1246"></a>
 <span class="sourceLineNo">1247</span><a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>    if (hfileCleaner != null) {<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span>    }<a name="line.1250"></a>
-<span class="sourceLineNo">1251</span><a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>    if (logCleaner != null) {<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span>    }<a name="line.1254"></a>
-<span class="sourceLineNo">1255</span><a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>  }<a name="line.1257"></a>
+<span class="sourceLineNo">1248</span>  boolean isCleanerChoreEnabled() {<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1249"></a>
+<span class="sourceLineNo">1250</span><a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>    if (hfileCleaner != null) {<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span>    }<a name="line.1253"></a>
+<span class="sourceLineNo">1254</span><a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>    if (logCleaner != null) {<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span>    }<a name="line.1257"></a>
 <span class="sourceLineNo">1258</span><a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>  @Override<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>  public TableDescriptors getTableDescriptors() {<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>    return this.tableDescriptors;<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>  }<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span><a name="line.1263"></a>
-<span class="sourceLineNo">1264</span>  @Override<a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>  public ServerManager getServerManager() {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    return this.serverManager;<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>  }<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span><a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>  @Override<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>    return this.fileSystemManager;<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span>  }<a name="line.1272"></a>
-<span class="sourceLineNo">1273</span><a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>  @Override<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>  public MasterWalManager getMasterWalManager() {<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>    return this.walManager;<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>  }<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span><a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>  @Override<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>  public TableStateManager getTableStateManager() {<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>    return tableStateManager;<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span>  }<a name="line.1282"></a>
-<span class="sourceLineNo">1283</span><a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>  /*<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>   * then they just die with a logged message.  This should be fine because<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>   *  need to install an unexpected exception handler.<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>   */<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>  private void startServiceThreads() throws IOException{<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span>   // Start the executor service pools<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span><a name="line.1303"></a>
-<span class="sourceLineNo">1304</span>   // We depend on there being only one instance of this executor running<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>   // tables.<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   // AccessController#postCompletedCreateTableAction<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   startProcedureExecutor();<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span><a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>    // Initial cleaner chore<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>    CleanerChore.initChorePool(conf);<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>   // Start log cleaner thread<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>   this.logCleaner =<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>      new LogCleaner(cleanerInterval,<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>         getMasterWalManager().getOldLogDir());<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span><a name="line.1321"></a>
-<span class="sourceLineNo">1322</span>    // start the hfile archive cleaner thread<a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>    params.put(MASTER, this);<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>        .getFileSystem(), archiveDir, params);<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span><a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>      replicationPeerManager);<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1332"></a>
-<span class="sourceLineNo">1333</span><a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>    serviceStarted = true;<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>    if (LOG.isTraceEnabled()) {<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>      LOG.trace("Started service threads");<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>    }<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>  }<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span><a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>  @Override<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>  protected void stopServiceThreads() {<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>    if (masterJettyServer != null) {<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>      LOG.info("Stopping master jetty server");<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>      try {<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>        masterJettyServer.stop();<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>      } catch (Exception e) {<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>      }<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span>    }<a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>    stopChores();<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>    if (this.mobCompactThread != null) {<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>      this.mobCompactThread.close();<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>    }<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>    super.stopServiceThreads();<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>    CleanerChore.shutDownChorePool();<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span><a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>    LOG.debug("Stopping service threads");<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span><a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>    if (this.quotaManager != null) {<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>      this.quotaManager.stop();<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>    }<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span><a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>    if (this.activeMasterManager != null) {<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>      this.activeMasterManager.stop();<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>    }<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span>    if (this.serverManager != null) {<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      this.serverManager.stop();<a name="line.1367"></a>
+<span class="sourceLineNo">1259</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>  }<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span><a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>  @Override<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span>  public TableDescriptors getTableDescriptors() {<a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>    return this.tableDescriptors;<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>  }<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span><a name="line.1266"></a>
+<span class="sourceLineNo">1267</span>  @Override<a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>  public ServerManager getServerManager() {<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>    return this.serverManager;<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>  }<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span><a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>  @Override<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span>    return this.fileSystemManager;<a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>  }<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span><a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>  @Override<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>  public MasterWalManager getMasterWalManager() {<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    return this.walManager;<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>  }<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span><a name="line.1281"></a>
+<span class="sourceLineNo">1282</span>  @Override<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>  public TableStateManager getTableStateManager() {<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>    return tableStateManager;<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>  }<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span><a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>  /*<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>   * then they just die with a logged message.  This should be fine because<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span>   *  need to install an unexpected exception handler.<a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>   */<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>  private void startServiceThreads() throws IOException{<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   // Start the executor service pools<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span><a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   // We depend on there being only one instance of this executor running<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   // tables.<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>   // AccessController#postCompletedCreateTableAction<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>   startProcedureExecutor();<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span><a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>    // Initial cleaner chore<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>    CleanerChore.initChorePool(conf);<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>   // Start log cleaner thread<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>   this.logCleaner =<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>      new LogCleaner(cleanerInterval,<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>         getMasterWalManager().getOldLogDir());<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span><a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>    // start the hfile archive cleaner thread<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>    params.put(MASTER, this);<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>        .getFileSystem(), archiveDir, params);<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1331"></a>
+<span class="sourceLineNo">1332</span><a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>      replicationPeerManager);<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span><a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>    serviceStarted = true;<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>    if (LOG.isTraceEnabled()) {<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>      LOG.trace("Started service threads");<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>    }<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>  }<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span><a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>  @Override<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>  protected void stopServiceThreads() {<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>    if (masterJettyServer != null) {<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>      LOG.info("Stopping master jetty server");<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>      try {<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span>        masterJettyServer.stop();<a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>      } catch (Exception e) {<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>      }<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>    }<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span>    stopChores();<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>    if (this.mobCompactThread != null) {<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>      this.mobCompactThread.close();<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span>    }<a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>    super.stopServiceThreads();<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>    CleanerChore.shutDownChorePool();<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span><a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>    LOG.debug("Stopping service threads");<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span><a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>    if (this.quotaManager != null) {<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>      this.quotaManager.stop();<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>    }<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span><a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>    if (this.activeMasterManager != null) {<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>      this.activeMasterManager.stop();<a name="line.1367"></a>
 <span class="sourceLineNo">1368</span>    }<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>    if (this.assignmentManager != null) {<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>      this.assignmentManager.stop();<a name="line.1370"></a>
+<span class="sourceLineNo">1369</span>    if (this.serverManager != null) {<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      this.serverManager.stop();<a name="line.1370"></a>
 <span class="sourceLineNo">1371</span>    }<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span><a name="line.1372"></a>
-<span class="sourceLineNo">1373</span>    stopProcedureExecutor();<a name="line.1373"></a>
-<span class="sourceLineNo">1374</span><a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>    if (this.walManager != null) {<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>      this.walManager.stop();<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>    }<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>    if (this.fileSystemManager != null) {<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>      this.fileSystemManager.stop();<a name="line.1379"></a>
+<span class="sourceLineNo">1372</span>    if (this.assignmentManager != null) {<a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>      this.assignmentManager.stop();<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>    }<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span><a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>    stopProcedureExecutor();<a name="line

<TRUNCATED>

[18/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
index a990153..d2da8f4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
@@ -593,7 +593,7 @@
 <span class="sourceLineNo">585</span>              abort(error, t);<a name="line.585"></a>
 <span class="sourceLineNo">586</span>            }<a name="line.586"></a>
 <span class="sourceLineNo">587</span>          }<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        }));<a name="line.588"></a>
+<span class="sourceLineNo">588</span>        }), getName() + ":becomeActiveMaster");<a name="line.588"></a>
 <span class="sourceLineNo">589</span>      }<a name="line.589"></a>
 <span class="sourceLineNo">590</span>      // Fall in here even if we have been aborted. Need to run the shutdown services and<a name="line.590"></a>
 <span class="sourceLineNo">591</span>      // the super run call will do this for us.<a name="line.591"></a>
@@ -932,7 +932,7 @@
 <span class="sourceLineNo">924</span>          HBaseFsck.createLockRetryCounterFactory(this.conf).create());<a name="line.924"></a>
 <span class="sourceLineNo">925</span>    }<a name="line.925"></a>
 <span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span>    status.setStatus("Initialze ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
+<span class="sourceLineNo">927</span>    status.setStatus("Initialize ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
 <span class="sourceLineNo">928</span>    this.serverManager = createServerManager(this);<a name="line.928"></a>
 <span class="sourceLineNo">929</span>    createProcedureExecutor();<a name="line.929"></a>
 <span class="sourceLineNo">930</span>    @SuppressWarnings("rawtypes")<a name="line.930"></a>
@@ -954,2890 +954,2893 @@
 <span class="sourceLineNo">946</span>        .collect(Collectors.toList());<a name="line.946"></a>
 <span class="sourceLineNo">947</span>    this.assignmentManager.setupRIT(ritList);<a name="line.947"></a>
 <span class="sourceLineNo">948</span><a name="line.948"></a>
-<span class="sourceLineNo">949</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.949"></a>
-<span class="sourceLineNo">950</span>    this.regionServerTracker.start(<a name="line.950"></a>
-<span class="sourceLineNo">951</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.951"></a>
-<span class="sourceLineNo">952</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.952"></a>
-<span class="sourceLineNo">953</span>      walManager.getLiveServersFromWALDir());<a name="line.953"></a>
-<span class="sourceLineNo">954</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.954"></a>
-<span class="sourceLineNo">955</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.955"></a>
-<span class="sourceLineNo">956</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.956"></a>
-<span class="sourceLineNo">957</span>    this.tableStateManager =<a name="line.957"></a>
-<span class="sourceLineNo">958</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.958"></a>
-<span class="sourceLineNo">959</span>        ?<a name="line.959"></a>
-<span class="sourceLineNo">960</span>        new MirroringTableStateManager(this):<a name="line.960"></a>
-<span class="sourceLineNo">961</span>        new TableStateManager(this);<a name="line.961"></a>
-<span class="sourceLineNo">962</span><a name="line.962"></a>
-<span class="sourceLineNo">963</span>    status.setStatus("Initializing ZK system trackers");<a name="line.963"></a>
-<span class="sourceLineNo">964</span>    initializeZKBasedSystemTrackers();<a name="line.964"></a>
-<span class="sourceLineNo">965</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.965"></a>
-<span class="sourceLineNo">966</span>    try {<a name="line.966"></a>
-<span class="sourceLineNo">967</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.967"></a>
-<span class="sourceLineNo">968</span>    } catch (IOException e) {<a name="line.968"></a>
-<span class="sourceLineNo">969</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.969"></a>
-<span class="sourceLineNo">970</span>          + " from file system", e);<a name="line.970"></a>
-<span class="sourceLineNo">971</span>    }<a name="line.971"></a>
-<span class="sourceLineNo">972</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.972"></a>
-<span class="sourceLineNo">973</span>    this.activeMaster = true;<a name="line.973"></a>
-<span class="sourceLineNo">974</span><a name="line.974"></a>
-<span class="sourceLineNo">975</span>    // This is for backwards compatibility<a name="line.975"></a>
-<span class="sourceLineNo">976</span>    // See HBASE-11393<a name="line.976"></a>
-<span class="sourceLineNo">977</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.977"></a>
-<span class="sourceLineNo">978</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.978"></a>
-<span class="sourceLineNo">979</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.979"></a>
-<span class="sourceLineNo">980</span>    tableCFsUpdater.copyTableCFs();<a name="line.980"></a>
-<span class="sourceLineNo">981</span><a name="line.981"></a>
-<span class="sourceLineNo">982</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.982"></a>
-<span class="sourceLineNo">983</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.983"></a>
-<span class="sourceLineNo">984</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.984"></a>
-<span class="sourceLineNo">985</span>      updateConfigurationForQuotasObserver(conf);<a name="line.985"></a>
-<span class="sourceLineNo">986</span>    }<a name="line.986"></a>
-<span class="sourceLineNo">987</span>    // initialize master side coprocessors before we start handling requests<a name="line.987"></a>
-<span class="sourceLineNo">988</span>    status.setStatus("Initializing master coprocessors");<a name="line.988"></a>
-<span class="sourceLineNo">989</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.989"></a>
-<span class="sourceLineNo">990</span><a name="line.990"></a>
-<span class="sourceLineNo">991</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.991"></a>
-<span class="sourceLineNo">992</span>    InitMetaProcedure initMetaProc = null;<a name="line.992"></a>
-<span class="sourceLineNo">993</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.993"></a>
-<span class="sourceLineNo">994</span>      .isOffline()) {<a name="line.994"></a>
-<span class="sourceLineNo">995</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.995"></a>
-<span class="sourceLineNo">996</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.996"></a>
-<span class="sourceLineNo">997</span>      if (optProc.isPresent()) {<a name="line.997"></a>
-<span class="sourceLineNo">998</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.998"></a>
-<span class="sourceLineNo">999</span>      } else {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span>        initMetaProc = new InitMetaProcedure();<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span>      }<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span>    }<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1006"></a>
+<span class="sourceLineNo">949</span>    // Start RegionServerTracker with listing of servers found with exiting SCPs -- these should<a name="line.949"></a>
+<span class="sourceLineNo">950</span>    // be registered in the deadServers set -- and with the list of servernames out on the<a name="line.950"></a>
+<span class="sourceLineNo">951</span>    // filesystem that COULD BE 'alive' (we'll schedule SCPs for each and let SCP figure it out).<a name="line.951"></a>
+<span class="sourceLineNo">952</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.952"></a>
+<span class="sourceLineNo">953</span>    this.regionServerTracker.start(<a name="line.953"></a>
+<span class="sourceLineNo">954</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.954"></a>
+<span class="sourceLineNo">955</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.955"></a>
+<span class="sourceLineNo">956</span>      walManager.getLiveServersFromWALDir());<a name="line.956"></a>
+<span class="sourceLineNo">957</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.957"></a>
+<span class="sourceLineNo">958</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.958"></a>
+<span class="sourceLineNo">959</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.959"></a>
+<span class="sourceLineNo">960</span>    this.tableStateManager =<a name="line.960"></a>
+<span class="sourceLineNo">961</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.961"></a>
+<span class="sourceLineNo">962</span>        ?<a name="line.962"></a>
+<span class="sourceLineNo">963</span>        new MirroringTableStateManager(this):<a name="line.963"></a>
+<span class="sourceLineNo">964</span>        new TableStateManager(this);<a name="line.964"></a>
+<span class="sourceLineNo">965</span><a name="line.965"></a>
+<span class="sourceLineNo">966</span>    status.setStatus("Initializing ZK system trackers");<a name="line.966"></a>
+<span class="sourceLineNo">967</span>    initializeZKBasedSystemTrackers();<a name="line.967"></a>
+<span class="sourceLineNo">968</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.968"></a>
+<span class="sourceLineNo">969</span>    try {<a name="line.969"></a>
+<span class="sourceLineNo">970</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.970"></a>
+<span class="sourceLineNo">971</span>    } catch (IOException e) {<a name="line.971"></a>
+<span class="sourceLineNo">972</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.972"></a>
+<span class="sourceLineNo">973</span>          + " from file system", e);<a name="line.973"></a>
+<span class="sourceLineNo">974</span>    }<a name="line.974"></a>
+<span class="sourceLineNo">975</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.975"></a>
+<span class="sourceLineNo">976</span>    this.activeMaster = true;<a name="line.976"></a>
+<span class="sourceLineNo">977</span><a name="line.977"></a>
+<span class="sourceLineNo">978</span>    // This is for backwards compatibility<a name="line.978"></a>
+<span class="sourceLineNo">979</span>    // See HBASE-11393<a name="line.979"></a>
+<span class="sourceLineNo">980</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.980"></a>
+<span class="sourceLineNo">981</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.981"></a>
+<span class="sourceLineNo">982</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.982"></a>
+<span class="sourceLineNo">983</span>    tableCFsUpdater.copyTableCFs();<a name="line.983"></a>
+<span class="sourceLineNo">984</span><a name="line.984"></a>
+<span class="sourceLineNo">985</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.985"></a>
+<span class="sourceLineNo">986</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.986"></a>
+<span class="sourceLineNo">987</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.987"></a>
+<span class="sourceLineNo">988</span>      updateConfigurationForQuotasObserver(conf);<a name="line.988"></a>
+<span class="sourceLineNo">989</span>    }<a name="line.989"></a>
+<span class="sourceLineNo">990</span>    // initialize master side coprocessors before we start handling requests<a name="line.990"></a>
+<span class="sourceLineNo">991</span>    status.setStatus("Initializing master coprocessors");<a name="line.991"></a>
+<span class="sourceLineNo">992</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.992"></a>
+<span class="sourceLineNo">993</span><a name="line.993"></a>
+<span class="sourceLineNo">994</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.994"></a>
+<span class="sourceLineNo">995</span>    InitMetaProcedure initMetaProc = null;<a name="line.995"></a>
+<span class="sourceLineNo">996</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.996"></a>
+<span class="sourceLineNo">997</span>      .isOffline()) {<a name="line.997"></a>
+<span class="sourceLineNo">998</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.998"></a>
+<span class="sourceLineNo">999</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.999"></a>
+<span class="sourceLineNo">1000</span>      if (optProc.isPresent()) {<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span>      } else {<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span>        initMetaProc = new InitMetaProcedure();<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span>      }<a name="line.1006"></a>
 <span class="sourceLineNo">1007</span>    }<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span><a name="line.1008"></a>
-<span class="sourceLineNo">1009</span>    // initialize load balancer<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span>    this.balancer.setMasterServices(this);<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span>    this.balancer.initialize();<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span><a name="line.1013"></a>
-<span class="sourceLineNo">1014</span>    // start up all service threads.<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span>    status.setStatus("Initializing master service threads");<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span>    startServiceThreads();<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span>    // wait meta to be initialized after we start procedure executor<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span>    if (initMetaProc != null) {<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span>      initMetaProc.await();<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span>    }<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span>    tableStateManager.start();<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span>    // Wake up this server to check in<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span>    sleeper.skipSleepCycle();<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span>    // Wait for region servers to report in.<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span>    String statusStr = "Wait for region servers to report in";<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span>    status.setStatus(statusStr);<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span>    LOG.info(Objects.toString(status));<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span>    waitForRegionServers(status);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span><a name="line.1032"></a>
-<span class="sourceLineNo">1033</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span>    if (isStopped()) {<a name="line.1034"></a>
-<span class="sourceLineNo">1035</span>      return;<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span>    }<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span><a name="line.1037"></a>
-<span class="sourceLineNo">1038</span>    //Initialize after meta as it scans meta<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span>    if (favoredNodesManager != null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span>      snapshotOfRegionAssignment.initialize();<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span>    }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span><a name="line.1045"></a>
-<span class="sourceLineNo">1046</span>    // Fix up assignment manager status<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span>    status.setStatus("Starting assignment manager");<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span>    this.assignmentManager.joinCluster();<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span><a name="line.1049"></a>
-<span class="sourceLineNo">1050</span>    // set cluster status again after user regions are assigned<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1051"></a>
+<span class="sourceLineNo">1008</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span>    }<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span><a name="line.1011"></a>
+<span class="sourceLineNo">1012</span>    // initialize load balancer<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span>    this.balancer.setMasterServices(this);<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span>    this.balancer.initialize();<a name="line.1015"></a>
+<span class="sourceLineNo">1016</span><a name="line.1016"></a>
+<span class="sourceLineNo">1017</span>    // start up all service threads.<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span>    status.setStatus("Initializing master service threads");<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span>    startServiceThreads();<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span>    // wait meta to be initialized after we start procedure executor<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span>    if (initMetaProc != null) {<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span>      initMetaProc.await();<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span>    }<a name="line.1023"></a>
+<span class="sourceLineNo">1024</span>    tableStateManager.start();<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span>    // Wake up this server to check in<a name="line.1025"></a>
+<span class="sourceLineNo">1026</span>    sleeper.skipSleepCycle();<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span><a name="line.1027"></a>
+<span class="sourceLineNo">1028</span>    // Wait for region servers to report in.<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span>    String statusStr = "Wait for region servers to report in";<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span>    status.setStatus(statusStr);<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span>    LOG.info(Objects.toString(status));<a name="line.1033"></a>
+<span class="sourceLineNo">1034</span>    waitForRegionServers(status);<a name="line.1034"></a>
+<span class="sourceLineNo">1035</span><a name="line.1035"></a>
+<span class="sourceLineNo">1036</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1036"></a>
+<span class="sourceLineNo">1037</span>    if (isStopped()) {<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span>      return;<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span>    }<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span><a name="line.1040"></a>
+<span class="sourceLineNo">1041</span>    //Initialize after meta as it scans meta<a name="line.1041"></a>
+<span class="sourceLineNo">1042</span>    if (favoredNodesManager != null) {<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span>      snapshotOfRegionAssignment.initialize();<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span>    }<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span><a name="line.1048"></a>
+<span class="sourceLineNo">1049</span>    // Fix up assignment manager status<a name="line.1049"></a>
+<span class="sourceLineNo">1050</span>    status.setStatus("Starting assignment manager");<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span>    this.assignmentManager.joinCluster();<a name="line.1051"></a>
 <span class="sourceLineNo">1052</span><a name="line.1052"></a>
-<span class="sourceLineNo">1053</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span>    this.balancerChore = new BalancerChore(this);<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span>    this.serverManager.startChore();<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span>    status.setStatus("Starting cluster schema service");<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span>    initClusterSchemaService();<a name="line.1066"></a>
+<span class="sourceLineNo">1053</span>    // set cluster status again after user regions are assigned<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span><a name="line.1055"></a>
+<span class="sourceLineNo">1056</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1057"></a>
+<span class="sourceLineNo">1058</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span>    this.balancerChore = new BalancerChore(this);<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1065"></a>
+<span class="sourceLineNo">1066</span>    this.serverManager.startChore();<a name="line.1066"></a>
 <span class="sourceLineNo">1067</span><a name="line.1067"></a>
-<span class="sourceLineNo">1068</span>    if (this.cpHost != null) {<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span>      try {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span>        this.cpHost.preMasterInitialization();<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span>      } catch (IOException e) {<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span>      }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span>    }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span><a name="line.1075"></a>
-<span class="sourceLineNo">1076</span>    status.markComplete("Initialization successful");<a name="line.1076"></a>
-<span class="sourceLineNo">1077</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span>    configurationManager.registerObserver(this.balancer);<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1082"></a>
-<span class="sourceLineNo">1083</span>    // Set master as 'initialized'.<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span>    setInitialized(true);<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span>    status.setStatus("Assign meta replicas");<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span>    metaBootstrap.assignMetaReplicas();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span>    status.setStatus("Starting quota manager");<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span>    initQuotaManager();<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span>      // Create the quota snapshot notifier<a name="line.1092"></a>
-<span class="sourceLineNo">1093</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span><a name="line.1098"></a>
-<span class="sourceLineNo">1099</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1100"></a>
-<span class="sourceLineNo">1101</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span>    }<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span><a name="line.1103"></a>
-<span class="sourceLineNo">1104</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span>    // master initialization. See HBASE-5916.<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span><a name="line.1108"></a>
-<span class="sourceLineNo">1109</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span><a name="line.1112"></a>
-<span class="sourceLineNo">1113</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span>    initMobCleaner();<a name="line.1114"></a>
+<span class="sourceLineNo">1068</span>    status.setStatus("Starting cluster schema service");<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span>    initClusterSchemaService();<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span><a name="line.1070"></a>
+<span class="sourceLineNo">1071</span>    if (this.cpHost != null) {<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span>      try {<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span>        this.cpHost.preMasterInitialization();<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span>      } catch (IOException e) {<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1075"></a>
+<span class="sourceLineNo">1076</span>      }<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span>    }<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span><a name="line.1078"></a>
+<span class="sourceLineNo">1079</span>    status.markComplete("Initialization successful");<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1081"></a>
+<span class="sourceLineNo">1082</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span>    configurationManager.registerObserver(this.balancer);<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1085"></a>
+<span class="sourceLineNo">1086</span>    // Set master as 'initialized'.<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span>    setInitialized(true);<a name="line.1087"></a>
+<span class="sourceLineNo">1088</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span>    status.setStatus("Assign meta replicas");<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span>    metaBootstrap.assignMetaReplicas();<a name="line.1091"></a>
+<span class="sourceLineNo">1092</span>    status.setStatus("Starting quota manager");<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span>    initQuotaManager();<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span>      // Create the quota snapshot notifier<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1097"></a>
+<span class="sourceLineNo">1098</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1099"></a>
+<span class="sourceLineNo">1100</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span><a name="line.1101"></a>
+<span class="sourceLineNo">1102</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span>    }<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span><a name="line.1106"></a>
+<span class="sourceLineNo">1107</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1107"></a>
+<span class="sourceLineNo">1108</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span>    // master initialization. See HBASE-5916.<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span><a name="line.1111"></a>
+<span class="sourceLineNo">1112</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1112"></a>
+<span class="sourceLineNo">1113</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1114"></a>
 <span class="sourceLineNo">1115</span><a name="line.1115"></a>
-<span class="sourceLineNo">1116</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1116"></a>
-<span class="sourceLineNo">1117</span>    if (this.cpHost != null) {<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span>      // don't let cp initialization errors kill the master<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span>      try {<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span>        this.cpHost.postStartMaster();<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span>      } catch (IOException ioe) {<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span>      }<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span>    }<a name="line.1124"></a>
-<span class="sourceLineNo">1125</span><a name="line.1125"></a>
-<span class="sourceLineNo">1126</span>    zombieDetector.interrupt();<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span><a name="line.1127"></a>
-<span class="sourceLineNo">1128</span>    /*<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span>     * in activeMasterManager thread, it should be fine.<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span>     */<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span>    long start = System.currentTimeMillis();<a name="line.1132"></a>
-<span class="sourceLineNo">1133</span>    this.balancer.postMasterStartupInitialize();<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span>    if (LOG.isDebugEnabled()) {<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span>    }<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span>  }<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span><a name="line.1139"></a>
-<span class="sourceLineNo">1140</span>  /**<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span>   */<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span>  @VisibleForTesting<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span>    if (!conf.getBoolean(<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1148"></a>
-<span class="sourceLineNo">1149</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span>      return;<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span>    }<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1152"></a>
-<span class="sourceLineNo">1153</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span>    if (length &gt; 0) {<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span>    }<a name="line.1157"></a>
-<span class="sourceLineNo">1158</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1159"></a>
-<span class="sourceLineNo">1160</span>  }<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span><a name="line.1161"></a>
-<span class="sourceLineNo">1162</span>  private void initMobCleaner() {<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1163"></a>
-<span class="sourceLineNo">1164</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span><a name="line.1165"></a>
-<span class="sourceLineNo">1166</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1167"></a>
-<span class="sourceLineNo">1168</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1168"></a>
-<span class="sourceLineNo">1169</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span>    } else {<a name="line.1171"></a>
-<span class="sourceLineNo">1172</span>      LOG<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span>    }<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>  }<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span><a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>  /**<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>   * &lt;p&gt;<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>   * &lt;/p&gt;<a name="line.1181"></a>
+<span class="sourceLineNo">1116</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span>    initMobCleaner();<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span><a name="line.1118"></a>
+<span class="sourceLineNo">1119</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span>    if (this.cpHost != null) {<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span>      // don't let cp initialization errors kill the master<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span>      try {<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span>        this.cpHost.postStartMaster();<a name="line.1123"></a>
+<span class="sourceLineNo">1124</span>      } catch (IOException ioe) {<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span>      }<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span>    }<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span><a name="line.1128"></a>
+<span class="sourceLineNo">1129</span>    zombieDetector.interrupt();<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span><a name="line.1130"></a>
+<span class="sourceLineNo">1131</span>    /*<a name="line.1131"></a>
+<span class="sourceLineNo">1132</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span>     * in activeMasterManager thread, it should be fine.<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span>     */<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span>    long start = System.currentTimeMillis();<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span>    this.balancer.postMasterStartupInitialize();<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span>    if (LOG.isDebugEnabled()) {<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span>    }<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span>  }<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span><a name="line.1142"></a>
+<span class="sourceLineNo">1143</span>  /**<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span>   */<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span>  @VisibleForTesting<a name="line.1147"></a>
+<span class="sourceLineNo">1148</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span>    if (!conf.getBoolean(<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1151"></a>
+<span class="sourceLineNo">1152</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span>      return;<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span>    }<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1155"></a>
+<span class="sourceLineNo">1156</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span>    if (length &gt; 0) {<a name="line.1158"></a>
+<span class="sourceLineNo">1159</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span>    }<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1162"></a>
+<span class="sourceLineNo">1163</span>  }<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span><a name="line.1164"></a>
+<span class="sourceLineNo">1165</span>  private void initMobCleaner() {<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1166"></a>
+<span class="sourceLineNo">1167</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span><a name="line.1168"></a>
+<span class="sourceLineNo">1169</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1170"></a>
+<span class="sourceLineNo">1171</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span>    } else {<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span>      LOG<a name="line.1175"></a>
+<span class="sourceLineNo">1176</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>    }<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>  }<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span><a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>  /**<a name="line.1181"></a>
 <span class="sourceLineNo">1182</span>   * &lt;p&gt;<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>   * Will be overridden in tests.<a name="line.1183"></a>
+<span class="sourceLineNo">1183</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1183"></a>
 <span class="sourceLineNo">1184</span>   * &lt;/p&gt;<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>   */<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>  @VisibleForTesting<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>    return new MasterMetaBootstrap(this);<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>  }<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span><a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>  /**<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>   * &lt;p&gt;<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>   * Create a {@link ServerManager} instance.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>   * &lt;/p&gt;<a name="line.1196"></a>
+<span class="sourceLineNo">1185</span>   * &lt;p&gt;<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>   * Will be overridden in tests.<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>   * &lt;/p&gt;<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>   */<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>  @VisibleForTesting<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span>    return new MasterMetaBootstrap(this);<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>  }<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span><a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>  /**<a name="line.1196"></a>
 <span class="sourceLineNo">1197</span>   * &lt;p&gt;<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>   * Will be overridden in tests.<a name="line.1198"></a>
+<span class="sourceLineNo">1198</span>   * Create a {@link ServerManager} instance.<a name="line.1198"></a>
 <span class="sourceLineNo">1199</span>   * &lt;/p&gt;<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>   */<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>  @VisibleForTesting<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>    // w/ a mocked up ServerManager.<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>    setupClusterConnection();<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    return new ServerManager(master);<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span>  }<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span><a name="line.1208"></a>
-<span class="sourceLineNo">1209</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>      throws IOException, InterruptedException {<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span>    this.serverManager.waitForRegionServers(status);<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>  }<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span><a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>  // Will be overridden in tests<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>  @VisibleForTesting<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    this.clusterSchemaService.startAsync();<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span>    try {<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span>    } catch (TimeoutException toe) {<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>    }<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>  }<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span><a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>  private void initQuotaManager() throws IOException {<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>    quotaManager.start();<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>    this.quotaManager = quotaManager;<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>  }<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span><a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>    return notifier;<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span>  }<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span><a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>  boolean isCatalogJanitorEnabled() {<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>    return catalogJanitorChore != null ?<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>  }<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span><a name="line.1244"></a>
-<span class="sourceLineNo">1245</span>  boolean isCleanerChoreEnabled() {<a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1246"></a>
+<span class="sourceLineNo">1200</span>   * &lt;p&gt;<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>   * Will be overridden in tests.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>   * &lt;/p&gt;<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>   */<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>  @VisibleForTesting<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>    // w/ a mocked up ServerManager.<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span>    setupClusterConnection();<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>    return new ServerManager(master);<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>  }<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span><a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>      throws IOException, InterruptedException {<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>    this.serverManager.waitForRegionServers(status);<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>  }<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span><a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>  // Will be overridden in tests<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span>  @VisibleForTesting<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>    this.clusterSchemaService.startAsync();<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>    try {<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>    } catch (TimeoutException toe) {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>    }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>  }<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span><a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>  private void initQuotaManager() throws IOException {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>    quotaManager.start();<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>    this.quotaManager = quotaManager;<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>  }<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span><a name="line.1236"></a>
+<span class="sourceLineNo">1237</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>    return notifier;<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>  }<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span><a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>  boolean isCatalogJanitorEnabled() {<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span>    return catalogJanitorChore != null ?<a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>  }<a name="line.1246"></a>
 <span class="sourceLineNo">1247</span><a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>    if (hfileCleaner != null) {<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span>    }<a name="line.1250"></a>
-<span class="sourceLineNo">1251</span><a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>    if (logCleaner != null) {<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span>    }<a name="line.1254"></a>
-<span class="sourceLineNo">1255</span><a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>  }<a name="line.1257"></a>
+<span class="sourceLineNo">1248</span>  boolean isCleanerChoreEnabled() {<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1249"></a>
+<span class="sourceLineNo">1250</span><a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>    if (hfileCleaner != null) {<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span>    }<a name="line.1253"></a>
+<span class="sourceLineNo">1254</span><a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>    if (logCleaner != null) {<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span>    }<a name="line.1257"></a>
 <span class="sourceLineNo">1258</span><a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>  @Override<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>  public TableDescriptors getTableDescriptors() {<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>    return this.tableDescriptors;<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>  }<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span><a name="line.1263"></a>
-<span class="sourceLineNo">1264</span>  @Override<a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>  public ServerManager getServerManager() {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    return this.serverManager;<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>  }<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span><a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>  @Override<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>    return this.fileSystemManager;<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span>  }<a name="line.1272"></a>
-<span class="sourceLineNo">1273</span><a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>  @Override<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>  public MasterWalManager getMasterWalManager() {<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>    return this.walManager;<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>  }<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span><a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>  @Override<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>  public TableStateManager getTableStateManager() {<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>    return tableStateManager;<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span>  }<a name="line.1282"></a>
-<span class="sourceLineNo">1283</span><a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>  /*<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>   * then they just die with a logged message.  This should be fine because<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>   *  need to install an unexpected exception handler.<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>   */<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>  private void startServiceThreads() throws IOException{<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span>   // Start the executor service pools<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span><a name="line.1303"></a>
-<span class="sourceLineNo">1304</span>   // We depend on there being only one instance of this executor running<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>   // tables.<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   // AccessController#postCompletedCreateTableAction<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   startProcedureExecutor();<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span><a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>    // Initial cleaner chore<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>    CleanerChore.initChorePool(conf);<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>   // Start log cleaner thread<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>   this.logCleaner =<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>      new LogCleaner(cleanerInterval,<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>         getMasterWalManager().getOldLogDir());<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span><a name="line.1321"></a>
-<span class="sourceLineNo">1322</span>    // start the hfile archive cleaner thread<a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>    params.put(MASTER, this);<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>        .getFileSystem(), archiveDir, params);<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span><a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>      replicationPeerManager);<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1332"></a>
-<span class="sourceLineNo">1333</span><a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>    serviceStarted = true;<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>    if (LOG.isTraceEnabled()) {<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>      LOG.trace("Started service threads");<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>    }<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>  }<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span><a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>  @Override<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>  protected void stopServiceThreads() {<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>    if (masterJettyServer != null) {<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>      LOG.info("Stopping master jetty server");<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>      try {<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>        masterJettyServer.stop();<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>      } catch (Exception e) {<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>      }<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span>    }<a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>    stopChores();<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>    if (this.mobCompactThread != null) {<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>      this.mobCompactThread.close();<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>    }<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>    super.stopServiceThreads();<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>    CleanerChore.shutDownChorePool();<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span><a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>    LOG.debug("Stopping service threads");<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span><a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>    if (this.quotaManager != null) {<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>      this.quotaManager.stop();<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>    }<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span><a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>    if (this.activeMasterManager != null) {<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>      this.activeMasterManager.stop();<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>    }<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span>    if (this.serverManager != null) {<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      this.serverManager.stop();<a name="line.1367"></a>
+<span class="sourceLineNo">1259</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>  }<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span><a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>  @Override<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span>  public TableDescriptors getTableDescriptors() {<a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>    return this.tableDescriptors;<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>  }<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span><a name="line.1266"></a>
+<span class="sourceLineNo">1267</span>  @Override<a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>  public ServerManager getServerManager() {<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>    return this.serverManager;<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>  }<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span><a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>  @Override<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span>    return this.fileSystemManager;<a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>  }<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span><a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>  @Override<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>  public MasterWalManager getMasterWalManager() {<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    return this.walManager;<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>  }<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span><a name="line.1281"></a>
+<span class="sourceLineNo">1282</span>  @Override<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>  public TableStateManager getTableStateManager() {<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>    return tableStateManager;<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>  }<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span><a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>  /*<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>   * then they just die with a logged message.  This should be fine because<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span>   *  need to install an unexpected exception handler.<a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>   */<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>  private void startServiceThreads() throws IOException{<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   // Start the executor service pools<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span><a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   // We depend on there being only one instance of this executor running<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   // tables.<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>   // AccessController#postCompletedCreateTableAction<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>   startProcedureExecutor();<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span><a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>    // Initial cleaner chore<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>    CleanerChore.initChorePool(conf);<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>   // Start log cleaner thread<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>   this.logCleaner =<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>      new LogCleaner(cleanerInterval,<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>         getMasterWalManager().getOldLogDir());<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span><a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>    // start the hfile archive cleaner thread<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>    params.put(MASTER, this);<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>        .getFileSystem(), archiveDir, params);<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1331"></a>
+<span class="sourceLineNo">1332</span><a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>      replicationPeerManager);<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span><a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>    serviceStarted = true;<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>    if (LOG.isTraceEnabled()) {<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>      LOG.trace("Started service threads");<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>    }<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>  }<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span><a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>  @Override<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>  protected void stopServiceThreads() {<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>    if (masterJettyServer != null) {<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>      LOG.info("Stopping master jetty server");<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>      try {<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span>        masterJettyServer.stop();<a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>      } catch (Exception e) {<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>      }<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>    }<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span>    stopChores();<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>    if (this.mobCompactThread != null) {<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>      this.mobCompactThread.close();<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span>    }<a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>    super.stopServiceThreads();<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>    CleanerChore.shutDownChorePool();<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span><a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>    LOG.debug("Stopping service threads");<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span><a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>    if (this.quotaManager != null) {<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>      this.quotaManager.stop();<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>    }<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span><a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>    if (this.activeMasterManager != null) {<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>      this.activeMasterManager.stop();<a name="line.1367"></a>
 <span class="sourceLineNo">1368</span>    }<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>    if (this.assignmentManager != null) {<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>      this.assignmentManager.stop();<a name="line.1370"></a>
+<span class="sourceLineNo">1369</span>    if (this.serverManager != null) {<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      this.serverManager.stop();<a name="line.1370"></a>
 <span class="sourceLineNo">1371</span>    }<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span><a name="line.1372"></a>
-<span class="sourceLineNo">1373</span>    stopProcedureExecutor();<a name="line.1373"></a>
-<span class="sourceLineNo">1374</span><a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>    if (this.walManager != null) {<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>      this.walManager.stop();<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>    }<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>    if (this.fileSystemManager != null) {<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>      this.fileSystemManager.stop();<a name="line.1379"></a>
+<span class="sourceLineNo">1372</span>    if (this.assignmentManager != null) {<a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>      this.assignmentManager.stop();<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>    }<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span><a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>    stopProcedureExecutor();<a name=

<TRUNCATED>

[23/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/c548e10d
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/c548e10d
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/c548e10d

Branch: refs/heads/asf-site
Commit: c548e10d8706ddd813c2655c7bf185c89f815557
Parents: 1158b28
Author: jenkins <bu...@apache.org>
Authored: Fri Sep 7 14:52:32 2018 +0000
Committer: jenkins <bu...@apache.org>
Committed: Fri Sep 7 14:52:32 2018 +0000

----------------------------------------------------------------------
 acid-semantics.html                             |    4 +-
 apache_hbase_reference_guide.pdf                |    4 +-
 book.html                                       |    2 +-
 bulk-loads.html                                 |    4 +-
 checkstyle-aggregate.html                       |  328 +-
 coc.html                                        |    4 +-
 dependencies.html                               |    4 +-
 dependency-convergence.html                     |    4 +-
 dependency-info.html                            |    4 +-
 dependency-management.html                      |    4 +-
 devapidocs/constant-values.html                 |    6 +-
 devapidocs/index-all.html                       |   10 +-
 .../hadoop/hbase/class-use/ServerName.html      |    8 +-
 .../master/HMaster.MasterStoppedException.html  |    4 +-
 .../master/HMaster.TableDescriptorGetter.html   |    4 +-
 .../org/apache/hadoop/hbase/master/HMaster.html |  312 +-
 .../hadoop/hbase/master/MasterWalManager.html   |   20 +-
 .../hbase/master/RegionServerTracker.html       |    8 +-
 .../hadoop/hbase/master/ServerManager.html      |   14 +-
 .../SplitLogManager.ResubmitDirective.html      |   10 +-
 .../hbase/master/SplitLogManager.Task.html      |   32 +-
 .../hbase/master/SplitLogManager.TaskBatch.html |   14 +-
 .../SplitLogManager.TerminationStatus.html      |   18 +-
 .../master/SplitLogManager.TimeoutMonitor.html  |    8 +-
 .../hadoop/hbase/master/SplitLogManager.html    |   16 +-
 .../store/wal/ProcedureWALFormat.html           |   22 +-
 .../store/wal/WALProcedureStore.PushType.html   |    8 +-
 .../procedure2/store/wal/WALProcedureStore.html |   88 +-
 .../HRegion.FlushResult.Result.html             |    4 +-
 .../org/apache/hadoop/hbase/Version.html        |    6 +-
 .../master/HMaster.InitializationMonitor.html   | 5693 +++++++++---------
 .../master/HMaster.MasterStoppedException.html  | 5693 +++++++++---------
 .../hbase/master/HMaster.RedirectServlet.html   | 5693 +++++++++---------
 .../master/HMaster.TableDescriptorGetter.html   | 5693 +++++++++---------
 .../org/apache/hadoop/hbase/master/HMaster.html | 5693 +++++++++---------
 .../hadoop/hbase/master/MasterWalManager.html   |  371 +-
 .../hbase/master/RegionServerTracker.html       |  136 +-
 .../ServerManager.FlushedSequenceIdFlusher.html |    6 +-
 .../hadoop/hbase/master/ServerManager.html      |    6 +-
 .../SplitLogManager.ResubmitDirective.html      |  824 +--
 .../hbase/master/SplitLogManager.Task.html      |  824 +--
 .../hbase/master/SplitLogManager.TaskBatch.html |  824 +--
 .../SplitLogManager.TerminationStatus.html      |  824 +--
 .../master/SplitLogManager.TimeoutMonitor.html  |  824 +--
 .../hadoop/hbase/master/SplitLogManager.html    |  824 +--
 ...cedureWALFormat.InvalidWALDataException.html |  343 +-
 .../store/wal/ProcedureWALFormat.Loader.html    |  343 +-
 .../store/wal/ProcedureWALFormat.html           |  343 +-
 .../wal/WALProcedureStore.LeaseRecovery.html    | 1757 +++---
 .../store/wal/WALProcedureStore.PushType.html   | 1757 +++---
 .../wal/WALProcedureStore.SyncMetrics.html      | 1757 +++---
 .../procedure2/store/wal/WALProcedureStore.html | 1757 +++---
 downloads.html                                  |    4 +-
 export_control.html                             |    4 +-
 index.html                                      |    4 +-
 integration.html                                |    4 +-
 issue-tracking.html                             |    4 +-
 license.html                                    |    4 +-
 mail-lists.html                                 |    4 +-
 metrics.html                                    |    4 +-
 old_news.html                                   |    4 +-
 plugin-management.html                          |    8 +-
 plugins.html                                    |    8 +-
 poweredbyhbase.html                             |    4 +-
 project-info.html                               |    4 +-
 project-reports.html                            |    4 +-
 project-summary.html                            |    4 +-
 pseudo-distributed.html                         |    4 +-
 replication.html                                |    4 +-
 resources.html                                  |    4 +-
 source-repository.html                          |    4 +-
 sponsors.html                                   |    4 +-
 supportingprojects.html                         |    4 +-
 team-list.html                                  |    4 +-
 74 files changed, 21568 insertions(+), 21487 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/acid-semantics.html
----------------------------------------------------------------------
diff --git a/acid-semantics.html b/acid-semantics.html
index f8ea534..9d774f1 100644
--- a/acid-semantics.html
+++ b/acid-semantics.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013;  
       Apache HBase (TM) ACID Properties
@@ -601,7 +601,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/apache_hbase_reference_guide.pdf
----------------------------------------------------------------------
diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf
index a0a028d..c727958 100644
--- a/apache_hbase_reference_guide.pdf
+++ b/apache_hbase_reference_guide.pdf
@@ -5,8 +5,8 @@
 /Author (Apache HBase Team)
 /Creator (Asciidoctor PDF 1.5.0.alpha.15, based on Prawn 2.2.2)
 /Producer (Apache HBase Team)
-/ModDate (D:20180906143300+00'00')
-/CreationDate (D:20180906144939+00'00')
+/ModDate (D:20180907143259+00'00')
+/CreationDate (D:20180907144903+00'00')
 >>
 endobj
 2 0 obj

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/book.html
----------------------------------------------------------------------
diff --git a/book.html b/book.html
index e416b41..c5c8bc2 100644
--- a/book.html
+++ b/book.html
@@ -41269,7 +41269,7 @@ org/apache/hadoop/hbase/security/access/AccessControlClient.revoke:(Lorg/apache/
 <div id="footer">
 <div id="footer-text">
 Version 3.0.0-SNAPSHOT<br>
-Last updated 2018-09-06 14:33:00 UTC
+Last updated 2018-09-07 14:32:59 UTC
 </div>
 </div>
 </body>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/bulk-loads.html
----------------------------------------------------------------------
diff --git a/bulk-loads.html b/bulk-loads.html
index d55dc11..f547d1e 100644
--- a/bulk-loads.html
+++ b/bulk-loads.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013;  
       Bulk Loads in Apache HBase (TM)
@@ -306,7 +306,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/checkstyle-aggregate.html
----------------------------------------------------------------------
diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html
index 58b9ddc..8d5f6b6 100644
--- a/checkstyle-aggregate.html
+++ b/checkstyle-aggregate.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Checkstyle Results</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -60410,830 +60410,830 @@
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>MethodLength</td>
-<td>Method length is 255 lines (max allowed is 150).</td>
+<td>Method length is 258 lines (max allowed is 150).</td>
 <td>883</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 3, expected level should be 4.</td>
-<td>1293</td></tr>
+<td>1296</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 3, expected level should be 4.</td>
-<td>1295</td></tr>
+<td>1298</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 3, expected level should be 4.</td>
-<td>1297</td></tr>
+<td>1300</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 3, expected level should be 4.</td>
-<td>1299</td></tr>
+<td>1302</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 3, expected level should be 4.</td>
-<td>1301</td></tr>
+<td>1304</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 3, expected level should be 4.</td>
-<td>1309</td></tr>
+<td>1312</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 3, expected level should be 4.</td>
-<td>1310</td></tr>
+<td>1313</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 3, expected level should be 4.</td>
-<td>1315</td></tr>
+<td>1318</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 3, expected level should be 4.</td>
-<td>1316</td></tr>
+<td>1319</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>1524</td></tr>
+<td>1527</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>1546</td></tr>
+<td>1549</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>1563</td></tr>
+<td>1566</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>1598</td></tr>
+<td>1601</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>whitespace</td>
 <td>MethodParamPad</td>
 <td>'(' is preceded with whitespace.</td>
-<td>1742</td></tr>
+<td>1745</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>whitespace</td>
 <td>MethodParamPad</td>
 <td>'(' is preceded with whitespace.</td>
-<td>1746</td></tr>
+<td>1749</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1758</td></tr>
+<td>1761</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1760</td></tr>
+<td>1763</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1762</td></tr>
+<td>1765</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1765</td></tr>
+<td>1768</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1768</td></tr>
+<td>1771</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1769</td></tr>
+<td>1772</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1771</td></tr>
+<td>1774</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1773</td></tr>
+<td>1776</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1774</td></tr>
+<td>1777</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>1775</td></tr>
+<td>1778</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>1781</td></tr>
+<td>1784</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1785</td></tr>
+<td>1788</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1787</td></tr>
+<td>1790</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1788</td></tr>
+<td>1791</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1791</td></tr>
+<td>1794</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1792</td></tr>
+<td>1795</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1794</td></tr>
+<td>1797</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1796</td></tr>
+<td>1799</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1797</td></tr>
+<td>1800</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>1798</td></tr>
+<td>1801</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 116).</td>
-<td>1817</td></tr>
+<td>1820</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 105).</td>
-<td>1819</td></tr>
+<td>1822</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1910</td></tr>
+<td>1913</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1912</td></tr>
+<td>1915</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1914</td></tr>
+<td>1917</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1921</td></tr>
+<td>1924</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1922</td></tr>
+<td>1925</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1924</td></tr>
+<td>1927</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1926</td></tr>
+<td>1929</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1927</td></tr>
+<td>1930</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1929</td></tr>
+<td>1932</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1931</td></tr>
+<td>1934</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1932</td></tr>
+<td>1935</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>1933</td></tr>
+<td>1936</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>1963</td></tr>
+<td>1966</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2213</td></tr>
+<td>2216</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>2214</td></tr>
+<td>2217</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2221</td></tr>
+<td>2224</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>2222</td></tr>
+<td>2225</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2228</td></tr>
+<td>2231</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>2229</td></tr>
+<td>2232</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2236</td></tr>
+<td>2239</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>2237</td></tr>
+<td>2240</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2242</td></tr>
+<td>2245</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2260</td></tr>
+<td>2263</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2262</td></tr>
+<td>2265</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2264</td></tr>
+<td>2267</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2270</td></tr>
+<td>2273</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2271</td></tr>
+<td>2274</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2273</td></tr>
+<td>2276</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2275</td></tr>
+<td>2278</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2276</td></tr>
+<td>2279</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2278</td></tr>
+<td>2281</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2280</td></tr>
+<td>2283</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2281</td></tr>
+<td>2284</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2282</td></tr>
+<td>2285</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2295</td></tr>
+<td>2298</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2297</td></tr>
+<td>2300</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2299</td></tr>
+<td>2302</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2300</td></tr>
+<td>2303</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2301</td></tr>
+<td>2304</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2303</td></tr>
+<td>2306</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2305</td></tr>
+<td>2308</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2306</td></tr>
+<td>2309</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2308</td></tr>
+<td>2311</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2310</td></tr>
+<td>2313</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2311</td></tr>
+<td>2314</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2312</td></tr>
+<td>2315</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2395</td></tr>
+<td>2398</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2397</td></tr>
+<td>2400</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2403</td></tr>
+<td>2406</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2404</td></tr>
+<td>2407</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' has incorrect indentation level 10, expected level should be one of the following: 12, 14.</td>
-<td>2405</td></tr>
+<td>2408</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' child has incorrect indentation level 12, expected level should be one of the following: 14, 16.</td>
-<td>2406</td></tr>
+<td>2409</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' has incorrect indentation level 12, expected level should be one of the following: 14, 16.</td>
-<td>2407</td></tr>
+<td>2410</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' child has incorrect indentation level 14, expected level should be one of the following: 16, 18.</td>
-<td>2408</td></tr>
+<td>2411</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' has incorrect indentation level 14, expected level should be one of the following: 16, 18.</td>
-<td>2409</td></tr>
+<td>2412</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' child has incorrect indentation level 16, expected level should be one of the following: 18, 20.</td>
-<td>2410</td></tr>
+<td>2413</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 14, expected level should be one of the following: 16, 18.</td>
-<td>2412</td></tr>
+<td>2415</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 12, expected level should be one of the following: 14, 16.</td>
-<td>2413</td></tr>
+<td>2416</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 10, expected level should be one of the following: 12, 14.</td>
-<td>2414</td></tr>
+<td>2417</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' child has incorrect indentation level 12, expected level should be one of the following: 14, 16.</td>
-<td>2415</td></tr>
+<td>2418</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 10, expected level should be one of the following: 12, 14.</td>
-<td>2416</td></tr>
+<td>2419</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2417</td></tr>
+<td>2420</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2419</td></tr>
+<td>2422</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2426</td></tr>
+<td>2429</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2427</td></tr>
+<td>2430</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2429</td></tr>
+<td>2432</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2431</td></tr>
+<td>2434</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2432</td></tr>
+<td>2435</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2434</td></tr>
+<td>2437</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2436</td></tr>
+<td>2439</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2437</td></tr>
+<td>2440</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2438</td></tr>
+<td>2441</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2448</td></tr>
+<td>2451</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2450</td></tr>
+<td>2453</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2452</td></tr>
+<td>2455</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2462</td></tr>
+<td>2465</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2463</td></tr>
+<td>2466</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2465</td></tr>
+<td>2468</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2467</td></tr>
+<td>2470</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2468</td></tr>
+<td>2471</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2470</td></tr>
+<td>2473</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2472</td></tr>
+<td>2475</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2473</td></tr>
+<td>2476</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2474</td></tr>
+<td>2477</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2486</td></tr>
+<td>2489</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2571</td></tr>
+<td>2574</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2575</td></tr>
+<td>2578</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2577</td></tr>
+<td>2580</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2579</td></tr>
+<td>2582</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2580</td></tr>
+<td>2583</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2581</td></tr>
+<td>2584</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>3041</td></tr>
+<td>3044</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>3217</td></tr>
+<td>3220</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>3219</td></tr>
+<td>3222</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>3312</td></tr>
+<td>3315</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'for' construct must use '{}'s.</td>
-<td>3342</td></tr>
+<td>3345</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>3349</td></tr>
+<td>3352</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>3354</td></tr>
+<td>3357</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>3383</td></tr>
+<td>3386</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 111).</td>
-<td>3481</td></tr></table></div>
+<td>3484</td></tr></table></div>
 <div class="section">
 <h3 id="org.apache.hadoop.hbase.master.HMasterCommandLine.java">org/apache/hadoop/hbase/master/HMasterCommandLine.java</h3>
 <table border="0" class="table table-striped">
@@ -61962,7 +61962,7 @@
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>187</td></tr></table></div>
+<td>192</td></tr></table></div>
 <div class="section">
 <h3 id="org.apache.hadoop.hbase.master.MetricsAssignmentManager.java">org/apache/hadoop/hbase/master/MetricsAssignmentManager.java</h3>
 <table border="0" class="table table-striped">
@@ -62646,91 +62646,91 @@
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>370</td></tr>
+<td>374</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>371</td></tr>
+<td>375</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'installed' must be private and have accessor methods.</td>
-<td>472</td></tr>
+<td>476</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'done' must be private and have accessor methods.</td>
-<td>473</td></tr>
+<td>477</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'error' must be private and have accessor methods.</td>
-<td>474</td></tr>
+<td>478</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'isDead' must be private and have accessor methods.</td>
-<td>475</td></tr>
+<td>479</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'last_update' must be private and have accessor methods.</td>
-<td>488</td></tr>
+<td>492</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'last_version' must be private and have accessor methods.</td>
-<td>489</td></tr>
+<td>493</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'cur_worker_name' must be private and have accessor methods.</td>
-<td>490</td></tr>
+<td>494</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'batch' must be private and have accessor methods.</td>
-<td>491</td></tr>
+<td>495</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'status' must be private and have accessor methods.</td>
-<td>492</td></tr>
+<td>496</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'incarnation' must be private and have accessor methods.</td>
-<td>493</td></tr>
+<td>497</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'unforcedResubmits' must be private and have accessor methods.</td>
-<td>494</td></tr>
+<td>498</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>design</td>
 <td>VisibilityModifier</td>
 <td>Variable 'resubmitThresholdReached' must be private and have accessor methods.</td>
-<td>495</td></tr>
+<td>499</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>546</td></tr></table></div>
+<td>550</td></tr></table></div>
 <div class="section">
 <h3 id="org.apache.hadoop.hbase.master.SplitOrMergeTracker.java">org/apache/hadoop/hbase/master/SplitOrMergeTracker.java</h3>
 <table border="0" class="table table-striped">
@@ -72444,49 +72444,49 @@
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>465</td></tr>
+<td>472</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>617</td></tr>
+<td>624</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>904</td></tr>
+<td>911</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>994</td></tr>
+<td>1001</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' child has incorrect indentation level 8, expected level should be 6.</td>
-<td>1044</td></tr>
+<td>1051</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>1141</td></tr>
+<td>1148</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>1273</td></tr>
+<td>1280</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>1287</td></tr></table></div>
+<td>1294</td></tr></table></div>
 <div class="section">
 <h3 id="org.apache.hadoop.hbase.procedure2.util.DelayedUtil.java">org/apache/hadoop/hbase/procedure2/util/DelayedUtil.java</h3>
 <table border="0" class="table table-striped">
@@ -118255,7 +118255,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/coc.html
----------------------------------------------------------------------
diff --git a/coc.html b/coc.html
index 99b5cf4..e75a70c 100644
--- a/coc.html
+++ b/coc.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; 
       Code of Conduct Policy
@@ -375,7 +375,7 @@ email to <a class="externalLink" href="mailto:private@hbase.apache.org">the priv
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/dependencies.html
----------------------------------------------------------------------
diff --git a/dependencies.html b/dependencies.html
index 5e5112c..9507d84 100644
--- a/dependencies.html
+++ b/dependencies.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Dependencies</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -440,7 +440,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/dependency-convergence.html
----------------------------------------------------------------------
diff --git a/dependency-convergence.html b/dependency-convergence.html
index 58e23dc..a2e19e9 100644
--- a/dependency-convergence.html
+++ b/dependency-convergence.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Reactor Dependency Convergence</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -879,7 +879,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/dependency-info.html
----------------------------------------------------------------------
diff --git a/dependency-info.html b/dependency-info.html
index 75c3427..68f1e64 100644
--- a/dependency-info.html
+++ b/dependency-info.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Dependency Information</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -313,7 +313,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/dependency-management.html
----------------------------------------------------------------------
diff --git a/dependency-management.html b/dependency-management.html
index 3c09354..3cf304a 100644
--- a/dependency-management.html
+++ b/dependency-management.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180906" />
+    <meta name="Date-Revision-yyyymmdd" content="20180907" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Dependency Management</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -1005,7 +1005,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-09-06</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-09-07</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/constant-values.html
----------------------------------------------------------------------
diff --git a/devapidocs/constant-values.html b/devapidocs/constant-values.html
index 2839122..cc2a714 100644
--- a/devapidocs/constant-values.html
+++ b/devapidocs/constant-values.html
@@ -3824,21 +3824,21 @@
 <!--   -->
 </a><code>public&nbsp;static&nbsp;final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
 <td><code><a href="org/apache/hadoop/hbase/Version.html#date">date</a></code></td>
-<td class="colLast"><code>"Thu Sep  6 14:44:09 UTC 2018"</code></td>
+<td class="colLast"><code>"Fri Sep  7 14:43:58 UTC 2018"</code></td>
 </tr>
 <tr class="rowColor">
 <td class="colFirst"><a name="org.apache.hadoop.hbase.Version.revision">
 <!--   -->
 </a><code>public&nbsp;static&nbsp;final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
 <td><code><a href="org/apache/hadoop/hbase/Version.html#revision">revision</a></code></td>
-<td class="colLast"><code>"f8b12805bbff18fc94e2ac6894f92276b1453350"</code></td>
+<td class="colLast"><code>"24f2893059e745aa6691cdd6cb37fc248880b3b9"</code></td>
 </tr>
 <tr class="altColor">
 <td class="colFirst"><a name="org.apache.hadoop.hbase.Version.srcChecksum">
 <!--   -->
 </a><code>public&nbsp;static&nbsp;final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
 <td><code><a href="org/apache/hadoop/hbase/Version.html#srcChecksum">srcChecksum</a></code></td>
-<td class="colLast"><code>"e6959349aa0db5aaa6ff1d2a92b23943"</code></td>
+<td class="colLast"><code>"7b31fbf46eb4d4d18049c8be8d853a38"</code></td>
 </tr>
 <tr class="rowColor">
 <td class="colFirst"><a name="org.apache.hadoop.hbase.Version.url">

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/index-all.html
----------------------------------------------------------------------
diff --git a/devapidocs/index-all.html b/devapidocs/index-all.html
index e56180d..b5af078 100644
--- a/devapidocs/index-all.html
+++ b/devapidocs/index-all.html
@@ -31964,6 +31964,11 @@
 <dd>
 <div class="block">Find list of coprocessors that extend/implement the given class/interface</div>
 </dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/master/ServerManager.html#findDeadServersAndProcess-java.util.Set-java.util.Set-">findDeadServersAndProcess(Set&lt;ServerName&gt;, Set&lt;ServerName&gt;)</a></span> - Method in class org.apache.hadoop.hbase.master.<a href="org/apache/hadoop/hbase/master/ServerManager.html" title="class in org.apache.hadoop.hbase.master">ServerManager</a></dt>
+<dd>
+<div class="block">Find out the region servers crashed between the crash of the previous master instance and the
+ current master instance and schedule SCP for them.</div>
+</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/io/util/Dictionary.html#findEntry-byte:A-int-int-">findEntry(byte[], int, int)</a></span> - Method in interface org.apache.hadoop.hbase.io.util.<a href="org/apache/hadoop/hbase/io/util/Dictionary.html" title="interface in org.apache.hadoop.hbase.io.util">Dictionary</a></dt>
 <dd>
 <div class="block">Finds the index of an entry.</div>
@@ -32016,11 +32021,6 @@
 </dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.html#findOrCreateOrphanTask-java.lang.String-">findOrCreateOrphanTask(String)</a></span> - Method in class org.apache.hadoop.hbase.coordination.<a href="org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.html" title="class in org.apache.hadoop.hbase.coordination">ZKSplitLogManagerCoordination</a></dt>
 <dd>&nbsp;</dd>
-<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/master/ServerManager.html#findOutDeadServersAndProcess-java.util.Set-java.util.Set-">findOutDeadServersAndProcess(Set&lt;ServerName&gt;, Set&lt;ServerName&gt;)</a></span> - Method in class org.apache.hadoop.hbase.master.<a href="org/apache/hadoop/hbase/master/ServerManager.html" title="class in org.apache.hadoop.hbase.master">ServerManager</a></dt>
-<dd>
-<div class="block">Find out the region servers crashed between the crash of the previous master instance and the
- current master instance and schedule SCP for them.</div>
-</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/http/HttpServer.Builder.html#findPort">findPort</a></span> - Variable in class org.apache.hadoop.hbase.http.<a href="org/apache/hadoop/hbase/http/HttpServer.Builder.html" title="class in org.apache.hadoop.hbase.http">HttpServer.Builder</a></dt>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/http/HttpServer.html#findPort">findPort</a></span> - Variable in class org.apache.hadoop.hbase.http.<a href="org/apache/hadoop/hbase/http/HttpServer.html" title="class in org.apache.hadoop.hbase.http">HttpServer</a></dt>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/class-use/ServerName.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/class-use/ServerName.html b/devapidocs/org/apache/hadoop/hbase/class-use/ServerName.html
index e96a268..5f03721 100644
--- a/devapidocs/org/apache/hadoop/hbase/class-use/ServerName.html
+++ b/devapidocs/org/apache/hadoop/hbase/class-use/ServerName.html
@@ -3868,16 +3868,16 @@ Input/OutputFormats, a table indexing MapReduce job, and utility methods.</div>
 </tr>
 <tr class="altColor">
 <td class="colFirst"><code>(package private) void</code></td>
-<td class="colLast"><span class="typeNameLabel">ServerManager.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html#findOutDeadServersAndProcess-java.util.Set-java.util.Set-">findOutDeadServersAndProcess</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;deadServersFromPE,
-                            <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;liveServersFromWALDir)</code>
+<td class="colLast"><span class="typeNameLabel">ServerManager.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html#findDeadServersAndProcess-java.util.Set-java.util.Set-">findDeadServersAndProcess</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;deadServersFromPE,
+                         <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;liveServersFromWALDir)</code>
 <div class="block">Find out the region servers crashed between the crash of the previous master instance and the
  current master instance and schedule SCP for them.</div>
 </td>
 </tr>
 <tr class="rowColor">
 <td class="colFirst"><code>(package private) void</code></td>
-<td class="colLast"><span class="typeNameLabel">ServerManager.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html#findOutDeadServersAndProcess-java.util.Set-java.util.Set-">findOutDeadServersAndProcess</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;deadServersFromPE,
-                            <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;liveServersFromWALDir)</code>
+<td class="colLast"><span class="typeNameLabel">ServerManager.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.html#findDeadServersAndProcess-java.util.Set-java.util.Set-">findDeadServersAndProcess</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;deadServersFromPE,
+                         <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;liveServersFromWALDir)</code>
 <div class="block">Find out the region servers crashed between the crash of the previous master instance and the
  current master instance and schedule SCP for them.</div>
 </td>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html b/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
index dde8752..8a92624 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
@@ -136,7 +136,7 @@
 </dl>
 <hr>
 <br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2919">HMaster.MasterStoppedException</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2922">HMaster.MasterStoppedException</a>
 extends <a href="../../../../../org/apache/hadoop/hbase/DoNotRetryIOException.html" title="class in org.apache.hadoop.hbase">DoNotRetryIOException</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
@@ -205,7 +205,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/DoNotRetryIOException.ht
 <ul class="blockListLast">
 <li class="blockList">
 <h4>MasterStoppedException</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html#line.2920">MasterStoppedException</a>()</pre>
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html#line.2923">MasterStoppedException</a>()</pre>
 </li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html b/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
index 1aede1a..6b6e5c6 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
@@ -105,7 +105,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>protected static interface <a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2339">HMaster.TableDescriptorGetter</a></pre>
+<pre>protected static interface <a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2342">HMaster.TableDescriptorGetter</a></pre>
 <div class="block">Implement to return TableDescriptor after pre-checks</div>
 </li>
 </ul>
@@ -150,7 +150,7 @@ var activeTableTab = "activeTableTab";
 <ul class="blockListLast">
 <li class="blockList">
 <h4>get</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html#line.2340">get</a>()
+<pre><a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html#line.2343">get</a>()
              throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>


[19/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html
index a990153..d2da8f4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html
@@ -593,7 +593,7 @@
 <span class="sourceLineNo">585</span>              abort(error, t);<a name="line.585"></a>
 <span class="sourceLineNo">586</span>            }<a name="line.586"></a>
 <span class="sourceLineNo">587</span>          }<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        }));<a name="line.588"></a>
+<span class="sourceLineNo">588</span>        }), getName() + ":becomeActiveMaster");<a name="line.588"></a>
 <span class="sourceLineNo">589</span>      }<a name="line.589"></a>
 <span class="sourceLineNo">590</span>      // Fall in here even if we have been aborted. Need to run the shutdown services and<a name="line.590"></a>
 <span class="sourceLineNo">591</span>      // the super run call will do this for us.<a name="line.591"></a>
@@ -932,7 +932,7 @@
 <span class="sourceLineNo">924</span>          HBaseFsck.createLockRetryCounterFactory(this.conf).create());<a name="line.924"></a>
 <span class="sourceLineNo">925</span>    }<a name="line.925"></a>
 <span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span>    status.setStatus("Initialze ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
+<span class="sourceLineNo">927</span>    status.setStatus("Initialize ServerManager and schedule SCP for crash servers");<a name="line.927"></a>
 <span class="sourceLineNo">928</span>    this.serverManager = createServerManager(this);<a name="line.928"></a>
 <span class="sourceLineNo">929</span>    createProcedureExecutor();<a name="line.929"></a>
 <span class="sourceLineNo">930</span>    @SuppressWarnings("rawtypes")<a name="line.930"></a>
@@ -954,2890 +954,2893 @@
 <span class="sourceLineNo">946</span>        .collect(Collectors.toList());<a name="line.946"></a>
 <span class="sourceLineNo">947</span>    this.assignmentManager.setupRIT(ritList);<a name="line.947"></a>
 <span class="sourceLineNo">948</span><a name="line.948"></a>
-<span class="sourceLineNo">949</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.949"></a>
-<span class="sourceLineNo">950</span>    this.regionServerTracker.start(<a name="line.950"></a>
-<span class="sourceLineNo">951</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.951"></a>
-<span class="sourceLineNo">952</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.952"></a>
-<span class="sourceLineNo">953</span>      walManager.getLiveServersFromWALDir());<a name="line.953"></a>
-<span class="sourceLineNo">954</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.954"></a>
-<span class="sourceLineNo">955</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.955"></a>
-<span class="sourceLineNo">956</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.956"></a>
-<span class="sourceLineNo">957</span>    this.tableStateManager =<a name="line.957"></a>
-<span class="sourceLineNo">958</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.958"></a>
-<span class="sourceLineNo">959</span>        ?<a name="line.959"></a>
-<span class="sourceLineNo">960</span>        new MirroringTableStateManager(this):<a name="line.960"></a>
-<span class="sourceLineNo">961</span>        new TableStateManager(this);<a name="line.961"></a>
-<span class="sourceLineNo">962</span><a name="line.962"></a>
-<span class="sourceLineNo">963</span>    status.setStatus("Initializing ZK system trackers");<a name="line.963"></a>
-<span class="sourceLineNo">964</span>    initializeZKBasedSystemTrackers();<a name="line.964"></a>
-<span class="sourceLineNo">965</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.965"></a>
-<span class="sourceLineNo">966</span>    try {<a name="line.966"></a>
-<span class="sourceLineNo">967</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.967"></a>
-<span class="sourceLineNo">968</span>    } catch (IOException e) {<a name="line.968"></a>
-<span class="sourceLineNo">969</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.969"></a>
-<span class="sourceLineNo">970</span>          + " from file system", e);<a name="line.970"></a>
-<span class="sourceLineNo">971</span>    }<a name="line.971"></a>
-<span class="sourceLineNo">972</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.972"></a>
-<span class="sourceLineNo">973</span>    this.activeMaster = true;<a name="line.973"></a>
-<span class="sourceLineNo">974</span><a name="line.974"></a>
-<span class="sourceLineNo">975</span>    // This is for backwards compatibility<a name="line.975"></a>
-<span class="sourceLineNo">976</span>    // See HBASE-11393<a name="line.976"></a>
-<span class="sourceLineNo">977</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.977"></a>
-<span class="sourceLineNo">978</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.978"></a>
-<span class="sourceLineNo">979</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.979"></a>
-<span class="sourceLineNo">980</span>    tableCFsUpdater.copyTableCFs();<a name="line.980"></a>
-<span class="sourceLineNo">981</span><a name="line.981"></a>
-<span class="sourceLineNo">982</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.982"></a>
-<span class="sourceLineNo">983</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.983"></a>
-<span class="sourceLineNo">984</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.984"></a>
-<span class="sourceLineNo">985</span>      updateConfigurationForQuotasObserver(conf);<a name="line.985"></a>
-<span class="sourceLineNo">986</span>    }<a name="line.986"></a>
-<span class="sourceLineNo">987</span>    // initialize master side coprocessors before we start handling requests<a name="line.987"></a>
-<span class="sourceLineNo">988</span>    status.setStatus("Initializing master coprocessors");<a name="line.988"></a>
-<span class="sourceLineNo">989</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.989"></a>
-<span class="sourceLineNo">990</span><a name="line.990"></a>
-<span class="sourceLineNo">991</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.991"></a>
-<span class="sourceLineNo">992</span>    InitMetaProcedure initMetaProc = null;<a name="line.992"></a>
-<span class="sourceLineNo">993</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.993"></a>
-<span class="sourceLineNo">994</span>      .isOffline()) {<a name="line.994"></a>
-<span class="sourceLineNo">995</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.995"></a>
-<span class="sourceLineNo">996</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.996"></a>
-<span class="sourceLineNo">997</span>      if (optProc.isPresent()) {<a name="line.997"></a>
-<span class="sourceLineNo">998</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.998"></a>
-<span class="sourceLineNo">999</span>      } else {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span>        initMetaProc = new InitMetaProcedure();<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span>      }<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span>    }<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1006"></a>
+<span class="sourceLineNo">949</span>    // Start RegionServerTracker with listing of servers found with exiting SCPs -- these should<a name="line.949"></a>
+<span class="sourceLineNo">950</span>    // be registered in the deadServers set -- and with the list of servernames out on the<a name="line.950"></a>
+<span class="sourceLineNo">951</span>    // filesystem that COULD BE 'alive' (we'll schedule SCPs for each and let SCP figure it out).<a name="line.951"></a>
+<span class="sourceLineNo">952</span>    this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);<a name="line.952"></a>
+<span class="sourceLineNo">953</span>    this.regionServerTracker.start(<a name="line.953"></a>
+<span class="sourceLineNo">954</span>      procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()<a name="line.954"></a>
+<span class="sourceLineNo">955</span>        .map(p -&gt; (ServerCrashProcedure) p).map(p -&gt; p.getServerName()).collect(Collectors.toSet()),<a name="line.955"></a>
+<span class="sourceLineNo">956</span>      walManager.getLiveServersFromWALDir());<a name="line.956"></a>
+<span class="sourceLineNo">957</span>    // This manager will be started AFTER hbase:meta is confirmed on line.<a name="line.957"></a>
+<span class="sourceLineNo">958</span>    // hbase.mirror.table.state.to.zookeeper is so hbase1 clients can connect. They read table<a name="line.958"></a>
+<span class="sourceLineNo">959</span>    // state from zookeeper while hbase2 reads it from hbase:meta. Disable if no hbase1 clients.<a name="line.959"></a>
+<span class="sourceLineNo">960</span>    this.tableStateManager =<a name="line.960"></a>
+<span class="sourceLineNo">961</span>      this.conf.getBoolean(MirroringTableStateManager.MIRROR_TABLE_STATE_TO_ZK_KEY, true)<a name="line.961"></a>
+<span class="sourceLineNo">962</span>        ?<a name="line.962"></a>
+<span class="sourceLineNo">963</span>        new MirroringTableStateManager(this):<a name="line.963"></a>
+<span class="sourceLineNo">964</span>        new TableStateManager(this);<a name="line.964"></a>
+<span class="sourceLineNo">965</span><a name="line.965"></a>
+<span class="sourceLineNo">966</span>    status.setStatus("Initializing ZK system trackers");<a name="line.966"></a>
+<span class="sourceLineNo">967</span>    initializeZKBasedSystemTrackers();<a name="line.967"></a>
+<span class="sourceLineNo">968</span>    status.setStatus("Loading last flushed sequence id of regions");<a name="line.968"></a>
+<span class="sourceLineNo">969</span>    try {<a name="line.969"></a>
+<span class="sourceLineNo">970</span>      this.serverManager.loadLastFlushedSequenceIds();<a name="line.970"></a>
+<span class="sourceLineNo">971</span>    } catch (IOException e) {<a name="line.971"></a>
+<span class="sourceLineNo">972</span>      LOG.debug("Failed to load last flushed sequence id of regions"<a name="line.972"></a>
+<span class="sourceLineNo">973</span>          + " from file system", e);<a name="line.973"></a>
+<span class="sourceLineNo">974</span>    }<a name="line.974"></a>
+<span class="sourceLineNo">975</span>    // Set ourselves as active Master now our claim has succeeded up in zk.<a name="line.975"></a>
+<span class="sourceLineNo">976</span>    this.activeMaster = true;<a name="line.976"></a>
+<span class="sourceLineNo">977</span><a name="line.977"></a>
+<span class="sourceLineNo">978</span>    // This is for backwards compatibility<a name="line.978"></a>
+<span class="sourceLineNo">979</span>    // See HBASE-11393<a name="line.979"></a>
+<span class="sourceLineNo">980</span>    status.setStatus("Update TableCFs node in ZNode");<a name="line.980"></a>
+<span class="sourceLineNo">981</span>    ReplicationPeerConfigUpgrader tableCFsUpdater =<a name="line.981"></a>
+<span class="sourceLineNo">982</span>        new ReplicationPeerConfigUpgrader(zooKeeper, conf);<a name="line.982"></a>
+<span class="sourceLineNo">983</span>    tableCFsUpdater.copyTableCFs();<a name="line.983"></a>
+<span class="sourceLineNo">984</span><a name="line.984"></a>
+<span class="sourceLineNo">985</span>    // Add the Observer to delete quotas on table deletion before starting all CPs by<a name="line.985"></a>
+<span class="sourceLineNo">986</span>    // default with quota support, avoiding if user specifically asks to not load this Observer.<a name="line.986"></a>
+<span class="sourceLineNo">987</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.987"></a>
+<span class="sourceLineNo">988</span>      updateConfigurationForQuotasObserver(conf);<a name="line.988"></a>
+<span class="sourceLineNo">989</span>    }<a name="line.989"></a>
+<span class="sourceLineNo">990</span>    // initialize master side coprocessors before we start handling requests<a name="line.990"></a>
+<span class="sourceLineNo">991</span>    status.setStatus("Initializing master coprocessors");<a name="line.991"></a>
+<span class="sourceLineNo">992</span>    this.cpHost = new MasterCoprocessorHost(this, this.conf);<a name="line.992"></a>
+<span class="sourceLineNo">993</span><a name="line.993"></a>
+<span class="sourceLineNo">994</span>    status.setStatus("Initializing meta table if this is a new deploy");<a name="line.994"></a>
+<span class="sourceLineNo">995</span>    InitMetaProcedure initMetaProc = null;<a name="line.995"></a>
+<span class="sourceLineNo">996</span>    if (assignmentManager.getRegionStates().getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO)<a name="line.996"></a>
+<span class="sourceLineNo">997</span>      .isOffline()) {<a name="line.997"></a>
+<span class="sourceLineNo">998</span>      Optional&lt;Procedure&lt;MasterProcedureEnv&gt;&gt; optProc = procedureExecutor.getProcedures().stream()<a name="line.998"></a>
+<span class="sourceLineNo">999</span>        .filter(p -&gt; p instanceof InitMetaProcedure).findAny();<a name="line.999"></a>
+<span class="sourceLineNo">1000</span>      if (optProc.isPresent()) {<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span>        initMetaProc = (InitMetaProcedure) optProc.get();<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span>      } else {<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span>        // schedule an init meta procedure if meta has not been deployed yet<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span>        initMetaProc = new InitMetaProcedure();<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span>        procedureExecutor.submitProcedure(initMetaProc);<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span>      }<a name="line.1006"></a>
 <span class="sourceLineNo">1007</span>    }<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span><a name="line.1008"></a>
-<span class="sourceLineNo">1009</span>    // initialize load balancer<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span>    this.balancer.setMasterServices(this);<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span>    this.balancer.initialize();<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span><a name="line.1013"></a>
-<span class="sourceLineNo">1014</span>    // start up all service threads.<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span>    status.setStatus("Initializing master service threads");<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span>    startServiceThreads();<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span>    // wait meta to be initialized after we start procedure executor<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span>    if (initMetaProc != null) {<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span>      initMetaProc.await();<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span>    }<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span>    tableStateManager.start();<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span>    // Wake up this server to check in<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span>    sleeper.skipSleepCycle();<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span>    // Wait for region servers to report in.<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span>    String statusStr = "Wait for region servers to report in";<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span>    status.setStatus(statusStr);<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span>    LOG.info(Objects.toString(status));<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span>    waitForRegionServers(status);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span><a name="line.1032"></a>
-<span class="sourceLineNo">1033</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span>    if (isStopped()) {<a name="line.1034"></a>
-<span class="sourceLineNo">1035</span>      return;<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span>    }<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span><a name="line.1037"></a>
-<span class="sourceLineNo">1038</span>    //Initialize after meta as it scans meta<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span>    if (favoredNodesManager != null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span>      snapshotOfRegionAssignment.initialize();<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span>    }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span><a name="line.1045"></a>
-<span class="sourceLineNo">1046</span>    // Fix up assignment manager status<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span>    status.setStatus("Starting assignment manager");<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span>    this.assignmentManager.joinCluster();<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span><a name="line.1049"></a>
-<span class="sourceLineNo">1050</span>    // set cluster status again after user regions are assigned<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1051"></a>
+<span class="sourceLineNo">1008</span>    if (this.balancer instanceof FavoredNodesPromoter) {<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span>      favoredNodesManager = new FavoredNodesManager(this);<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span>    }<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span><a name="line.1011"></a>
+<span class="sourceLineNo">1012</span>    // initialize load balancer<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span>    this.balancer.setMasterServices(this);<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span>    this.balancer.initialize();<a name="line.1015"></a>
+<span class="sourceLineNo">1016</span><a name="line.1016"></a>
+<span class="sourceLineNo">1017</span>    // start up all service threads.<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span>    status.setStatus("Initializing master service threads");<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span>    startServiceThreads();<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span>    // wait meta to be initialized after we start procedure executor<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span>    if (initMetaProc != null) {<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span>      initMetaProc.await();<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span>    }<a name="line.1023"></a>
+<span class="sourceLineNo">1024</span>    tableStateManager.start();<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span>    // Wake up this server to check in<a name="line.1025"></a>
+<span class="sourceLineNo">1026</span>    sleeper.skipSleepCycle();<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span><a name="line.1027"></a>
+<span class="sourceLineNo">1028</span>    // Wait for region servers to report in.<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span>    // With this as part of master initialization, it precludes our being able to start a single<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span>    // server that is both Master and RegionServer. Needs more thought. TODO.<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span>    String statusStr = "Wait for region servers to report in";<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span>    status.setStatus(statusStr);<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span>    LOG.info(Objects.toString(status));<a name="line.1033"></a>
+<span class="sourceLineNo">1034</span>    waitForRegionServers(status);<a name="line.1034"></a>
+<span class="sourceLineNo">1035</span><a name="line.1035"></a>
+<span class="sourceLineNo">1036</span>    // Check if master is shutting down because issue initializing regionservers or balancer.<a name="line.1036"></a>
+<span class="sourceLineNo">1037</span>    if (isStopped()) {<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span>      return;<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span>    }<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span><a name="line.1040"></a>
+<span class="sourceLineNo">1041</span>    //Initialize after meta as it scans meta<a name="line.1041"></a>
+<span class="sourceLineNo">1042</span>    if (favoredNodesManager != null) {<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span>      SnapshotOfRegionAssignmentFromMeta snapshotOfRegionAssignment =<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span>          new SnapshotOfRegionAssignmentFromMeta(getConnection());<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span>      snapshotOfRegionAssignment.initialize();<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span>      favoredNodesManager.initialize(snapshotOfRegionAssignment);<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span>    }<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span><a name="line.1048"></a>
+<span class="sourceLineNo">1049</span>    // Fix up assignment manager status<a name="line.1049"></a>
+<span class="sourceLineNo">1050</span>    status.setStatus("Starting assignment manager");<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span>    this.assignmentManager.joinCluster();<a name="line.1051"></a>
 <span class="sourceLineNo">1052</span><a name="line.1052"></a>
-<span class="sourceLineNo">1053</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span>    this.balancerChore = new BalancerChore(this);<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span>    this.serverManager.startChore();<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span>    status.setStatus("Starting cluster schema service");<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span>    initClusterSchemaService();<a name="line.1066"></a>
+<span class="sourceLineNo">1053</span>    // set cluster status again after user regions are assigned<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span>    this.balancer.setClusterMetrics(getClusterMetricsWithoutCoprocessor());<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span><a name="line.1055"></a>
+<span class="sourceLineNo">1056</span>    // Start balancer and meta catalog janitor after meta and regions have been assigned.<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span>    status.setStatus("Starting balancer and catalog janitor");<a name="line.1057"></a>
+<span class="sourceLineNo">1058</span>    this.clusterStatusChore = new ClusterStatusChore(this, balancer);<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span>    getChoreService().scheduleChore(clusterStatusChore);<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span>    this.balancerChore = new BalancerChore(this);<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span>    getChoreService().scheduleChore(balancerChore);<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span>    this.normalizerChore = new RegionNormalizerChore(this);<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span>    getChoreService().scheduleChore(normalizerChore);<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span>    this.catalogJanitorChore = new CatalogJanitor(this);<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span>    getChoreService().scheduleChore(catalogJanitorChore);<a name="line.1065"></a>
+<span class="sourceLineNo">1066</span>    this.serverManager.startChore();<a name="line.1066"></a>
 <span class="sourceLineNo">1067</span><a name="line.1067"></a>
-<span class="sourceLineNo">1068</span>    if (this.cpHost != null) {<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span>      try {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span>        this.cpHost.preMasterInitialization();<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span>      } catch (IOException e) {<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span>      }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span>    }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span><a name="line.1075"></a>
-<span class="sourceLineNo">1076</span>    status.markComplete("Initialization successful");<a name="line.1076"></a>
-<span class="sourceLineNo">1077</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span>    configurationManager.registerObserver(this.balancer);<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1082"></a>
-<span class="sourceLineNo">1083</span>    // Set master as 'initialized'.<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span>    setInitialized(true);<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span>    status.setStatus("Assign meta replicas");<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span>    metaBootstrap.assignMetaReplicas();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span>    status.setStatus("Starting quota manager");<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span>    initQuotaManager();<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span>      // Create the quota snapshot notifier<a name="line.1092"></a>
-<span class="sourceLineNo">1093</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span><a name="line.1098"></a>
-<span class="sourceLineNo">1099</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1100"></a>
-<span class="sourceLineNo">1101</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span>    }<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span><a name="line.1103"></a>
-<span class="sourceLineNo">1104</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span>    // master initialization. See HBASE-5916.<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span><a name="line.1108"></a>
-<span class="sourceLineNo">1109</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span><a name="line.1112"></a>
-<span class="sourceLineNo">1113</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span>    initMobCleaner();<a name="line.1114"></a>
+<span class="sourceLineNo">1068</span>    status.setStatus("Starting cluster schema service");<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span>    initClusterSchemaService();<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span><a name="line.1070"></a>
+<span class="sourceLineNo">1071</span>    if (this.cpHost != null) {<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span>      try {<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span>        this.cpHost.preMasterInitialization();<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span>      } catch (IOException e) {<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span>        LOG.error("Coprocessor preMasterInitialization() hook failed", e);<a name="line.1075"></a>
+<span class="sourceLineNo">1076</span>      }<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span>    }<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span><a name="line.1078"></a>
+<span class="sourceLineNo">1079</span>    status.markComplete("Initialization successful");<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span>    LOG.info(String.format("Master has completed initialization %.3fsec",<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span>       (System.currentTimeMillis() - masterActiveTime) / 1000.0f));<a name="line.1081"></a>
+<span class="sourceLineNo">1082</span>    this.masterFinishedInitializationTime = System.currentTimeMillis();<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span>    configurationManager.registerObserver(this.balancer);<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span>    configurationManager.registerObserver(this.hfileCleaner);<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span>    configurationManager.registerObserver(this.logCleaner);<a name="line.1085"></a>
+<span class="sourceLineNo">1086</span>    // Set master as 'initialized'.<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span>    setInitialized(true);<a name="line.1087"></a>
+<span class="sourceLineNo">1088</span>    assignmentManager.checkIfShouldMoveSystemRegionAsync();<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span>    status.setStatus("Assign meta replicas");<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span>    MasterMetaBootstrap metaBootstrap = createMetaBootstrap();<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span>    metaBootstrap.assignMetaReplicas();<a name="line.1091"></a>
+<span class="sourceLineNo">1092</span>    status.setStatus("Starting quota manager");<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span>    initQuotaManager();<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span>    if (QuotaUtil.isQuotaEnabled(conf)) {<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span>      // Create the quota snapshot notifier<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span>      spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span>      spaceQuotaSnapshotNotifier.initialize(getClusterConnection());<a name="line.1097"></a>
+<span class="sourceLineNo">1098</span>      this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span>      // Start the chore to read the region FS space reports and act on them<a name="line.1099"></a>
+<span class="sourceLineNo">1100</span>      getChoreService().scheduleChore(quotaObserverChore);<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span><a name="line.1101"></a>
+<span class="sourceLineNo">1102</span>      this.snapshotQuotaChore = new SnapshotQuotaObserverChore(this, getMasterMetrics());<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span>      // Start the chore to read snapshots and add their usage to table/NS quotas<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span>      getChoreService().scheduleChore(snapshotQuotaChore);<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span>    }<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span><a name="line.1106"></a>
+<span class="sourceLineNo">1107</span>    // clear the dead servers with same host name and port of online server because we are not<a name="line.1107"></a>
+<span class="sourceLineNo">1108</span>    // removing dead server with same hostname and port of rs which is trying to check in before<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span>    // master initialization. See HBASE-5916.<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span>    this.serverManager.clearDeadServersWithSameHostNameAndPortOfOnlineServer();<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span><a name="line.1111"></a>
+<span class="sourceLineNo">1112</span>    // Check and set the znode ACLs if needed in case we are overtaking a non-secure configuration<a name="line.1112"></a>
+<span class="sourceLineNo">1113</span>    status.setStatus("Checking ZNode ACLs");<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span>    zooKeeper.checkAndSetZNodeAcls();<a name="line.1114"></a>
 <span class="sourceLineNo">1115</span><a name="line.1115"></a>
-<span class="sourceLineNo">1116</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1116"></a>
-<span class="sourceLineNo">1117</span>    if (this.cpHost != null) {<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span>      // don't let cp initialization errors kill the master<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span>      try {<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span>        this.cpHost.postStartMaster();<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span>      } catch (IOException ioe) {<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span>      }<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span>    }<a name="line.1124"></a>
-<span class="sourceLineNo">1125</span><a name="line.1125"></a>
-<span class="sourceLineNo">1126</span>    zombieDetector.interrupt();<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span><a name="line.1127"></a>
-<span class="sourceLineNo">1128</span>    /*<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span>     * in activeMasterManager thread, it should be fine.<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span>     */<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span>    long start = System.currentTimeMillis();<a name="line.1132"></a>
-<span class="sourceLineNo">1133</span>    this.balancer.postMasterStartupInitialize();<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span>    if (LOG.isDebugEnabled()) {<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span>    }<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span>  }<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span><a name="line.1139"></a>
-<span class="sourceLineNo">1140</span>  /**<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span>   */<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span>  @VisibleForTesting<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span>    if (!conf.getBoolean(<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1148"></a>
-<span class="sourceLineNo">1149</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span>      return;<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span>    }<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1152"></a>
-<span class="sourceLineNo">1153</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span>    if (length &gt; 0) {<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span>    }<a name="line.1157"></a>
-<span class="sourceLineNo">1158</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1159"></a>
-<span class="sourceLineNo">1160</span>  }<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span><a name="line.1161"></a>
-<span class="sourceLineNo">1162</span>  private void initMobCleaner() {<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1163"></a>
-<span class="sourceLineNo">1164</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span><a name="line.1165"></a>
-<span class="sourceLineNo">1166</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1167"></a>
-<span class="sourceLineNo">1168</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1168"></a>
-<span class="sourceLineNo">1169</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span>    } else {<a name="line.1171"></a>
-<span class="sourceLineNo">1172</span>      LOG<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span>    }<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>  }<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span><a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>  /**<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>   * &lt;p&gt;<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>   * &lt;/p&gt;<a name="line.1181"></a>
+<span class="sourceLineNo">1116</span>    status.setStatus("Initializing MOB Cleaner");<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span>    initMobCleaner();<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span><a name="line.1118"></a>
+<span class="sourceLineNo">1119</span>    status.setStatus("Calling postStartMaster coprocessors");<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span>    if (this.cpHost != null) {<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span>      // don't let cp initialization errors kill the master<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span>      try {<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span>        this.cpHost.postStartMaster();<a name="line.1123"></a>
+<span class="sourceLineNo">1124</span>      } catch (IOException ioe) {<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span>        LOG.error("Coprocessor postStartMaster() hook failed", ioe);<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span>      }<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span>    }<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span><a name="line.1128"></a>
+<span class="sourceLineNo">1129</span>    zombieDetector.interrupt();<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span><a name="line.1130"></a>
+<span class="sourceLineNo">1131</span>    /*<a name="line.1131"></a>
+<span class="sourceLineNo">1132</span>     * After master has started up, lets do balancer post startup initialization. Since this runs<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span>     * in activeMasterManager thread, it should be fine.<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span>     */<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span>    long start = System.currentTimeMillis();<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span>    this.balancer.postMasterStartupInitialize();<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span>    if (LOG.isDebugEnabled()) {<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span>      LOG.debug("Balancer post startup initialization complete, took " + (<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span>          (System.currentTimeMillis() - start) / 1000) + " seconds");<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span>    }<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span>  }<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span><a name="line.1142"></a>
+<span class="sourceLineNo">1143</span>  /**<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span>   * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span>   * automatically remove quotas for a table when that table is deleted.<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span>   */<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span>  @VisibleForTesting<a name="line.1147"></a>
+<span class="sourceLineNo">1148</span>  public void updateConfigurationForQuotasObserver(Configuration conf) {<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span>    // We're configured to not delete quotas on table deletion, so we don't need to add the obs.<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span>    if (!conf.getBoolean(<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,<a name="line.1151"></a>
+<span class="sourceLineNo">1152</span>          MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span>      return;<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span>    }<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span>    String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);<a name="line.1155"></a>
+<span class="sourceLineNo">1156</span>    final int length = null == masterCoprocs ? 0 : masterCoprocs.length;<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span>    String[] updatedCoprocs = new String[length + 1];<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span>    if (length &gt; 0) {<a name="line.1158"></a>
+<span class="sourceLineNo">1159</span>      System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span>    }<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span>    updatedCoprocs[length] = MasterQuotasObserver.class.getName();<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span>    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);<a name="line.1162"></a>
+<span class="sourceLineNo">1163</span>  }<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span><a name="line.1164"></a>
+<span class="sourceLineNo">1165</span>  private void initMobCleaner() {<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span>    this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);<a name="line.1166"></a>
+<span class="sourceLineNo">1167</span>    getChoreService().scheduleChore(expiredMobFileCleanerChore);<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span><a name="line.1168"></a>
+<span class="sourceLineNo">1169</span>    int mobCompactionPeriod = conf.getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span>        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD);<a name="line.1170"></a>
+<span class="sourceLineNo">1171</span>    if (mobCompactionPeriod &gt; 0) {<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span>      this.mobCompactChore = new MobCompactionChore(this, mobCompactionPeriod);<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span>      getChoreService().scheduleChore(mobCompactChore);<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span>    } else {<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span>      LOG<a name="line.1175"></a>
+<span class="sourceLineNo">1176</span>        .info("The period is " + mobCompactionPeriod + " seconds, MobCompactionChore is disabled");<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>    }<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>    this.mobCompactThread = new MasterMobCompactionThread(this);<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>  }<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span><a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>  /**<a name="line.1181"></a>
 <span class="sourceLineNo">1182</span>   * &lt;p&gt;<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>   * Will be overridden in tests.<a name="line.1183"></a>
+<span class="sourceLineNo">1183</span>   * Create a {@link MasterMetaBootstrap} instance.<a name="line.1183"></a>
 <span class="sourceLineNo">1184</span>   * &lt;/p&gt;<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>   */<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>  @VisibleForTesting<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>    return new MasterMetaBootstrap(this);<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>  }<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span><a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>  /**<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>   * &lt;p&gt;<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>   * Create a {@link ServerManager} instance.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>   * &lt;/p&gt;<a name="line.1196"></a>
+<span class="sourceLineNo">1185</span>   * &lt;p&gt;<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>   * Will be overridden in tests.<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>   * &lt;/p&gt;<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>   */<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>  @VisibleForTesting<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>  protected MasterMetaBootstrap createMetaBootstrap() {<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>    // w/ a mocked up MasterMetaBootstrap.<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span>    return new MasterMetaBootstrap(this);<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>  }<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span><a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>  /**<a name="line.1196"></a>
 <span class="sourceLineNo">1197</span>   * &lt;p&gt;<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>   * Will be overridden in tests.<a name="line.1198"></a>
+<span class="sourceLineNo">1198</span>   * Create a {@link ServerManager} instance.<a name="line.1198"></a>
 <span class="sourceLineNo">1199</span>   * &lt;/p&gt;<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>   */<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>  @VisibleForTesting<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>    // w/ a mocked up ServerManager.<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>    setupClusterConnection();<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    return new ServerManager(master);<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span>  }<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span><a name="line.1208"></a>
-<span class="sourceLineNo">1209</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>      throws IOException, InterruptedException {<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span>    this.serverManager.waitForRegionServers(status);<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>  }<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span><a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>  // Will be overridden in tests<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>  @VisibleForTesting<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    this.clusterSchemaService.startAsync();<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span>    try {<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span>    } catch (TimeoutException toe) {<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>    }<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>  }<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span><a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>  private void initQuotaManager() throws IOException {<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>    quotaManager.start();<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>    this.quotaManager = quotaManager;<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>  }<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span><a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>    return notifier;<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span>  }<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span><a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>  boolean isCatalogJanitorEnabled() {<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>    return catalogJanitorChore != null ?<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>  }<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span><a name="line.1244"></a>
-<span class="sourceLineNo">1245</span>  boolean isCleanerChoreEnabled() {<a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1246"></a>
+<span class="sourceLineNo">1200</span>   * &lt;p&gt;<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>   * Will be overridden in tests.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>   * &lt;/p&gt;<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>   */<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>  @VisibleForTesting<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span>  protected ServerManager createServerManager(final MasterServices master) throws IOException {<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span>    // We put this out here in a method so can do a Mockito.spy and stub it out<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>    // w/ a mocked up ServerManager.<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span>    setupClusterConnection();<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>    return new ServerManager(master);<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>  }<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span><a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>  private void waitForRegionServers(final MonitoredTask status)<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>      throws IOException, InterruptedException {<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>    this.serverManager.waitForRegionServers(status);<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>  }<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span><a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>  // Will be overridden in tests<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span>  @VisibleForTesting<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>  protected void initClusterSchemaService() throws IOException, InterruptedException {<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>    this.clusterSchemaService = new ClusterSchemaServiceImpl(this);<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>    this.clusterSchemaService.startAsync();<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>    try {<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span>      this.clusterSchemaService.awaitRunning(getConfiguration().getInt(<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>        HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS,<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>        DEFAULT_HBASE_MASTER_WAIT_ON_SERVICE_IN_SECONDS), TimeUnit.SECONDS);<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>    } catch (TimeoutException toe) {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span>      throw new IOException("Timedout starting ClusterSchemaService", toe);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>    }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>  }<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span><a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>  private void initQuotaManager() throws IOException {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>    MasterQuotaManager quotaManager = new MasterQuotaManager(this);<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>    quotaManager.start();<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>    this.quotaManager = quotaManager;<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>  }<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span><a name="line.1236"></a>
+<span class="sourceLineNo">1237</span>  private SpaceQuotaSnapshotNotifier createQuotaSnapshotNotifier() {<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>    SpaceQuotaSnapshotNotifier notifier =<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>        SpaceQuotaSnapshotNotifierFactory.getInstance().create(getConfiguration());<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>    return notifier;<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>  }<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span><a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>  boolean isCatalogJanitorEnabled() {<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span>    return catalogJanitorChore != null ?<a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>      catalogJanitorChore.getEnabled() : false;<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>  }<a name="line.1246"></a>
 <span class="sourceLineNo">1247</span><a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>    if (hfileCleaner != null) {<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span>    }<a name="line.1250"></a>
-<span class="sourceLineNo">1251</span><a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>    if (logCleaner != null) {<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span>    }<a name="line.1254"></a>
-<span class="sourceLineNo">1255</span><a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>  }<a name="line.1257"></a>
+<span class="sourceLineNo">1248</span>  boolean isCleanerChoreEnabled() {<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span>    boolean hfileCleanerFlag = true, logCleanerFlag = true;<a name="line.1249"></a>
+<span class="sourceLineNo">1250</span><a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>    if (hfileCleaner != null) {<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>      hfileCleanerFlag = hfileCleaner.getEnabled();<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span>    }<a name="line.1253"></a>
+<span class="sourceLineNo">1254</span><a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>    if (logCleaner != null) {<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>      logCleanerFlag = logCleaner.getEnabled();<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span>    }<a name="line.1257"></a>
 <span class="sourceLineNo">1258</span><a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>  @Override<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>  public TableDescriptors getTableDescriptors() {<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>    return this.tableDescriptors;<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>  }<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span><a name="line.1263"></a>
-<span class="sourceLineNo">1264</span>  @Override<a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>  public ServerManager getServerManager() {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    return this.serverManager;<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>  }<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span><a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>  @Override<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>    return this.fileSystemManager;<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span>  }<a name="line.1272"></a>
-<span class="sourceLineNo">1273</span><a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>  @Override<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>  public MasterWalManager getMasterWalManager() {<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>    return this.walManager;<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>  }<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span><a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>  @Override<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>  public TableStateManager getTableStateManager() {<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>    return tableStateManager;<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span>  }<a name="line.1282"></a>
-<span class="sourceLineNo">1283</span><a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>  /*<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>   * then they just die with a logged message.  This should be fine because<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>   *  need to install an unexpected exception handler.<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>   */<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>  private void startServiceThreads() throws IOException{<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span>   // Start the executor service pools<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span><a name="line.1303"></a>
-<span class="sourceLineNo">1304</span>   // We depend on there being only one instance of this executor running<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>   // tables.<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   // AccessController#postCompletedCreateTableAction<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   startProcedureExecutor();<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span><a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>    // Initial cleaner chore<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>    CleanerChore.initChorePool(conf);<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>   // Start log cleaner thread<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>   this.logCleaner =<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>      new LogCleaner(cleanerInterval,<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>         getMasterWalManager().getOldLogDir());<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span><a name="line.1321"></a>
-<span class="sourceLineNo">1322</span>    // start the hfile archive cleaner thread<a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>    params.put(MASTER, this);<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>        .getFileSystem(), archiveDir, params);<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span><a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>      replicationPeerManager);<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1332"></a>
-<span class="sourceLineNo">1333</span><a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>    serviceStarted = true;<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>    if (LOG.isTraceEnabled()) {<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>      LOG.trace("Started service threads");<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>    }<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>  }<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span><a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>  @Override<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>  protected void stopServiceThreads() {<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>    if (masterJettyServer != null) {<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>      LOG.info("Stopping master jetty server");<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>      try {<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>        masterJettyServer.stop();<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>      } catch (Exception e) {<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>      }<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span>    }<a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>    stopChores();<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>    if (this.mobCompactThread != null) {<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>      this.mobCompactThread.close();<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>    }<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>    super.stopServiceThreads();<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>    CleanerChore.shutDownChorePool();<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span><a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>    LOG.debug("Stopping service threads");<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span><a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>    if (this.quotaManager != null) {<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>      this.quotaManager.stop();<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>    }<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span><a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>    if (this.activeMasterManager != null) {<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>      this.activeMasterManager.stop();<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>    }<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span>    if (this.serverManager != null) {<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      this.serverManager.stop();<a name="line.1367"></a>
+<span class="sourceLineNo">1259</span>    return (hfileCleanerFlag &amp;&amp; logCleanerFlag);<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>  }<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span><a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>  @Override<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span>  public TableDescriptors getTableDescriptors() {<a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>    return this.tableDescriptors;<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>  }<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span><a name="line.1266"></a>
+<span class="sourceLineNo">1267</span>  @Override<a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>  public ServerManager getServerManager() {<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>    return this.serverManager;<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>  }<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span><a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>  @Override<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>  public MasterFileSystem getMasterFileSystem() {<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span>    return this.fileSystemManager;<a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>  }<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span><a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>  @Override<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>  public MasterWalManager getMasterWalManager() {<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    return this.walManager;<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>  }<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span><a name="line.1281"></a>
+<span class="sourceLineNo">1282</span>  @Override<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>  public TableStateManager getTableStateManager() {<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>    return tableStateManager;<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>  }<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span><a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>  /*<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>   * Start up all services. If any of these threads gets an unhandled exception<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>   * then they just die with a logged message.  This should be fine because<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>   * in general, we do not expect the master to get such unhandled exceptions<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>   *  as OOMEs; it should be lightly loaded. See what HRegionServer does if<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span>   *  need to install an unexpected exception handler.<a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>   */<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>  private void startServiceThreads() throws IOException{<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   // Start the executor service pools<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION,<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>      conf.getInt("hbase.master.executor.openregion.threads", 5));<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>   this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION,<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>      conf.getInt("hbase.master.executor.closeregion.threads", 5));<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>   this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS,<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>      conf.getInt("hbase.master.executor.serverops.threads", 5));<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>   this.executorService.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS,<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>      conf.getInt("hbase.master.executor.meta.serverops.threads", 5));<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span>   this.executorService.startExecutorService(ExecutorType.M_LOG_REPLAY_OPS,<a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>      conf.getInt("hbase.master.executor.logreplayops.threads", 10));<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span><a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   // We depend on there being only one instance of this executor running<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   // at a time.  To do concurrency, would need fencing of enable/disable of<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   // tables.<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>   // Any time changing this maxThreads to &gt; 1, pls see the comment at<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>   // AccessController#postCompletedCreateTableAction<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>   this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>   startProcedureExecutor();<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span><a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>    // Initial cleaner chore<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>    CleanerChore.initChorePool(conf);<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>   // Start log cleaner thread<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>   int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 600 * 1000);<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>   this.logCleaner =<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>      new LogCleaner(cleanerInterval,<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span>         this, conf, getMasterWalManager().getFileSystem(),<a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>         getMasterWalManager().getOldLogDir());<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>    getChoreService().scheduleChore(logCleaner);<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span><a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>    // start the hfile archive cleaner thread<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>    Path archiveDir = HFileArchiveUtil.getArchivePath(conf);<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>    Map&lt;String, Object&gt; params = new HashMap&lt;&gt;();<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>    params.put(MASTER, this);<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>    this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>        .getFileSystem(), archiveDir, params);<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span>    getChoreService().scheduleChore(hfileCleaner);<a name="line.1331"></a>
+<span class="sourceLineNo">1332</span><a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>    replicationBarrierCleaner = new ReplicationBarrierCleaner(conf, this, getConnection(),<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>      replicationPeerManager);<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>    getChoreService().scheduleChore(replicationBarrierCleaner);<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span><a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>    serviceStarted = true;<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>    if (LOG.isTraceEnabled()) {<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>      LOG.trace("Started service threads");<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>    }<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>  }<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span><a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>  @Override<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>  protected void stopServiceThreads() {<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>    if (masterJettyServer != null) {<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>      LOG.info("Stopping master jetty server");<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>      try {<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span>        masterJettyServer.stop();<a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>      } catch (Exception e) {<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>        LOG.error("Failed to stop master jetty server", e);<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>      }<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>    }<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span>    stopChores();<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>    if (this.mobCompactThread != null) {<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>      this.mobCompactThread.close();<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span>    }<a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>    super.stopServiceThreads();<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>    CleanerChore.shutDownChorePool();<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span><a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>    LOG.debug("Stopping service threads");<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span><a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>    if (this.quotaManager != null) {<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>      this.quotaManager.stop();<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>    }<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span><a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>    if (this.activeMasterManager != null) {<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>      this.activeMasterManager.stop();<a name="line.1367"></a>
 <span class="sourceLineNo">1368</span>    }<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>    if (this.assignmentManager != null) {<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>      this.assignmentManager.stop();<a name="line.1370"></a>
+<span class="sourceLineNo">1369</span>    if (this.serverManager != null) {<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      this.serverManager.stop();<a name="line.1370"></a>
 <span class="sourceLineNo">1371</span>    }<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span><a name="line.1372"></a>
-<span class="sourceLineNo">1373</span>    stopProcedureExecutor();<a name="line.1373"></a>
-<span class="sourceLineNo">1374</span><a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>    if (this.walManager != null) {<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>      this.walManager.stop();<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>    }<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>    if (this.fileSystemManager != null) {<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>      this.fileSystemManager.stop();<a name="line.1379"></a>
+<span class="sourceLineNo">1372</span>    if (this.assignmentManager != null) {<a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>      this.assignmentManager.stop();<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>    }<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span><a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>    stopProcedureExecutor();<a name="line

<TRUNCATED>

[07/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html
index 4a42bda..5815be5 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html
@@ -91,177 +91,178 @@
 <span class="sourceLineNo">083</span>      // Ignore the last log which is current active log.<a name="line.83"></a>
 <span class="sourceLineNo">084</span>      while (logs.hasNext()) {<a name="line.84"></a>
 <span class="sourceLineNo">085</span>        ProcedureWALFile log = logs.next();<a name="line.85"></a>
-<span class="sourceLineNo">086</span>        log.open();<a name="line.86"></a>
-<span class="sourceLineNo">087</span>        try {<a name="line.87"></a>
-<span class="sourceLineNo">088</span>          reader.read(log);<a name="line.88"></a>
-<span class="sourceLineNo">089</span>        } finally {<a name="line.89"></a>
-<span class="sourceLineNo">090</span>          log.close();<a name="line.90"></a>
-<span class="sourceLineNo">091</span>        }<a name="line.91"></a>
-<span class="sourceLineNo">092</span>      }<a name="line.92"></a>
-<span class="sourceLineNo">093</span>      reader.finish();<a name="line.93"></a>
-<span class="sourceLineNo">094</span><a name="line.94"></a>
-<span class="sourceLineNo">095</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.95"></a>
-<span class="sourceLineNo">096</span>      tracker.setPartialFlag(false);<a name="line.96"></a>
-<span class="sourceLineNo">097</span>      tracker.resetUpdates();<a name="line.97"></a>
-<span class="sourceLineNo">098</span>    } finally {<a name="line.98"></a>
-<span class="sourceLineNo">099</span>      tracker.setKeepDeletes(false);<a name="line.99"></a>
-<span class="sourceLineNo">100</span>    }<a name="line.100"></a>
-<span class="sourceLineNo">101</span>  }<a name="line.101"></a>
-<span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.103"></a>
-<span class="sourceLineNo">104</span>      throws IOException {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>    header.writeDelimitedTo(stream);<a name="line.105"></a>
-<span class="sourceLineNo">106</span>  }<a name="line.106"></a>
-<span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span>  /*<a name="line.108"></a>
-<span class="sourceLineNo">109</span>   * +-----------------+<a name="line.109"></a>
-<span class="sourceLineNo">110</span>   * | END OF WAL DATA | &lt;---+<a name="line.110"></a>
-<span class="sourceLineNo">111</span>   * +-----------------+     |<a name="line.111"></a>
-<span class="sourceLineNo">112</span>   * |                 |     |<a name="line.112"></a>
-<span class="sourceLineNo">113</span>   * |     Tracker     |     |<a name="line.113"></a>
-<span class="sourceLineNo">114</span>   * |                 |     |<a name="line.114"></a>
-<span class="sourceLineNo">115</span>   * +-----------------+     |<a name="line.115"></a>
-<span class="sourceLineNo">116</span>   * |     version     |     |<a name="line.116"></a>
-<span class="sourceLineNo">117</span>   * +-----------------+     |<a name="line.117"></a>
-<span class="sourceLineNo">118</span>   * |  TRAILER_MAGIC  |     |<a name="line.118"></a>
-<span class="sourceLineNo">119</span>   * +-----------------+     |<a name="line.119"></a>
-<span class="sourceLineNo">120</span>   * |      offset     |-----+<a name="line.120"></a>
-<span class="sourceLineNo">121</span>   * +-----------------+<a name="line.121"></a>
-<span class="sourceLineNo">122</span>   */<a name="line.122"></a>
-<span class="sourceLineNo">123</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.123"></a>
-<span class="sourceLineNo">124</span>      throws IOException {<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    long offset = stream.getPos();<a name="line.125"></a>
-<span class="sourceLineNo">126</span><a name="line.126"></a>
-<span class="sourceLineNo">127</span>    // Write EOF Entry<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    ProcedureWALEntry.newBuilder()<a name="line.128"></a>
-<span class="sourceLineNo">129</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.129"></a>
-<span class="sourceLineNo">130</span>      .build().writeDelimitedTo(stream);<a name="line.130"></a>
-<span class="sourceLineNo">131</span><a name="line.131"></a>
-<span class="sourceLineNo">132</span>    // Write Tracker<a name="line.132"></a>
-<span class="sourceLineNo">133</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.133"></a>
-<span class="sourceLineNo">134</span><a name="line.134"></a>
-<span class="sourceLineNo">135</span>    stream.write(TRAILER_VERSION);<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    StreamUtils.writeLong(stream, offset);<a name="line.137"></a>
-<span class="sourceLineNo">138</span>    return stream.getPos() - offset;<a name="line.138"></a>
-<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
-<span class="sourceLineNo">140</span><a name="line.140"></a>
-<span class="sourceLineNo">141</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.141"></a>
-<span class="sourceLineNo">142</span>      throws IOException {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>    ProcedureWALHeader header;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    try {<a name="line.144"></a>
-<span class="sourceLineNo">145</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.145"></a>
-<span class="sourceLineNo">146</span>    } catch (InvalidProtocolBufferException e) {<a name="line.146"></a>
-<span class="sourceLineNo">147</span>      throw new InvalidWALDataException(e);<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    }<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span>    if (header == null) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    }<a name="line.152"></a>
-<span class="sourceLineNo">153</span><a name="line.153"></a>
-<span class="sourceLineNo">154</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.155"></a>
-<span class="sourceLineNo">156</span>          " expected " + HEADER_VERSION);<a name="line.156"></a>
-<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
-<span class="sourceLineNo">158</span><a name="line.158"></a>
-<span class="sourceLineNo">159</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    }<a name="line.161"></a>
-<span class="sourceLineNo">162</span><a name="line.162"></a>
-<span class="sourceLineNo">163</span>    return header;<a name="line.163"></a>
-<span class="sourceLineNo">164</span>  }<a name="line.164"></a>
-<span class="sourceLineNo">165</span><a name="line.165"></a>
-<span class="sourceLineNo">166</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.166"></a>
-<span class="sourceLineNo">167</span>      throws IOException {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    long trailerPos = size - 17;<a name="line.169"></a>
-<span class="sourceLineNo">170</span><a name="line.170"></a>
-<span class="sourceLineNo">171</span>    if (trailerPos &lt; startPos) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.172"></a>
-<span class="sourceLineNo">173</span>    }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>    stream.seek(trailerPos);<a name="line.175"></a>
-<span class="sourceLineNo">176</span>    int version = stream.read();<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    if (version != TRAILER_VERSION) {<a name="line.177"></a>
-<span class="sourceLineNo">178</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.178"></a>
-<span class="sourceLineNo">179</span>          " expected " + TRAILER_VERSION);<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    }<a name="line.180"></a>
-<span class="sourceLineNo">181</span><a name="line.181"></a>
-<span class="sourceLineNo">182</span>    long magic = StreamUtils.readLong(stream);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    if (magic != TRAILER_MAGIC) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.184"></a>
-<span class="sourceLineNo">185</span>          " expected " + TRAILER_MAGIC);<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    stream.seek(trailerOffset);<a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    }<a name="line.194"></a>
-<span class="sourceLineNo">195</span><a name="line.195"></a>
-<span class="sourceLineNo">196</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      .setVersion(version)<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      .setTrackerPos(stream.getPos())<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      .build();<a name="line.199"></a>
-<span class="sourceLineNo">200</span>    return trailer;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>  }<a name="line.201"></a>
-<span class="sourceLineNo">202</span><a name="line.202"></a>
-<span class="sourceLineNo">203</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.203"></a>
-<span class="sourceLineNo">204</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.204"></a>
-<span class="sourceLineNo">205</span>  }<a name="line.205"></a>
-<span class="sourceLineNo">206</span><a name="line.206"></a>
-<span class="sourceLineNo">207</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      Procedure proc, Procedure[] subprocs) throws IOException {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.209"></a>
-<span class="sourceLineNo">210</span>    builder.setType(type);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.211"></a>
-<span class="sourceLineNo">212</span>    if (subprocs != null) {<a name="line.212"></a>
-<span class="sourceLineNo">213</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      }<a name="line.215"></a>
-<span class="sourceLineNo">216</span>    }<a name="line.216"></a>
-<span class="sourceLineNo">217</span>    builder.build().writeDelimitedTo(slot);<a name="line.217"></a>
-<span class="sourceLineNo">218</span>  }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span>  public static void writeInsert(ByteSlot slot, Procedure proc)<a name="line.220"></a>
-<span class="sourceLineNo">221</span>      throws IOException {<a name="line.221"></a>
-<span class="sourceLineNo">222</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.222"></a>
-<span class="sourceLineNo">223</span>  }<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>  public static void writeInsert(ByteSlot slot, Procedure proc, Procedure[] subprocs)<a name="line.225"></a>
-<span class="sourceLineNo">226</span>      throws IOException {<a name="line.226"></a>
-<span class="sourceLineNo">227</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.227"></a>
-<span class="sourceLineNo">228</span>  }<a name="line.228"></a>
-<span class="sourceLineNo">229</span><a name="line.229"></a>
-<span class="sourceLineNo">230</span>  public static void writeUpdate(ByteSlot slot, Procedure proc)<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      throws IOException {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>  }<a name="line.233"></a>
-<span class="sourceLineNo">234</span><a name="line.234"></a>
-<span class="sourceLineNo">235</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      throws IOException {<a name="line.236"></a>
-<span class="sourceLineNo">237</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    builder.setProcId(procId);<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    builder.build().writeDelimitedTo(slot);<a name="line.240"></a>
-<span class="sourceLineNo">241</span>  }<a name="line.241"></a>
-<span class="sourceLineNo">242</span><a name="line.242"></a>
-<span class="sourceLineNo">243</span>  public static void writeDelete(ByteSlot slot, Procedure proc, long[] subprocs)<a name="line.243"></a>
-<span class="sourceLineNo">244</span>      throws IOException {<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    builder.setProcId(proc.getProcId());<a name="line.247"></a>
-<span class="sourceLineNo">248</span>    if (subprocs != null) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        builder.addChildId(subprocs[i]);<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span>    builder.build().writeDelimitedTo(slot);<a name="line.254"></a>
-<span class="sourceLineNo">255</span>  }<a name="line.255"></a>
-<span class="sourceLineNo">256</span>}<a name="line.256"></a>
+<span class="sourceLineNo">086</span>        LOG.debug("Loading WAL id={}", log.getLogId());<a name="line.86"></a>
+<span class="sourceLineNo">087</span>        log.open();<a name="line.87"></a>
+<span class="sourceLineNo">088</span>        try {<a name="line.88"></a>
+<span class="sourceLineNo">089</span>          reader.read(log);<a name="line.89"></a>
+<span class="sourceLineNo">090</span>        } finally {<a name="line.90"></a>
+<span class="sourceLineNo">091</span>          log.close();<a name="line.91"></a>
+<span class="sourceLineNo">092</span>        }<a name="line.92"></a>
+<span class="sourceLineNo">093</span>      }<a name="line.93"></a>
+<span class="sourceLineNo">094</span>      reader.finish();<a name="line.94"></a>
+<span class="sourceLineNo">095</span><a name="line.95"></a>
+<span class="sourceLineNo">096</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.96"></a>
+<span class="sourceLineNo">097</span>      tracker.setPartialFlag(false);<a name="line.97"></a>
+<span class="sourceLineNo">098</span>      tracker.resetUpdates();<a name="line.98"></a>
+<span class="sourceLineNo">099</span>    } finally {<a name="line.99"></a>
+<span class="sourceLineNo">100</span>      tracker.setKeepDeletes(false);<a name="line.100"></a>
+<span class="sourceLineNo">101</span>    }<a name="line.101"></a>
+<span class="sourceLineNo">102</span>  }<a name="line.102"></a>
+<span class="sourceLineNo">103</span><a name="line.103"></a>
+<span class="sourceLineNo">104</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.104"></a>
+<span class="sourceLineNo">105</span>      throws IOException {<a name="line.105"></a>
+<span class="sourceLineNo">106</span>    header.writeDelimitedTo(stream);<a name="line.106"></a>
+<span class="sourceLineNo">107</span>  }<a name="line.107"></a>
+<span class="sourceLineNo">108</span><a name="line.108"></a>
+<span class="sourceLineNo">109</span>  /*<a name="line.109"></a>
+<span class="sourceLineNo">110</span>   * +-----------------+<a name="line.110"></a>
+<span class="sourceLineNo">111</span>   * | END OF WAL DATA | &lt;---+<a name="line.111"></a>
+<span class="sourceLineNo">112</span>   * +-----------------+     |<a name="line.112"></a>
+<span class="sourceLineNo">113</span>   * |                 |     |<a name="line.113"></a>
+<span class="sourceLineNo">114</span>   * |     Tracker     |     |<a name="line.114"></a>
+<span class="sourceLineNo">115</span>   * |                 |     |<a name="line.115"></a>
+<span class="sourceLineNo">116</span>   * +-----------------+     |<a name="line.116"></a>
+<span class="sourceLineNo">117</span>   * |     version     |     |<a name="line.117"></a>
+<span class="sourceLineNo">118</span>   * +-----------------+     |<a name="line.118"></a>
+<span class="sourceLineNo">119</span>   * |  TRAILER_MAGIC  |     |<a name="line.119"></a>
+<span class="sourceLineNo">120</span>   * +-----------------+     |<a name="line.120"></a>
+<span class="sourceLineNo">121</span>   * |      offset     |-----+<a name="line.121"></a>
+<span class="sourceLineNo">122</span>   * +-----------------+<a name="line.122"></a>
+<span class="sourceLineNo">123</span>   */<a name="line.123"></a>
+<span class="sourceLineNo">124</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.124"></a>
+<span class="sourceLineNo">125</span>      throws IOException {<a name="line.125"></a>
+<span class="sourceLineNo">126</span>    long offset = stream.getPos();<a name="line.126"></a>
+<span class="sourceLineNo">127</span><a name="line.127"></a>
+<span class="sourceLineNo">128</span>    // Write EOF Entry<a name="line.128"></a>
+<span class="sourceLineNo">129</span>    ProcedureWALEntry.newBuilder()<a name="line.129"></a>
+<span class="sourceLineNo">130</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.130"></a>
+<span class="sourceLineNo">131</span>      .build().writeDelimitedTo(stream);<a name="line.131"></a>
+<span class="sourceLineNo">132</span><a name="line.132"></a>
+<span class="sourceLineNo">133</span>    // Write Tracker<a name="line.133"></a>
+<span class="sourceLineNo">134</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span>    stream.write(TRAILER_VERSION);<a name="line.136"></a>
+<span class="sourceLineNo">137</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    StreamUtils.writeLong(stream, offset);<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    return stream.getPos() - offset;<a name="line.139"></a>
+<span class="sourceLineNo">140</span>  }<a name="line.140"></a>
+<span class="sourceLineNo">141</span><a name="line.141"></a>
+<span class="sourceLineNo">142</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.142"></a>
+<span class="sourceLineNo">143</span>      throws IOException {<a name="line.143"></a>
+<span class="sourceLineNo">144</span>    ProcedureWALHeader header;<a name="line.144"></a>
+<span class="sourceLineNo">145</span>    try {<a name="line.145"></a>
+<span class="sourceLineNo">146</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    } catch (InvalidProtocolBufferException e) {<a name="line.147"></a>
+<span class="sourceLineNo">148</span>      throw new InvalidWALDataException(e);<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    }<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span>    if (header == null) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.152"></a>
+<span class="sourceLineNo">153</span>    }<a name="line.153"></a>
+<span class="sourceLineNo">154</span><a name="line.154"></a>
+<span class="sourceLineNo">155</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.156"></a>
+<span class="sourceLineNo">157</span>          " expected " + HEADER_VERSION);<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.160"></a>
+<span class="sourceLineNo">161</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.161"></a>
+<span class="sourceLineNo">162</span>    }<a name="line.162"></a>
+<span class="sourceLineNo">163</span><a name="line.163"></a>
+<span class="sourceLineNo">164</span>    return header;<a name="line.164"></a>
+<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
+<span class="sourceLineNo">166</span><a name="line.166"></a>
+<span class="sourceLineNo">167</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.167"></a>
+<span class="sourceLineNo">168</span>      throws IOException {<a name="line.168"></a>
+<span class="sourceLineNo">169</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.169"></a>
+<span class="sourceLineNo">170</span>    long trailerPos = size - 17;<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span>    if (trailerPos &lt; startPos) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.173"></a>
+<span class="sourceLineNo">174</span>    }<a name="line.174"></a>
+<span class="sourceLineNo">175</span><a name="line.175"></a>
+<span class="sourceLineNo">176</span>    stream.seek(trailerPos);<a name="line.176"></a>
+<span class="sourceLineNo">177</span>    int version = stream.read();<a name="line.177"></a>
+<span class="sourceLineNo">178</span>    if (version != TRAILER_VERSION) {<a name="line.178"></a>
+<span class="sourceLineNo">179</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.179"></a>
+<span class="sourceLineNo">180</span>          " expected " + TRAILER_VERSION);<a name="line.180"></a>
+<span class="sourceLineNo">181</span>    }<a name="line.181"></a>
+<span class="sourceLineNo">182</span><a name="line.182"></a>
+<span class="sourceLineNo">183</span>    long magic = StreamUtils.readLong(stream);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    if (magic != TRAILER_MAGIC) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.185"></a>
+<span class="sourceLineNo">186</span>          " expected " + TRAILER_MAGIC);<a name="line.186"></a>
+<span class="sourceLineNo">187</span>    }<a name="line.187"></a>
+<span class="sourceLineNo">188</span><a name="line.188"></a>
+<span class="sourceLineNo">189</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.189"></a>
+<span class="sourceLineNo">190</span>    stream.seek(trailerOffset);<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.193"></a>
+<span class="sourceLineNo">194</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    }<a name="line.195"></a>
+<span class="sourceLineNo">196</span><a name="line.196"></a>
+<span class="sourceLineNo">197</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.197"></a>
+<span class="sourceLineNo">198</span>      .setVersion(version)<a name="line.198"></a>
+<span class="sourceLineNo">199</span>      .setTrackerPos(stream.getPos())<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      .build();<a name="line.200"></a>
+<span class="sourceLineNo">201</span>    return trailer;<a name="line.201"></a>
+<span class="sourceLineNo">202</span>  }<a name="line.202"></a>
+<span class="sourceLineNo">203</span><a name="line.203"></a>
+<span class="sourceLineNo">204</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.205"></a>
+<span class="sourceLineNo">206</span>  }<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      Procedure proc, Procedure[] subprocs) throws IOException {<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.210"></a>
+<span class="sourceLineNo">211</span>    builder.setType(type);<a name="line.211"></a>
+<span class="sourceLineNo">212</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.212"></a>
+<span class="sourceLineNo">213</span>    if (subprocs != null) {<a name="line.213"></a>
+<span class="sourceLineNo">214</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.214"></a>
+<span class="sourceLineNo">215</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.215"></a>
+<span class="sourceLineNo">216</span>      }<a name="line.216"></a>
+<span class="sourceLineNo">217</span>    }<a name="line.217"></a>
+<span class="sourceLineNo">218</span>    builder.build().writeDelimitedTo(slot);<a name="line.218"></a>
+<span class="sourceLineNo">219</span>  }<a name="line.219"></a>
+<span class="sourceLineNo">220</span><a name="line.220"></a>
+<span class="sourceLineNo">221</span>  public static void writeInsert(ByteSlot slot, Procedure proc)<a name="line.221"></a>
+<span class="sourceLineNo">222</span>      throws IOException {<a name="line.222"></a>
+<span class="sourceLineNo">223</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.223"></a>
+<span class="sourceLineNo">224</span>  }<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span>  public static void writeInsert(ByteSlot slot, Procedure proc, Procedure[] subprocs)<a name="line.226"></a>
+<span class="sourceLineNo">227</span>      throws IOException {<a name="line.227"></a>
+<span class="sourceLineNo">228</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.228"></a>
+<span class="sourceLineNo">229</span>  }<a name="line.229"></a>
+<span class="sourceLineNo">230</span><a name="line.230"></a>
+<span class="sourceLineNo">231</span>  public static void writeUpdate(ByteSlot slot, Procedure proc)<a name="line.231"></a>
+<span class="sourceLineNo">232</span>      throws IOException {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.233"></a>
+<span class="sourceLineNo">234</span>  }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.236"></a>
+<span class="sourceLineNo">237</span>      throws IOException {<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    builder.setProcId(procId);<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    builder.build().writeDelimitedTo(slot);<a name="line.241"></a>
+<span class="sourceLineNo">242</span>  }<a name="line.242"></a>
+<span class="sourceLineNo">243</span><a name="line.243"></a>
+<span class="sourceLineNo">244</span>  public static void writeDelete(ByteSlot slot, Procedure proc, long[] subprocs)<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      throws IOException {<a name="line.245"></a>
+<span class="sourceLineNo">246</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.246"></a>
+<span class="sourceLineNo">247</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    builder.setProcId(proc.getProcId());<a name="line.248"></a>
+<span class="sourceLineNo">249</span>    if (subprocs != null) {<a name="line.249"></a>
+<span class="sourceLineNo">250</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.250"></a>
+<span class="sourceLineNo">251</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.251"></a>
+<span class="sourceLineNo">252</span>        builder.addChildId(subprocs[i]);<a name="line.252"></a>
+<span class="sourceLineNo">253</span>      }<a name="line.253"></a>
+<span class="sourceLineNo">254</span>    }<a name="line.254"></a>
+<span class="sourceLineNo">255</span>    builder.build().writeDelimitedTo(slot);<a name="line.255"></a>
+<span class="sourceLineNo">256</span>  }<a name="line.256"></a>
+<span class="sourceLineNo">257</span>}<a name="line.257"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html
index 4a42bda..5815be5 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html
@@ -91,177 +91,178 @@
 <span class="sourceLineNo">083</span>      // Ignore the last log which is current active log.<a name="line.83"></a>
 <span class="sourceLineNo">084</span>      while (logs.hasNext()) {<a name="line.84"></a>
 <span class="sourceLineNo">085</span>        ProcedureWALFile log = logs.next();<a name="line.85"></a>
-<span class="sourceLineNo">086</span>        log.open();<a name="line.86"></a>
-<span class="sourceLineNo">087</span>        try {<a name="line.87"></a>
-<span class="sourceLineNo">088</span>          reader.read(log);<a name="line.88"></a>
-<span class="sourceLineNo">089</span>        } finally {<a name="line.89"></a>
-<span class="sourceLineNo">090</span>          log.close();<a name="line.90"></a>
-<span class="sourceLineNo">091</span>        }<a name="line.91"></a>
-<span class="sourceLineNo">092</span>      }<a name="line.92"></a>
-<span class="sourceLineNo">093</span>      reader.finish();<a name="line.93"></a>
-<span class="sourceLineNo">094</span><a name="line.94"></a>
-<span class="sourceLineNo">095</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.95"></a>
-<span class="sourceLineNo">096</span>      tracker.setPartialFlag(false);<a name="line.96"></a>
-<span class="sourceLineNo">097</span>      tracker.resetUpdates();<a name="line.97"></a>
-<span class="sourceLineNo">098</span>    } finally {<a name="line.98"></a>
-<span class="sourceLineNo">099</span>      tracker.setKeepDeletes(false);<a name="line.99"></a>
-<span class="sourceLineNo">100</span>    }<a name="line.100"></a>
-<span class="sourceLineNo">101</span>  }<a name="line.101"></a>
-<span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.103"></a>
-<span class="sourceLineNo">104</span>      throws IOException {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>    header.writeDelimitedTo(stream);<a name="line.105"></a>
-<span class="sourceLineNo">106</span>  }<a name="line.106"></a>
-<span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span>  /*<a name="line.108"></a>
-<span class="sourceLineNo">109</span>   * +-----------------+<a name="line.109"></a>
-<span class="sourceLineNo">110</span>   * | END OF WAL DATA | &lt;---+<a name="line.110"></a>
-<span class="sourceLineNo">111</span>   * +-----------------+     |<a name="line.111"></a>
-<span class="sourceLineNo">112</span>   * |                 |     |<a name="line.112"></a>
-<span class="sourceLineNo">113</span>   * |     Tracker     |     |<a name="line.113"></a>
-<span class="sourceLineNo">114</span>   * |                 |     |<a name="line.114"></a>
-<span class="sourceLineNo">115</span>   * +-----------------+     |<a name="line.115"></a>
-<span class="sourceLineNo">116</span>   * |     version     |     |<a name="line.116"></a>
-<span class="sourceLineNo">117</span>   * +-----------------+     |<a name="line.117"></a>
-<span class="sourceLineNo">118</span>   * |  TRAILER_MAGIC  |     |<a name="line.118"></a>
-<span class="sourceLineNo">119</span>   * +-----------------+     |<a name="line.119"></a>
-<span class="sourceLineNo">120</span>   * |      offset     |-----+<a name="line.120"></a>
-<span class="sourceLineNo">121</span>   * +-----------------+<a name="line.121"></a>
-<span class="sourceLineNo">122</span>   */<a name="line.122"></a>
-<span class="sourceLineNo">123</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.123"></a>
-<span class="sourceLineNo">124</span>      throws IOException {<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    long offset = stream.getPos();<a name="line.125"></a>
-<span class="sourceLineNo">126</span><a name="line.126"></a>
-<span class="sourceLineNo">127</span>    // Write EOF Entry<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    ProcedureWALEntry.newBuilder()<a name="line.128"></a>
-<span class="sourceLineNo">129</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.129"></a>
-<span class="sourceLineNo">130</span>      .build().writeDelimitedTo(stream);<a name="line.130"></a>
-<span class="sourceLineNo">131</span><a name="line.131"></a>
-<span class="sourceLineNo">132</span>    // Write Tracker<a name="line.132"></a>
-<span class="sourceLineNo">133</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.133"></a>
-<span class="sourceLineNo">134</span><a name="line.134"></a>
-<span class="sourceLineNo">135</span>    stream.write(TRAILER_VERSION);<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    StreamUtils.writeLong(stream, offset);<a name="line.137"></a>
-<span class="sourceLineNo">138</span>    return stream.getPos() - offset;<a name="line.138"></a>
-<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
-<span class="sourceLineNo">140</span><a name="line.140"></a>
-<span class="sourceLineNo">141</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.141"></a>
-<span class="sourceLineNo">142</span>      throws IOException {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>    ProcedureWALHeader header;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    try {<a name="line.144"></a>
-<span class="sourceLineNo">145</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.145"></a>
-<span class="sourceLineNo">146</span>    } catch (InvalidProtocolBufferException e) {<a name="line.146"></a>
-<span class="sourceLineNo">147</span>      throw new InvalidWALDataException(e);<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    }<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span>    if (header == null) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    }<a name="line.152"></a>
-<span class="sourceLineNo">153</span><a name="line.153"></a>
-<span class="sourceLineNo">154</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.155"></a>
-<span class="sourceLineNo">156</span>          " expected " + HEADER_VERSION);<a name="line.156"></a>
-<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
-<span class="sourceLineNo">158</span><a name="line.158"></a>
-<span class="sourceLineNo">159</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    }<a name="line.161"></a>
-<span class="sourceLineNo">162</span><a name="line.162"></a>
-<span class="sourceLineNo">163</span>    return header;<a name="line.163"></a>
-<span class="sourceLineNo">164</span>  }<a name="line.164"></a>
-<span class="sourceLineNo">165</span><a name="line.165"></a>
-<span class="sourceLineNo">166</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.166"></a>
-<span class="sourceLineNo">167</span>      throws IOException {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    long trailerPos = size - 17;<a name="line.169"></a>
-<span class="sourceLineNo">170</span><a name="line.170"></a>
-<span class="sourceLineNo">171</span>    if (trailerPos &lt; startPos) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.172"></a>
-<span class="sourceLineNo">173</span>    }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>    stream.seek(trailerPos);<a name="line.175"></a>
-<span class="sourceLineNo">176</span>    int version = stream.read();<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    if (version != TRAILER_VERSION) {<a name="line.177"></a>
-<span class="sourceLineNo">178</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.178"></a>
-<span class="sourceLineNo">179</span>          " expected " + TRAILER_VERSION);<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    }<a name="line.180"></a>
-<span class="sourceLineNo">181</span><a name="line.181"></a>
-<span class="sourceLineNo">182</span>    long magic = StreamUtils.readLong(stream);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    if (magic != TRAILER_MAGIC) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.184"></a>
-<span class="sourceLineNo">185</span>          " expected " + TRAILER_MAGIC);<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    stream.seek(trailerOffset);<a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    }<a name="line.194"></a>
-<span class="sourceLineNo">195</span><a name="line.195"></a>
-<span class="sourceLineNo">196</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      .setVersion(version)<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      .setTrackerPos(stream.getPos())<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      .build();<a name="line.199"></a>
-<span class="sourceLineNo">200</span>    return trailer;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>  }<a name="line.201"></a>
-<span class="sourceLineNo">202</span><a name="line.202"></a>
-<span class="sourceLineNo">203</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.203"></a>
-<span class="sourceLineNo">204</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.204"></a>
-<span class="sourceLineNo">205</span>  }<a name="line.205"></a>
-<span class="sourceLineNo">206</span><a name="line.206"></a>
-<span class="sourceLineNo">207</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      Procedure proc, Procedure[] subprocs) throws IOException {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.209"></a>
-<span class="sourceLineNo">210</span>    builder.setType(type);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.211"></a>
-<span class="sourceLineNo">212</span>    if (subprocs != null) {<a name="line.212"></a>
-<span class="sourceLineNo">213</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      }<a name="line.215"></a>
-<span class="sourceLineNo">216</span>    }<a name="line.216"></a>
-<span class="sourceLineNo">217</span>    builder.build().writeDelimitedTo(slot);<a name="line.217"></a>
-<span class="sourceLineNo">218</span>  }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span>  public static void writeInsert(ByteSlot slot, Procedure proc)<a name="line.220"></a>
-<span class="sourceLineNo">221</span>      throws IOException {<a name="line.221"></a>
-<span class="sourceLineNo">222</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.222"></a>
-<span class="sourceLineNo">223</span>  }<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>  public static void writeInsert(ByteSlot slot, Procedure proc, Procedure[] subprocs)<a name="line.225"></a>
-<span class="sourceLineNo">226</span>      throws IOException {<a name="line.226"></a>
-<span class="sourceLineNo">227</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.227"></a>
-<span class="sourceLineNo">228</span>  }<a name="line.228"></a>
-<span class="sourceLineNo">229</span><a name="line.229"></a>
-<span class="sourceLineNo">230</span>  public static void writeUpdate(ByteSlot slot, Procedure proc)<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      throws IOException {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>  }<a name="line.233"></a>
-<span class="sourceLineNo">234</span><a name="line.234"></a>
-<span class="sourceLineNo">235</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      throws IOException {<a name="line.236"></a>
-<span class="sourceLineNo">237</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    builder.setProcId(procId);<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    builder.build().writeDelimitedTo(slot);<a name="line.240"></a>
-<span class="sourceLineNo">241</span>  }<a name="line.241"></a>
-<span class="sourceLineNo">242</span><a name="line.242"></a>
-<span class="sourceLineNo">243</span>  public static void writeDelete(ByteSlot slot, Procedure proc, long[] subprocs)<a name="line.243"></a>
-<span class="sourceLineNo">244</span>      throws IOException {<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    builder.setProcId(proc.getProcId());<a name="line.247"></a>
-<span class="sourceLineNo">248</span>    if (subprocs != null) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        builder.addChildId(subprocs[i]);<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span>    builder.build().writeDelimitedTo(slot);<a name="line.254"></a>
-<span class="sourceLineNo">255</span>  }<a name="line.255"></a>
-<span class="sourceLineNo">256</span>}<a name="line.256"></a>
+<span class="sourceLineNo">086</span>        LOG.debug("Loading WAL id={}", log.getLogId());<a name="line.86"></a>
+<span class="sourceLineNo">087</span>        log.open();<a name="line.87"></a>
+<span class="sourceLineNo">088</span>        try {<a name="line.88"></a>
+<span class="sourceLineNo">089</span>          reader.read(log);<a name="line.89"></a>
+<span class="sourceLineNo">090</span>        } finally {<a name="line.90"></a>
+<span class="sourceLineNo">091</span>          log.close();<a name="line.91"></a>
+<span class="sourceLineNo">092</span>        }<a name="line.92"></a>
+<span class="sourceLineNo">093</span>      }<a name="line.93"></a>
+<span class="sourceLineNo">094</span>      reader.finish();<a name="line.94"></a>
+<span class="sourceLineNo">095</span><a name="line.95"></a>
+<span class="sourceLineNo">096</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.96"></a>
+<span class="sourceLineNo">097</span>      tracker.setPartialFlag(false);<a name="line.97"></a>
+<span class="sourceLineNo">098</span>      tracker.resetUpdates();<a name="line.98"></a>
+<span class="sourceLineNo">099</span>    } finally {<a name="line.99"></a>
+<span class="sourceLineNo">100</span>      tracker.setKeepDeletes(false);<a name="line.100"></a>
+<span class="sourceLineNo">101</span>    }<a name="line.101"></a>
+<span class="sourceLineNo">102</span>  }<a name="line.102"></a>
+<span class="sourceLineNo">103</span><a name="line.103"></a>
+<span class="sourceLineNo">104</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.104"></a>
+<span class="sourceLineNo">105</span>      throws IOException {<a name="line.105"></a>
+<span class="sourceLineNo">106</span>    header.writeDelimitedTo(stream);<a name="line.106"></a>
+<span class="sourceLineNo">107</span>  }<a name="line.107"></a>
+<span class="sourceLineNo">108</span><a name="line.108"></a>
+<span class="sourceLineNo">109</span>  /*<a name="line.109"></a>
+<span class="sourceLineNo">110</span>   * +-----------------+<a name="line.110"></a>
+<span class="sourceLineNo">111</span>   * | END OF WAL DATA | &lt;---+<a name="line.111"></a>
+<span class="sourceLineNo">112</span>   * +-----------------+     |<a name="line.112"></a>
+<span class="sourceLineNo">113</span>   * |                 |     |<a name="line.113"></a>
+<span class="sourceLineNo">114</span>   * |     Tracker     |     |<a name="line.114"></a>
+<span class="sourceLineNo">115</span>   * |                 |     |<a name="line.115"></a>
+<span class="sourceLineNo">116</span>   * +-----------------+     |<a name="line.116"></a>
+<span class="sourceLineNo">117</span>   * |     version     |     |<a name="line.117"></a>
+<span class="sourceLineNo">118</span>   * +-----------------+     |<a name="line.118"></a>
+<span class="sourceLineNo">119</span>   * |  TRAILER_MAGIC  |     |<a name="line.119"></a>
+<span class="sourceLineNo">120</span>   * +-----------------+     |<a name="line.120"></a>
+<span class="sourceLineNo">121</span>   * |      offset     |-----+<a name="line.121"></a>
+<span class="sourceLineNo">122</span>   * +-----------------+<a name="line.122"></a>
+<span class="sourceLineNo">123</span>   */<a name="line.123"></a>
+<span class="sourceLineNo">124</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.124"></a>
+<span class="sourceLineNo">125</span>      throws IOException {<a name="line.125"></a>
+<span class="sourceLineNo">126</span>    long offset = stream.getPos();<a name="line.126"></a>
+<span class="sourceLineNo">127</span><a name="line.127"></a>
+<span class="sourceLineNo">128</span>    // Write EOF Entry<a name="line.128"></a>
+<span class="sourceLineNo">129</span>    ProcedureWALEntry.newBuilder()<a name="line.129"></a>
+<span class="sourceLineNo">130</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.130"></a>
+<span class="sourceLineNo">131</span>      .build().writeDelimitedTo(stream);<a name="line.131"></a>
+<span class="sourceLineNo">132</span><a name="line.132"></a>
+<span class="sourceLineNo">133</span>    // Write Tracker<a name="line.133"></a>
+<span class="sourceLineNo">134</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span>    stream.write(TRAILER_VERSION);<a name="line.136"></a>
+<span class="sourceLineNo">137</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    StreamUtils.writeLong(stream, offset);<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    return stream.getPos() - offset;<a name="line.139"></a>
+<span class="sourceLineNo">140</span>  }<a name="line.140"></a>
+<span class="sourceLineNo">141</span><a name="line.141"></a>
+<span class="sourceLineNo">142</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.142"></a>
+<span class="sourceLineNo">143</span>      throws IOException {<a name="line.143"></a>
+<span class="sourceLineNo">144</span>    ProcedureWALHeader header;<a name="line.144"></a>
+<span class="sourceLineNo">145</span>    try {<a name="line.145"></a>
+<span class="sourceLineNo">146</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    } catch (InvalidProtocolBufferException e) {<a name="line.147"></a>
+<span class="sourceLineNo">148</span>      throw new InvalidWALDataException(e);<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    }<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span>    if (header == null) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.152"></a>
+<span class="sourceLineNo">153</span>    }<a name="line.153"></a>
+<span class="sourceLineNo">154</span><a name="line.154"></a>
+<span class="sourceLineNo">155</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.156"></a>
+<span class="sourceLineNo">157</span>          " expected " + HEADER_VERSION);<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.160"></a>
+<span class="sourceLineNo">161</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.161"></a>
+<span class="sourceLineNo">162</span>    }<a name="line.162"></a>
+<span class="sourceLineNo">163</span><a name="line.163"></a>
+<span class="sourceLineNo">164</span>    return header;<a name="line.164"></a>
+<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
+<span class="sourceLineNo">166</span><a name="line.166"></a>
+<span class="sourceLineNo">167</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.167"></a>
+<span class="sourceLineNo">168</span>      throws IOException {<a name="line.168"></a>
+<span class="sourceLineNo">169</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.169"></a>
+<span class="sourceLineNo">170</span>    long trailerPos = size - 17;<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span>    if (trailerPos &lt; startPos) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.173"></a>
+<span class="sourceLineNo">174</span>    }<a name="line.174"></a>
+<span class="sourceLineNo">175</span><a name="line.175"></a>
+<span class="sourceLineNo">176</span>    stream.seek(trailerPos);<a name="line.176"></a>
+<span class="sourceLineNo">177</span>    int version = stream.read();<a name="line.177"></a>
+<span class="sourceLineNo">178</span>    if (version != TRAILER_VERSION) {<a name="line.178"></a>
+<span class="sourceLineNo">179</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.179"></a>
+<span class="sourceLineNo">180</span>          " expected " + TRAILER_VERSION);<a name="line.180"></a>
+<span class="sourceLineNo">181</span>    }<a name="line.181"></a>
+<span class="sourceLineNo">182</span><a name="line.182"></a>
+<span class="sourceLineNo">183</span>    long magic = StreamUtils.readLong(stream);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    if (magic != TRAILER_MAGIC) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.185"></a>
+<span class="sourceLineNo">186</span>          " expected " + TRAILER_MAGIC);<a name="line.186"></a>
+<span class="sourceLineNo">187</span>    }<a name="line.187"></a>
+<span class="sourceLineNo">188</span><a name="line.188"></a>
+<span class="sourceLineNo">189</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.189"></a>
+<span class="sourceLineNo">190</span>    stream.seek(trailerOffset);<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.193"></a>
+<span class="sourceLineNo">194</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    }<a name="line.195"></a>
+<span class="sourceLineNo">196</span><a name="line.196"></a>
+<span class="sourceLineNo">197</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.197"></a>
+<span class="sourceLineNo">198</span>      .setVersion(version)<a name="line.198"></a>
+<span class="sourceLineNo">199</span>      .setTrackerPos(stream.getPos())<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      .build();<a name="line.200"></a>
+<span class="sourceLineNo">201</span>    return trailer;<a name="line.201"></a>
+<span class="sourceLineNo">202</span>  }<a name="line.202"></a>
+<span class="sourceLineNo">203</span><a name="line.203"></a>
+<span class="sourceLineNo">204</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.205"></a>
+<span class="sourceLineNo">206</span>  }<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      Procedure proc, Procedure[] subprocs) throws IOException {<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.210"></a>
+<span class="sourceLineNo">211</span>    builder.setType(type);<a name="line.211"></a>
+<span class="sourceLineNo">212</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.212"></a>
+<span class="sourceLineNo">213</span>    if (subprocs != null) {<a name="line.213"></a>
+<span class="sourceLineNo">214</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.214"></a>
+<span class="sourceLineNo">215</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.215"></a>
+<span class="sourceLineNo">216</span>      }<a name="line.216"></a>
+<span class="sourceLineNo">217</span>    }<a name="line.217"></a>
+<span class="sourceLineNo">218</span>    builder.build().writeDelimitedTo(slot);<a name="line.218"></a>
+<span class="sourceLineNo">219</span>  }<a name="line.219"></a>
+<span class="sourceLineNo">220</span><a name="line.220"></a>
+<span class="sourceLineNo">221</span>  public static void writeInsert(ByteSlot slot, Procedure proc)<a name="line.221"></a>
+<span class="sourceLineNo">222</span>      throws IOException {<a name="line.222"></a>
+<span class="sourceLineNo">223</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.223"></a>
+<span class="sourceLineNo">224</span>  }<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span>  public static void writeInsert(ByteSlot slot, Procedure proc, Procedure[] subprocs)<a name="line.226"></a>
+<span class="sourceLineNo">227</span>      throws IOException {<a name="line.227"></a>
+<span class="sourceLineNo">228</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.228"></a>
+<span class="sourceLineNo">229</span>  }<a name="line.229"></a>
+<span class="sourceLineNo">230</span><a name="line.230"></a>
+<span class="sourceLineNo">231</span>  public static void writeUpdate(ByteSlot slot, Procedure proc)<a name="line.231"></a>
+<span class="sourceLineNo">232</span>      throws IOException {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.233"></a>
+<span class="sourceLineNo">234</span>  }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.236"></a>
+<span class="sourceLineNo">237</span>      throws IOException {<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    builder.setProcId(procId);<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    builder.build().writeDelimitedTo(slot);<a name="line.241"></a>
+<span class="sourceLineNo">242</span>  }<a name="line.242"></a>
+<span class="sourceLineNo">243</span><a name="line.243"></a>
+<span class="sourceLineNo">244</span>  public static void writeDelete(ByteSlot slot, Procedure proc, long[] subprocs)<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      throws IOException {<a name="line.245"></a>
+<span class="sourceLineNo">246</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.246"></a>
+<span class="sourceLineNo">247</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    builder.setProcId(proc.getProcId());<a name="line.248"></a>
+<span class="sourceLineNo">249</span>    if (subprocs != null) {<a name="line.249"></a>
+<span class="sourceLineNo">250</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.250"></a>
+<span class="sourceLineNo">251</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.251"></a>
+<span class="sourceLineNo">252</span>        builder.addChildId(subprocs[i]);<a name="line.252"></a>
+<span class="sourceLineNo">253</span>      }<a name="line.253"></a>
+<span class="sourceLineNo">254</span>    }<a name="line.254"></a>
+<span class="sourceLineNo">255</span>    builder.build().writeDelimitedTo(slot);<a name="line.255"></a>
+<span class="sourceLineNo">256</span>  }<a name="line.256"></a>
+<span class="sourceLineNo">257</span>}<a name="line.257"></a>
 
 
 


[13/23] hbase-site git commit: Published site at 24f2893059e745aa6691cdd6cb37fc248880b3b9.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/c548e10d/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html
index 681e263..7cdf584 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html
@@ -182,7 +182,7 @@
 <span class="sourceLineNo">174</span>      }<a name="line.174"></a>
 <span class="sourceLineNo">175</span>      FileStatus[] logfiles = FSUtils.listStatus(fs, logDir, filter);<a name="line.175"></a>
 <span class="sourceLineNo">176</span>      if (logfiles == null || logfiles.length == 0) {<a name="line.176"></a>
-<span class="sourceLineNo">177</span>        LOG.info(logDir + " is empty dir, no logs to split");<a name="line.177"></a>
+<span class="sourceLineNo">177</span>        LOG.info("{} dir is empty, no logs to split.", logDir);<a name="line.177"></a>
 <span class="sourceLineNo">178</span>      } else {<a name="line.178"></a>
 <span class="sourceLineNo">179</span>        Collections.addAll(fileStatus, logfiles);<a name="line.179"></a>
 <span class="sourceLineNo">180</span>      }<a name="line.180"></a>
@@ -243,426 +243,430 @@
 <span class="sourceLineNo">235</span>      PathFilter filter) throws IOException {<a name="line.235"></a>
 <span class="sourceLineNo">236</span>    MonitoredTask status = TaskMonitor.get().createStatus("Doing distributed log split in " +<a name="line.236"></a>
 <span class="sourceLineNo">237</span>      logDirs + " for serverName=" + serverNames);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    status.setStatus("Checking directory contents...");<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.240"></a>
-<span class="sourceLineNo">241</span>    LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.241"></a>
-<span class="sourceLineNo">242</span>      " for " + serverNames);<a name="line.242"></a>
-<span class="sourceLineNo">243</span>    long t = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
-<span class="sourceLineNo">244</span>    long totalSize = 0;<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    TaskBatch batch = new TaskBatch();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    for (FileStatus lf : logfiles) {<a name="line.246"></a>
-<span class="sourceLineNo">247</span>      // TODO If the log file is still being written to - which is most likely<a name="line.247"></a>
-<span class="sourceLineNo">248</span>      // the case for the last log file - then its length will show up here<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      // as zero. The size of such a file can only be retrieved after<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      // recover-lease is done. totalSize will be under in most cases and the<a name="line.250"></a>
-<span class="sourceLineNo">251</span>      // metrics that it drives will also be under-reported.<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      totalSize += lf.getLen();<a name="line.252"></a>
-<span class="sourceLineNo">253</span>      String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.253"></a>
-<span class="sourceLineNo">254</span>      if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.254"></a>
-<span class="sourceLineNo">255</span>        throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.255"></a>
-<span class="sourceLineNo">256</span>      }<a name="line.256"></a>
-<span class="sourceLineNo">257</span>    }<a name="line.257"></a>
-<span class="sourceLineNo">258</span>    waitForSplittingCompletion(batch, status);<a name="line.258"></a>
-<span class="sourceLineNo">259</span><a name="line.259"></a>
-<span class="sourceLineNo">260</span>    if (batch.done != batch.installed) {<a name="line.260"></a>
-<span class="sourceLineNo">261</span>      batch.isDead = true;<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.262"></a>
-<span class="sourceLineNo">263</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.263"></a>
-<span class="sourceLineNo">264</span>          + " but only " + batch.done + " done");<a name="line.264"></a>
-<span class="sourceLineNo">265</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      status.abort(msg);<a name="line.266"></a>
-<span class="sourceLineNo">267</span>      throw new IOException(msg);<a name="line.267"></a>
-<span class="sourceLineNo">268</span>    }<a name="line.268"></a>
-<span class="sourceLineNo">269</span>    for (Path logDir : logDirs) {<a name="line.269"></a>
-<span class="sourceLineNo">270</span>      status.setStatus("Cleaning up log directory...");<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      try {<a name="line.272"></a>
-<span class="sourceLineNo">273</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.273"></a>
-<span class="sourceLineNo">274</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.274"></a>
-<span class="sourceLineNo">275</span>        }<a name="line.275"></a>
-<span class="sourceLineNo">276</span>      } catch (IOException ioe) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.277"></a>
-<span class="sourceLineNo">278</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.278"></a>
-<span class="sourceLineNo">279</span>          LOG.warn("Returning success without actually splitting and "<a name="line.279"></a>
-<span class="sourceLineNo">280</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.280"></a>
-<span class="sourceLineNo">281</span>              + Arrays.toString(files), ioe);<a name="line.281"></a>
-<span class="sourceLineNo">282</span>        } else {<a name="line.282"></a>
-<span class="sourceLineNo">283</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.283"></a>
-<span class="sourceLineNo">284</span>        }<a name="line.284"></a>
-<span class="sourceLineNo">285</span>      }<a name="line.285"></a>
-<span class="sourceLineNo">286</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.286"></a>
-<span class="sourceLineNo">287</span>    }<a name="line.287"></a>
-<span class="sourceLineNo">288</span>    String msg =<a name="line.288"></a>
-<span class="sourceLineNo">289</span>        "finished splitting (more than or equal to) " + totalSize + " bytes in " + batch.installed<a name="line.289"></a>
-<span class="sourceLineNo">290</span>            + " log files in " + logDirs + " in "<a name="line.290"></a>
-<span class="sourceLineNo">291</span>            + (EnvironmentEdgeManager.currentTime() - t) + "ms";<a name="line.291"></a>
-<span class="sourceLineNo">292</span>    status.markComplete(msg);<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    LOG.info(msg);<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    return totalSize;<a name="line.294"></a>
-<span class="sourceLineNo">295</span>  }<a name="line.295"></a>
-<span class="sourceLineNo">296</span><a name="line.296"></a>
-<span class="sourceLineNo">297</span>  /**<a name="line.297"></a>
-<span class="sourceLineNo">298</span>   * Add a task entry to coordination if it is not already there.<a name="line.298"></a>
-<span class="sourceLineNo">299</span>   * @param taskname the path of the log to be split<a name="line.299"></a>
-<span class="sourceLineNo">300</span>   * @param batch the batch this task belongs to<a name="line.300"></a>
-<span class="sourceLineNo">301</span>   * @return true if a new entry is created, false if it is already there.<a name="line.301"></a>
-<span class="sourceLineNo">302</span>   */<a name="line.302"></a>
-<span class="sourceLineNo">303</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.303"></a>
-<span class="sourceLineNo">304</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.304"></a>
-<span class="sourceLineNo">305</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.305"></a>
-<span class="sourceLineNo">306</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.306"></a>
-<span class="sourceLineNo">307</span>    if (oldtask == null) {<a name="line.307"></a>
-<span class="sourceLineNo">308</span>      // publish the task in the coordination engine<a name="line.308"></a>
-<span class="sourceLineNo">309</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.309"></a>
-<span class="sourceLineNo">310</span>      return true;<a name="line.310"></a>
-<span class="sourceLineNo">311</span>    }<a name="line.311"></a>
-<span class="sourceLineNo">312</span>    return false;<a name="line.312"></a>
-<span class="sourceLineNo">313</span>  }<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.315"></a>
-<span class="sourceLineNo">316</span>    synchronized (batch) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.317"></a>
-<span class="sourceLineNo">318</span>        try {<a name="line.318"></a>
-<span class="sourceLineNo">319</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.319"></a>
-<span class="sourceLineNo">320</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.320"></a>
-<span class="sourceLineNo">321</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.321"></a>
-<span class="sourceLineNo">322</span>          int actual = activeTasks(batch);<a name="line.322"></a>
-<span class="sourceLineNo">323</span>          if (remaining != actual) {<a name="line.323"></a>
-<span class="sourceLineNo">324</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.324"></a>
-<span class="sourceLineNo">325</span>          }<a name="line.325"></a>
-<span class="sourceLineNo">326</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.326"></a>
-<span class="sourceLineNo">327</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.327"></a>
-<span class="sourceLineNo">328</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.328"></a>
-<span class="sourceLineNo">329</span>                + remainingTasks);<a name="line.329"></a>
-<span class="sourceLineNo">330</span>          }<a name="line.330"></a>
-<span class="sourceLineNo">331</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span>            LOG.warn("No more task remaining, splitting "<a name="line.332"></a>
-<span class="sourceLineNo">333</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.333"></a>
-<span class="sourceLineNo">334</span>                + ", active tasks in map " + actual);<a name="line.334"></a>
-<span class="sourceLineNo">335</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.335"></a>
-<span class="sourceLineNo">336</span>              return;<a name="line.336"></a>
-<span class="sourceLineNo">337</span>            }<a name="line.337"></a>
-<span class="sourceLineNo">338</span>          }<a name="line.338"></a>
-<span class="sourceLineNo">339</span>          batch.wait(100);<a name="line.339"></a>
-<span class="sourceLineNo">340</span>          if (server.isStopped()) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.341"></a>
-<span class="sourceLineNo">342</span>            return;<a name="line.342"></a>
-<span class="sourceLineNo">343</span>          }<a name="line.343"></a>
-<span class="sourceLineNo">344</span>        } catch (InterruptedException e) {<a name="line.344"></a>
-<span class="sourceLineNo">345</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.345"></a>
-<span class="sourceLineNo">346</span>          Thread.currentThread().interrupt();<a name="line.346"></a>
-<span class="sourceLineNo">347</span>          return;<a name="line.347"></a>
-<span class="sourceLineNo">348</span>        }<a name="line.348"></a>
-<span class="sourceLineNo">349</span>      }<a name="line.349"></a>
-<span class="sourceLineNo">350</span>    }<a name="line.350"></a>
-<span class="sourceLineNo">351</span>  }<a name="line.351"></a>
-<span class="sourceLineNo">352</span><a name="line.352"></a>
-<span class="sourceLineNo">353</span>  @VisibleForTesting<a name="line.353"></a>
-<span class="sourceLineNo">354</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.354"></a>
-<span class="sourceLineNo">355</span>    return tasks;<a name="line.355"></a>
-<span class="sourceLineNo">356</span>  }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span>  private int activeTasks(final TaskBatch batch) {<a name="line.358"></a>
-<span class="sourceLineNo">359</span>    int count = 0;<a name="line.359"></a>
-<span class="sourceLineNo">360</span>    for (Task t : tasks.values()) {<a name="line.360"></a>
-<span class="sourceLineNo">361</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.361"></a>
-<span class="sourceLineNo">362</span>        count++;<a name="line.362"></a>
-<span class="sourceLineNo">363</span>      }<a name="line.363"></a>
-<span class="sourceLineNo">364</span>    }<a name="line.364"></a>
-<span class="sourceLineNo">365</span>    return count;<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span>  }<a name="line.367"></a>
-<span class="sourceLineNo">368</span><a name="line.368"></a>
-<span class="sourceLineNo">369</span>  /**<a name="line.369"></a>
-<span class="sourceLineNo">370</span>   * @param path<a name="line.370"></a>
-<span class="sourceLineNo">371</span>   * @param batch<a name="line.371"></a>
-<span class="sourceLineNo">372</span>   * @return null on success, existing task on error<a name="line.372"></a>
-<span class="sourceLineNo">373</span>   */<a name="line.373"></a>
-<span class="sourceLineNo">374</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span>    Task oldtask;<a name="line.375"></a>
-<span class="sourceLineNo">376</span>    // batch.installed is only changed via this function and<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    // a single thread touches batch.installed.<a name="line.377"></a>
-<span class="sourceLineNo">378</span>    Task newtask = new Task();<a name="line.378"></a>
-<span class="sourceLineNo">379</span>    newtask.batch = batch;<a name="line.379"></a>
-<span class="sourceLineNo">380</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    if (oldtask == null) {<a name="line.381"></a>
-<span class="sourceLineNo">382</span>      batch.installed++;<a name="line.382"></a>
-<span class="sourceLineNo">383</span>      return null;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    }<a name="line.384"></a>
-<span class="sourceLineNo">385</span>    // new task was not used.<a name="line.385"></a>
-<span class="sourceLineNo">386</span>    synchronized (oldtask) {<a name="line.386"></a>
-<span class="sourceLineNo">387</span>      if (oldtask.isOrphan()) {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>        if (oldtask.status == SUCCESS) {<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          // The task is already done. Do not install the batch for this<a name="line.389"></a>
-<span class="sourceLineNo">390</span>          // task because it might be too late for setDone() to update<a name="line.390"></a>
-<span class="sourceLineNo">391</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.391"></a>
-<span class="sourceLineNo">392</span>          // this task to complete.<a name="line.392"></a>
-<span class="sourceLineNo">393</span>          return (null);<a name="line.393"></a>
-<span class="sourceLineNo">394</span>        }<a name="line.394"></a>
-<span class="sourceLineNo">395</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.395"></a>
-<span class="sourceLineNo">396</span>          oldtask.batch = batch;<a name="line.396"></a>
-<span class="sourceLineNo">397</span>          batch.installed++;<a name="line.397"></a>
-<span class="sourceLineNo">398</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.398"></a>
-<span class="sourceLineNo">399</span>          return null;<a name="line.399"></a>
-<span class="sourceLineNo">400</span>        }<a name="line.400"></a>
-<span class="sourceLineNo">401</span>        while (oldtask.status == FAILURE) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.402"></a>
-<span class="sourceLineNo">403</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.403"></a>
-<span class="sourceLineNo">404</span>          try {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>            oldtask.wait();<a name="line.405"></a>
-<span class="sourceLineNo">406</span>          } catch (InterruptedException e) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span>            Thread.currentThread().interrupt();<a name="line.407"></a>
-<span class="sourceLineNo">408</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.408"></a>
-<span class="sourceLineNo">409</span>            // fall through to return failure<a name="line.409"></a>
-<span class="sourceLineNo">410</span>            break;<a name="line.410"></a>
-<span class="sourceLineNo">411</span>          }<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        }<a name="line.412"></a>
-<span class="sourceLineNo">413</span>        if (oldtask.status != DELETED) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span>          LOG.warn("Failure because previously failed task"<a name="line.414"></a>
-<span class="sourceLineNo">415</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.415"></a>
-<span class="sourceLineNo">416</span>          return oldtask;<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        }<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        // reinsert the newTask and it must succeed this time<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        if (t == null) {<a name="line.420"></a>
-<span class="sourceLineNo">421</span>          batch.installed++;<a name="line.421"></a>
-<span class="sourceLineNo">422</span>          return null;<a name="line.422"></a>
-<span class="sourceLineNo">423</span>        }<a name="line.423"></a>
-<span class="sourceLineNo">424</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        assert false : "Deleted task still present in tasks map";<a name="line.425"></a>
-<span class="sourceLineNo">426</span>        return t;<a name="line.426"></a>
-<span class="sourceLineNo">427</span>      }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.428"></a>
-<span class="sourceLineNo">429</span>      return oldtask;<a name="line.429"></a>
-<span class="sourceLineNo">430</span>    }<a name="line.430"></a>
-<span class="sourceLineNo">431</span>  }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span>  public void stop() {<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (choreService != null) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      choreService.shutdown();<a name="line.435"></a>
-<span class="sourceLineNo">436</span>    }<a name="line.436"></a>
-<span class="sourceLineNo">437</span>    if (timeoutMonitor != null) {<a name="line.437"></a>
-<span class="sourceLineNo">438</span>      timeoutMonitor.cancel(true);<a name="line.438"></a>
-<span class="sourceLineNo">439</span>    }<a name="line.439"></a>
-<span class="sourceLineNo">440</span>  }<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span>  void handleDeadWorker(ServerName workerName) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.443"></a>
-<span class="sourceLineNo">444</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.444"></a>
-<span class="sourceLineNo">445</span>    synchronized (deadWorkersLock) {<a name="line.445"></a>
-<span class="sourceLineNo">446</span>      if (deadWorkers == null) {<a name="line.446"></a>
-<span class="sourceLineNo">447</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.447"></a>
-<span class="sourceLineNo">448</span>      }<a name="line.448"></a>
-<span class="sourceLineNo">449</span>      deadWorkers.add(workerName);<a name="line.449"></a>
-<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
-<span class="sourceLineNo">451</span>    LOG.info("dead splitlog worker " + workerName);<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  }<a name="line.452"></a>
-<span class="sourceLineNo">453</span><a name="line.453"></a>
-<span class="sourceLineNo">454</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.454"></a>
-<span class="sourceLineNo">455</span>    synchronized (deadWorkersLock) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span>      if (deadWorkers == null) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      }<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      deadWorkers.addAll(serverNames);<a name="line.459"></a>
-<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
-<span class="sourceLineNo">461</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.461"></a>
-<span class="sourceLineNo">462</span>  }<a name="line.462"></a>
-<span class="sourceLineNo">463</span><a name="line.463"></a>
-<span class="sourceLineNo">464</span>  /**<a name="line.464"></a>
-<span class="sourceLineNo">465</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.465"></a>
-<span class="sourceLineNo">466</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.466"></a>
-<span class="sourceLineNo">467</span>   * &lt;p&gt;<a name="line.467"></a>
-<span class="sourceLineNo">468</span>   * All access is synchronized.<a name="line.468"></a>
-<span class="sourceLineNo">469</span>   */<a name="line.469"></a>
-<span class="sourceLineNo">470</span>  @InterfaceAudience.Private<a name="line.470"></a>
-<span class="sourceLineNo">471</span>  public static class TaskBatch {<a name="line.471"></a>
-<span class="sourceLineNo">472</span>    public int installed = 0;<a name="line.472"></a>
-<span class="sourceLineNo">473</span>    public int done = 0;<a name="line.473"></a>
-<span class="sourceLineNo">474</span>    public int error = 0;<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    public volatile boolean isDead = false;<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span>    @Override<a name="line.477"></a>
-<span class="sourceLineNo">478</span>    public String toString() {<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.479"></a>
-<span class="sourceLineNo">480</span>    }<a name="line.480"></a>
-<span class="sourceLineNo">481</span>  }<a name="line.481"></a>
-<span class="sourceLineNo">482</span><a name="line.482"></a>
-<span class="sourceLineNo">483</span>  /**<a name="line.483"></a>
-<span class="sourceLineNo">484</span>   * in memory state of an active task.<a name="line.484"></a>
-<span class="sourceLineNo">485</span>   */<a name="line.485"></a>
-<span class="sourceLineNo">486</span>  @InterfaceAudience.Private<a name="line.486"></a>
-<span class="sourceLineNo">487</span>  public static class Task {<a name="line.487"></a>
-<span class="sourceLineNo">488</span>    public volatile long last_update;<a name="line.488"></a>
-<span class="sourceLineNo">489</span>    public volatile int last_version;<a name="line.489"></a>
-<span class="sourceLineNo">490</span>    public volatile ServerName cur_worker_name;<a name="line.490"></a>
-<span class="sourceLineNo">491</span>    public volatile TaskBatch batch;<a name="line.491"></a>
-<span class="sourceLineNo">492</span>    public volatile TerminationStatus status;<a name="line.492"></a>
-<span class="sourceLineNo">493</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.493"></a>
-<span class="sourceLineNo">494</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    public volatile boolean resubmitThresholdReached;<a name="line.495"></a>
-<span class="sourceLineNo">496</span><a name="line.496"></a>
-<span class="sourceLineNo">497</span>    @Override<a name="line.497"></a>
-<span class="sourceLineNo">498</span>    public String toString() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.499"></a>
-<span class="sourceLineNo">500</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.500"></a>
-<span class="sourceLineNo">501</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.501"></a>
-<span class="sourceLineNo">502</span>    }<a name="line.502"></a>
-<span class="sourceLineNo">503</span><a name="line.503"></a>
-<span class="sourceLineNo">504</span>    public Task() {<a name="line.504"></a>
-<span class="sourceLineNo">505</span>      last_version = -1;<a name="line.505"></a>
-<span class="sourceLineNo">506</span>      status = IN_PROGRESS;<a name="line.506"></a>
-<span class="sourceLineNo">507</span>      setUnassigned();<a name="line.507"></a>
-<span class="sourceLineNo">508</span>    }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span>    public boolean isOrphan() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span>      return (batch == null || batch.isDead);<a name="line.511"></a>
+<span class="sourceLineNo">238</span>    long totalSize = 0;<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    TaskBatch batch = null;<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    long startTime = 0;<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    FileStatus[] logfiles = getFileList(logDirs, filter);<a name="line.241"></a>
+<span class="sourceLineNo">242</span>    if (logfiles.length != 0) {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>      status.setStatus("Checking directory contents...");<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      SplitLogCounters.tot_mgr_log_split_batch_start.increment();<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      LOG.info("Started splitting " + logfiles.length + " logs in " + logDirs +<a name="line.245"></a>
+<span class="sourceLineNo">246</span>          " for " + serverNames);<a name="line.246"></a>
+<span class="sourceLineNo">247</span>      startTime = EnvironmentEdgeManager.currentTime();<a name="line.247"></a>
+<span class="sourceLineNo">248</span>      batch = new TaskBatch();<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      for (FileStatus lf : logfiles) {<a name="line.249"></a>
+<span class="sourceLineNo">250</span>        // TODO If the log file is still being written to - which is most likely<a name="line.250"></a>
+<span class="sourceLineNo">251</span>        // the case for the last log file - then its length will show up here<a name="line.251"></a>
+<span class="sourceLineNo">252</span>        // as zero. The size of such a file can only be retrieved after<a name="line.252"></a>
+<span class="sourceLineNo">253</span>        // recover-lease is done. totalSize will be under in most cases and the<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        // metrics that it drives will also be under-reported.<a name="line.254"></a>
+<span class="sourceLineNo">255</span>        totalSize += lf.getLen();<a name="line.255"></a>
+<span class="sourceLineNo">256</span>        String pathToLog = FSUtils.removeWALRootPath(lf.getPath(), conf);<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        if (!enqueueSplitTask(pathToLog, batch)) {<a name="line.257"></a>
+<span class="sourceLineNo">258</span>          throw new IOException("duplicate log split scheduled for " + lf.getPath());<a name="line.258"></a>
+<span class="sourceLineNo">259</span>        }<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      }<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      waitForSplittingCompletion(batch, status);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    if (batch != null &amp;&amp; batch.done != batch.installed) {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      batch.isDead = true;<a name="line.265"></a>
+<span class="sourceLineNo">266</span>      SplitLogCounters.tot_mgr_log_split_batch_err.increment();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      LOG.warn("error while splitting logs in " + logDirs + " installed = " + batch.installed<a name="line.267"></a>
+<span class="sourceLineNo">268</span>          + " but only " + batch.done + " done");<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      String msg = "error or interrupted while splitting logs in " + logDirs + " Task = " + batch;<a name="line.269"></a>
+<span class="sourceLineNo">270</span>      status.abort(msg);<a name="line.270"></a>
+<span class="sourceLineNo">271</span>      throw new IOException(msg);<a name="line.271"></a>
+<span class="sourceLineNo">272</span>    }<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    for (Path logDir : logDirs) {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      status.setStatus("Cleaning up log directory...");<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      final FileSystem fs = logDir.getFileSystem(conf);<a name="line.275"></a>
+<span class="sourceLineNo">276</span>      try {<a name="line.276"></a>
+<span class="sourceLineNo">277</span>        if (fs.exists(logDir) &amp;&amp; !fs.delete(logDir, false)) {<a name="line.277"></a>
+<span class="sourceLineNo">278</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir);<a name="line.278"></a>
+<span class="sourceLineNo">279</span>        }<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      } catch (IOException ioe) {<a name="line.280"></a>
+<span class="sourceLineNo">281</span>        FileStatus[] files = fs.listStatus(logDir);<a name="line.281"></a>
+<span class="sourceLineNo">282</span>        if (files != null &amp;&amp; files.length &gt; 0) {<a name="line.282"></a>
+<span class="sourceLineNo">283</span>          LOG.warn("Returning success without actually splitting and "<a name="line.283"></a>
+<span class="sourceLineNo">284</span>              + "deleting all the log files in path " + logDir + ": "<a name="line.284"></a>
+<span class="sourceLineNo">285</span>              + Arrays.toString(files), ioe);<a name="line.285"></a>
+<span class="sourceLineNo">286</span>        } else {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>          LOG.warn("Unable to delete log src dir. Ignoring. " + logDir, ioe);<a name="line.287"></a>
+<span class="sourceLineNo">288</span>        }<a name="line.288"></a>
+<span class="sourceLineNo">289</span>      }<a name="line.289"></a>
+<span class="sourceLineNo">290</span>      SplitLogCounters.tot_mgr_log_split_batch_success.increment();<a name="line.290"></a>
+<span class="sourceLineNo">291</span>    }<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    String msg = "Finished splitting (more than or equal to) " + totalSize +<a name="line.292"></a>
+<span class="sourceLineNo">293</span>        " bytes in " + ((batch == null)? 0: batch.installed) +<a name="line.293"></a>
+<span class="sourceLineNo">294</span>        " log files in " + logDirs + " in " +<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        ((startTime == 0)? startTime: (EnvironmentEdgeManager.currentTime() - startTime)) + "ms";<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    status.markComplete(msg);<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    LOG.info(msg);<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    return totalSize;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
+<span class="sourceLineNo">300</span><a name="line.300"></a>
+<span class="sourceLineNo">301</span>  /**<a name="line.301"></a>
+<span class="sourceLineNo">302</span>   * Add a task entry to coordination if it is not already there.<a name="line.302"></a>
+<span class="sourceLineNo">303</span>   * @param taskname the path of the log to be split<a name="line.303"></a>
+<span class="sourceLineNo">304</span>   * @param batch the batch this task belongs to<a name="line.304"></a>
+<span class="sourceLineNo">305</span>   * @return true if a new entry is created, false if it is already there.<a name="line.305"></a>
+<span class="sourceLineNo">306</span>   */<a name="line.306"></a>
+<span class="sourceLineNo">307</span>  boolean enqueueSplitTask(String taskname, TaskBatch batch) {<a name="line.307"></a>
+<span class="sourceLineNo">308</span>    lastTaskCreateTime = EnvironmentEdgeManager.currentTime();<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    String task = getSplitLogManagerCoordination().prepareTask(taskname);<a name="line.309"></a>
+<span class="sourceLineNo">310</span>    Task oldtask = createTaskIfAbsent(task, batch);<a name="line.310"></a>
+<span class="sourceLineNo">311</span>    if (oldtask == null) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span>      // publish the task in the coordination engine<a name="line.312"></a>
+<span class="sourceLineNo">313</span>      getSplitLogManagerCoordination().submitTask(task);<a name="line.313"></a>
+<span class="sourceLineNo">314</span>      return true;<a name="line.314"></a>
+<span class="sourceLineNo">315</span>    }<a name="line.315"></a>
+<span class="sourceLineNo">316</span>    return false;<a name="line.316"></a>
+<span class="sourceLineNo">317</span>  }<a name="line.317"></a>
+<span class="sourceLineNo">318</span><a name="line.318"></a>
+<span class="sourceLineNo">319</span>  private void waitForSplittingCompletion(TaskBatch batch, MonitoredTask status) {<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    synchronized (batch) {<a name="line.320"></a>
+<span class="sourceLineNo">321</span>      while ((batch.done + batch.error) != batch.installed) {<a name="line.321"></a>
+<span class="sourceLineNo">322</span>        try {<a name="line.322"></a>
+<span class="sourceLineNo">323</span>          status.setStatus("Waiting for distributed tasks to finish. " + " scheduled="<a name="line.323"></a>
+<span class="sourceLineNo">324</span>              + batch.installed + " done=" + batch.done + " error=" + batch.error);<a name="line.324"></a>
+<span class="sourceLineNo">325</span>          int remaining = batch.installed - (batch.done + batch.error);<a name="line.325"></a>
+<span class="sourceLineNo">326</span>          int actual = activeTasks(batch);<a name="line.326"></a>
+<span class="sourceLineNo">327</span>          if (remaining != actual) {<a name="line.327"></a>
+<span class="sourceLineNo">328</span>            LOG.warn("Expected " + remaining + " active tasks, but actually there are " + actual);<a name="line.328"></a>
+<span class="sourceLineNo">329</span>          }<a name="line.329"></a>
+<span class="sourceLineNo">330</span>          int remainingTasks = getSplitLogManagerCoordination().remainingTasksInCoordination();<a name="line.330"></a>
+<span class="sourceLineNo">331</span>          if (remainingTasks &gt;= 0 &amp;&amp; actual &gt; remainingTasks) {<a name="line.331"></a>
+<span class="sourceLineNo">332</span>            LOG.warn("Expected at least" + actual + " tasks remaining, but actually there are "<a name="line.332"></a>
+<span class="sourceLineNo">333</span>                + remainingTasks);<a name="line.333"></a>
+<span class="sourceLineNo">334</span>          }<a name="line.334"></a>
+<span class="sourceLineNo">335</span>          if (remainingTasks == 0 || actual == 0) {<a name="line.335"></a>
+<span class="sourceLineNo">336</span>            LOG.warn("No more task remaining, splitting "<a name="line.336"></a>
+<span class="sourceLineNo">337</span>                + "should have completed. Remaining tasks is " + remainingTasks<a name="line.337"></a>
+<span class="sourceLineNo">338</span>                + ", active tasks in map " + actual);<a name="line.338"></a>
+<span class="sourceLineNo">339</span>            if (remainingTasks == 0 &amp;&amp; actual == 0) {<a name="line.339"></a>
+<span class="sourceLineNo">340</span>              return;<a name="line.340"></a>
+<span class="sourceLineNo">341</span>            }<a name="line.341"></a>
+<span class="sourceLineNo">342</span>          }<a name="line.342"></a>
+<span class="sourceLineNo">343</span>          batch.wait(100);<a name="line.343"></a>
+<span class="sourceLineNo">344</span>          if (server.isStopped()) {<a name="line.344"></a>
+<span class="sourceLineNo">345</span>            LOG.warn("Stopped while waiting for log splits to be completed");<a name="line.345"></a>
+<span class="sourceLineNo">346</span>            return;<a name="line.346"></a>
+<span class="sourceLineNo">347</span>          }<a name="line.347"></a>
+<span class="sourceLineNo">348</span>        } catch (InterruptedException e) {<a name="line.348"></a>
+<span class="sourceLineNo">349</span>          LOG.warn("Interrupted while waiting for log splits to be completed");<a name="line.349"></a>
+<span class="sourceLineNo">350</span>          Thread.currentThread().interrupt();<a name="line.350"></a>
+<span class="sourceLineNo">351</span>          return;<a name="line.351"></a>
+<span class="sourceLineNo">352</span>        }<a name="line.352"></a>
+<span class="sourceLineNo">353</span>      }<a name="line.353"></a>
+<span class="sourceLineNo">354</span>    }<a name="line.354"></a>
+<span class="sourceLineNo">355</span>  }<a name="line.355"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span>  @VisibleForTesting<a name="line.357"></a>
+<span class="sourceLineNo">358</span>  ConcurrentMap&lt;String, Task&gt; getTasks() {<a name="line.358"></a>
+<span class="sourceLineNo">359</span>    return tasks;<a name="line.359"></a>
+<span class="sourceLineNo">360</span>  }<a name="line.360"></a>
+<span class="sourceLineNo">361</span><a name="line.361"></a>
+<span class="sourceLineNo">362</span>  private int activeTasks(final TaskBatch batch) {<a name="line.362"></a>
+<span class="sourceLineNo">363</span>    int count = 0;<a name="line.363"></a>
+<span class="sourceLineNo">364</span>    for (Task t : tasks.values()) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span>      if (t.batch == batch &amp;&amp; t.status == TerminationStatus.IN_PROGRESS) {<a name="line.365"></a>
+<span class="sourceLineNo">366</span>        count++;<a name="line.366"></a>
+<span class="sourceLineNo">367</span>      }<a name="line.367"></a>
+<span class="sourceLineNo">368</span>    }<a name="line.368"></a>
+<span class="sourceLineNo">369</span>    return count;<a name="line.369"></a>
+<span class="sourceLineNo">370</span><a name="line.370"></a>
+<span class="sourceLineNo">371</span>  }<a name="line.371"></a>
+<span class="sourceLineNo">372</span><a name="line.372"></a>
+<span class="sourceLineNo">373</span>  /**<a name="line.373"></a>
+<span class="sourceLineNo">374</span>   * @param path<a name="line.374"></a>
+<span class="sourceLineNo">375</span>   * @param batch<a name="line.375"></a>
+<span class="sourceLineNo">376</span>   * @return null on success, existing task on error<a name="line.376"></a>
+<span class="sourceLineNo">377</span>   */<a name="line.377"></a>
+<span class="sourceLineNo">378</span>  private Task createTaskIfAbsent(String path, TaskBatch batch) {<a name="line.378"></a>
+<span class="sourceLineNo">379</span>    Task oldtask;<a name="line.379"></a>
+<span class="sourceLineNo">380</span>    // batch.installed is only changed via this function and<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    // a single thread touches batch.installed.<a name="line.381"></a>
+<span class="sourceLineNo">382</span>    Task newtask = new Task();<a name="line.382"></a>
+<span class="sourceLineNo">383</span>    newtask.batch = batch;<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    oldtask = tasks.putIfAbsent(path, newtask);<a name="line.384"></a>
+<span class="sourceLineNo">385</span>    if (oldtask == null) {<a name="line.385"></a>
+<span class="sourceLineNo">386</span>      batch.installed++;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      return null;<a name="line.387"></a>
+<span class="sourceLineNo">388</span>    }<a name="line.388"></a>
+<span class="sourceLineNo">389</span>    // new task was not used.<a name="line.389"></a>
+<span class="sourceLineNo">390</span>    synchronized (oldtask) {<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      if (oldtask.isOrphan()) {<a name="line.391"></a>
+<span class="sourceLineNo">392</span>        if (oldtask.status == SUCCESS) {<a name="line.392"></a>
+<span class="sourceLineNo">393</span>          // The task is already done. Do not install the batch for this<a name="line.393"></a>
+<span class="sourceLineNo">394</span>          // task because it might be too late for setDone() to update<a name="line.394"></a>
+<span class="sourceLineNo">395</span>          // batch.done. There is no need for the batch creator to wait for<a name="line.395"></a>
+<span class="sourceLineNo">396</span>          // this task to complete.<a name="line.396"></a>
+<span class="sourceLineNo">397</span>          return (null);<a name="line.397"></a>
+<span class="sourceLineNo">398</span>        }<a name="line.398"></a>
+<span class="sourceLineNo">399</span>        if (oldtask.status == IN_PROGRESS) {<a name="line.399"></a>
+<span class="sourceLineNo">400</span>          oldtask.batch = batch;<a name="line.400"></a>
+<span class="sourceLineNo">401</span>          batch.installed++;<a name="line.401"></a>
+<span class="sourceLineNo">402</span>          LOG.debug("Previously orphan task " + path + " is now being waited upon");<a name="line.402"></a>
+<span class="sourceLineNo">403</span>          return null;<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        }<a name="line.404"></a>
+<span class="sourceLineNo">405</span>        while (oldtask.status == FAILURE) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>          LOG.debug("wait for status of task " + path + " to change to DELETED");<a name="line.406"></a>
+<span class="sourceLineNo">407</span>          SplitLogCounters.tot_mgr_wait_for_zk_delete.increment();<a name="line.407"></a>
+<span class="sourceLineNo">408</span>          try {<a name="line.408"></a>
+<span class="sourceLineNo">409</span>            oldtask.wait();<a name="line.409"></a>
+<span class="sourceLineNo">410</span>          } catch (InterruptedException e) {<a name="line.410"></a>
+<span class="sourceLineNo">411</span>            Thread.currentThread().interrupt();<a name="line.411"></a>
+<span class="sourceLineNo">412</span>            LOG.warn("Interrupted when waiting for znode delete callback");<a name="line.412"></a>
+<span class="sourceLineNo">413</span>            // fall through to return failure<a name="line.413"></a>
+<span class="sourceLineNo">414</span>            break;<a name="line.414"></a>
+<span class="sourceLineNo">415</span>          }<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        }<a name="line.416"></a>
+<span class="sourceLineNo">417</span>        if (oldtask.status != DELETED) {<a name="line.417"></a>
+<span class="sourceLineNo">418</span>          LOG.warn("Failure because previously failed task"<a name="line.418"></a>
+<span class="sourceLineNo">419</span>              + " state still present. Waiting for znode delete callback" + " path=" + path);<a name="line.419"></a>
+<span class="sourceLineNo">420</span>          return oldtask;<a name="line.420"></a>
+<span class="sourceLineNo">421</span>        }<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        // reinsert the newTask and it must succeed this time<a name="line.422"></a>
+<span class="sourceLineNo">423</span>        Task t = tasks.putIfAbsent(path, newtask);<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        if (t == null) {<a name="line.424"></a>
+<span class="sourceLineNo">425</span>          batch.installed++;<a name="line.425"></a>
+<span class="sourceLineNo">426</span>          return null;<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        }<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        LOG.error(HBaseMarkers.FATAL, "Logic error. Deleted task still present in tasks map");<a name="line.428"></a>
+<span class="sourceLineNo">429</span>        assert false : "Deleted task still present in tasks map";<a name="line.429"></a>
+<span class="sourceLineNo">430</span>        return t;<a name="line.430"></a>
+<span class="sourceLineNo">431</span>      }<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      LOG.warn("Failure because two threads can't wait for the same task; path=" + path);<a name="line.432"></a>
+<span class="sourceLineNo">433</span>      return oldtask;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>    }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>  }<a name="line.435"></a>
+<span class="sourceLineNo">436</span><a name="line.436"></a>
+<span class="sourceLineNo">437</span>  public void stop() {<a name="line.437"></a>
+<span class="sourceLineNo">438</span>    if (choreService != null) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span>      choreService.shutdown();<a name="line.439"></a>
+<span class="sourceLineNo">440</span>    }<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    if (timeoutMonitor != null) {<a name="line.441"></a>
+<span class="sourceLineNo">442</span>      timeoutMonitor.cancel(true);<a name="line.442"></a>
+<span class="sourceLineNo">443</span>    }<a name="line.443"></a>
+<span class="sourceLineNo">444</span>  }<a name="line.444"></a>
+<span class="sourceLineNo">445</span><a name="line.445"></a>
+<span class="sourceLineNo">446</span>  void handleDeadWorker(ServerName workerName) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span>    // resubmit the tasks on the TimeoutMonitor thread. Makes it easier<a name="line.447"></a>
+<span class="sourceLineNo">448</span>    // to reason about concurrency. Makes it easier to retry.<a name="line.448"></a>
+<span class="sourceLineNo">449</span>    synchronized (deadWorkersLock) {<a name="line.449"></a>
+<span class="sourceLineNo">450</span>      if (deadWorkers == null) {<a name="line.450"></a>
+<span class="sourceLineNo">451</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      }<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      deadWorkers.add(workerName);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    }<a name="line.454"></a>
+<span class="sourceLineNo">455</span>    LOG.info("Dead splitlog worker {}", workerName);<a name="line.455"></a>
+<span class="sourceLineNo">456</span>  }<a name="line.456"></a>
+<span class="sourceLineNo">457</span><a name="line.457"></a>
+<span class="sourceLineNo">458</span>  void handleDeadWorkers(Set&lt;ServerName&gt; serverNames) {<a name="line.458"></a>
+<span class="sourceLineNo">459</span>    synchronized (deadWorkersLock) {<a name="line.459"></a>
+<span class="sourceLineNo">460</span>      if (deadWorkers == null) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>        deadWorkers = new HashSet&lt;&gt;(100);<a name="line.461"></a>
+<span class="sourceLineNo">462</span>      }<a name="line.462"></a>
+<span class="sourceLineNo">463</span>      deadWorkers.addAll(serverNames);<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    }<a name="line.464"></a>
+<span class="sourceLineNo">465</span>    LOG.info("dead splitlog workers " + serverNames);<a name="line.465"></a>
+<span class="sourceLineNo">466</span>  }<a name="line.466"></a>
+<span class="sourceLineNo">467</span><a name="line.467"></a>
+<span class="sourceLineNo">468</span>  /**<a name="line.468"></a>
+<span class="sourceLineNo">469</span>   * Keeps track of the batch of tasks submitted together by a caller in splitLogDistributed().<a name="line.469"></a>
+<span class="sourceLineNo">470</span>   * Clients threads use this object to wait for all their tasks to be done.<a name="line.470"></a>
+<span class="sourceLineNo">471</span>   * &lt;p&gt;<a name="line.471"></a>
+<span class="sourceLineNo">472</span>   * All access is synchronized.<a name="line.472"></a>
+<span class="sourceLineNo">473</span>   */<a name="line.473"></a>
+<span class="sourceLineNo">474</span>  @InterfaceAudience.Private<a name="line.474"></a>
+<span class="sourceLineNo">475</span>  public static class TaskBatch {<a name="line.475"></a>
+<span class="sourceLineNo">476</span>    public int installed = 0;<a name="line.476"></a>
+<span class="sourceLineNo">477</span>    public int done = 0;<a name="line.477"></a>
+<span class="sourceLineNo">478</span>    public int error = 0;<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    public volatile boolean isDead = false;<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span>    @Override<a name="line.481"></a>
+<span class="sourceLineNo">482</span>    public String toString() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>      return ("installed = " + installed + " done = " + done + " error = " + error);<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    }<a name="line.484"></a>
+<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
+<span class="sourceLineNo">486</span><a name="line.486"></a>
+<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
+<span class="sourceLineNo">488</span>   * in memory state of an active task.<a name="line.488"></a>
+<span class="sourceLineNo">489</span>   */<a name="line.489"></a>
+<span class="sourceLineNo">490</span>  @InterfaceAudience.Private<a name="line.490"></a>
+<span class="sourceLineNo">491</span>  public static class Task {<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    public volatile long last_update;<a name="line.492"></a>
+<span class="sourceLineNo">493</span>    public volatile int last_version;<a name="line.493"></a>
+<span class="sourceLineNo">494</span>    public volatile ServerName cur_worker_name;<a name="line.494"></a>
+<span class="sourceLineNo">495</span>    public volatile TaskBatch batch;<a name="line.495"></a>
+<span class="sourceLineNo">496</span>    public volatile TerminationStatus status;<a name="line.496"></a>
+<span class="sourceLineNo">497</span>    public volatile AtomicInteger incarnation = new AtomicInteger(0);<a name="line.497"></a>
+<span class="sourceLineNo">498</span>    public final AtomicInteger unforcedResubmits = new AtomicInteger();<a name="line.498"></a>
+<span class="sourceLineNo">499</span>    public volatile boolean resubmitThresholdReached;<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span>    @Override<a name="line.501"></a>
+<span class="sourceLineNo">502</span>    public String toString() {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>      return ("last_update = " + last_update + " last_version = " + last_version<a name="line.503"></a>
+<span class="sourceLineNo">504</span>          + " cur_worker_name = " + cur_worker_name + " status = " + status + " incarnation = "<a name="line.504"></a>
+<span class="sourceLineNo">505</span>          + incarnation + " resubmits = " + unforcedResubmits.get() + " batch = " + batch);<a name="line.505"></a>
+<span class="sourceLineNo">506</span>    }<a name="line.506"></a>
+<span class="sourceLineNo">507</span><a name="line.507"></a>
+<span class="sourceLineNo">508</span>    public Task() {<a name="line.508"></a>
+<span class="sourceLineNo">509</span>      last_version = -1;<a name="line.509"></a>
+<span class="sourceLineNo">510</span>      status = IN_PROGRESS;<a name="line.510"></a>
+<span class="sourceLineNo">511</span>      setUnassigned();<a name="line.511"></a>
 <span class="sourceLineNo">512</span>    }<a name="line.512"></a>
 <span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span>    public boolean isUnassigned() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span>      return (cur_worker_name == null);<a name="line.515"></a>
+<span class="sourceLineNo">514</span>    public boolean isOrphan() {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>      return (batch == null || batch.isDead);<a name="line.515"></a>
 <span class="sourceLineNo">516</span>    }<a name="line.516"></a>
 <span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span>    public void heartbeatNoDetails(long time) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span>      last_update = time;<a name="line.519"></a>
+<span class="sourceLineNo">518</span>    public boolean isUnassigned() {<a name="line.518"></a>
+<span class="sourceLineNo">519</span>      return (cur_worker_name == null);<a name="line.519"></a>
 <span class="sourceLineNo">520</span>    }<a name="line.520"></a>
 <span class="sourceLineNo">521</span><a name="line.521"></a>
-<span class="sourceLineNo">522</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span>      last_version = version;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>      last_update = time;<a name="line.524"></a>
-<span class="sourceLineNo">525</span>      cur_worker_name = worker;<a name="line.525"></a>
-<span class="sourceLineNo">526</span>    }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span>    public void setUnassigned() {<a name="line.528"></a>
-<span class="sourceLineNo">529</span>      cur_worker_name = null;<a name="line.529"></a>
-<span class="sourceLineNo">530</span>      last_update = -1;<a name="line.530"></a>
-<span class="sourceLineNo">531</span>    }<a name="line.531"></a>
-<span class="sourceLineNo">532</span>  }<a name="line.532"></a>
-<span class="sourceLineNo">533</span><a name="line.533"></a>
-<span class="sourceLineNo">534</span>  /**<a name="line.534"></a>
-<span class="sourceLineNo">535</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.535"></a>
-<span class="sourceLineNo">536</span>   */<a name="line.536"></a>
-<span class="sourceLineNo">537</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.537"></a>
-<span class="sourceLineNo">538</span>    private long lastLog = 0;<a name="line.538"></a>
-<span class="sourceLineNo">539</span><a name="line.539"></a>
-<span class="sourceLineNo">540</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.540"></a>
-<span class="sourceLineNo">541</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.541"></a>
-<span class="sourceLineNo">542</span>    }<a name="line.542"></a>
+<span class="sourceLineNo">522</span>    public void heartbeatNoDetails(long time) {<a name="line.522"></a>
+<span class="sourceLineNo">523</span>      last_update = time;<a name="line.523"></a>
+<span class="sourceLineNo">524</span>    }<a name="line.524"></a>
+<span class="sourceLineNo">525</span><a name="line.525"></a>
+<span class="sourceLineNo">526</span>    public void heartbeat(long time, int version, ServerName worker) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span>      last_version = version;<a name="line.527"></a>
+<span class="sourceLineNo">528</span>      last_update = time;<a name="line.528"></a>
+<span class="sourceLineNo">529</span>      cur_worker_name = worker;<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    }<a name="line.530"></a>
+<span class="sourceLineNo">531</span><a name="line.531"></a>
+<span class="sourceLineNo">532</span>    public void setUnassigned() {<a name="line.532"></a>
+<span class="sourceLineNo">533</span>      cur_worker_name = null;<a name="line.533"></a>
+<span class="sourceLineNo">534</span>      last_update = -1;<a name="line.534"></a>
+<span class="sourceLineNo">535</span>    }<a name="line.535"></a>
+<span class="sourceLineNo">536</span>  }<a name="line.536"></a>
+<span class="sourceLineNo">537</span><a name="line.537"></a>
+<span class="sourceLineNo">538</span>  /**<a name="line.538"></a>
+<span class="sourceLineNo">539</span>   * Periodically checks all active tasks and resubmits the ones that have timed out<a name="line.539"></a>
+<span class="sourceLineNo">540</span>   */<a name="line.540"></a>
+<span class="sourceLineNo">541</span>  private class TimeoutMonitor extends ScheduledChore {<a name="line.541"></a>
+<span class="sourceLineNo">542</span>    private long lastLog = 0;<a name="line.542"></a>
 <span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>    @Override<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    protected void chore() {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.546"></a>
+<span class="sourceLineNo">544</span>    public TimeoutMonitor(final int period, Stoppable stopper) {<a name="line.544"></a>
+<span class="sourceLineNo">545</span>      super("SplitLogManager Timeout Monitor", stopper, period);<a name="line.545"></a>
+<span class="sourceLineNo">546</span>    }<a name="line.546"></a>
 <span class="sourceLineNo">547</span><a name="line.547"></a>
-<span class="sourceLineNo">548</span>      int resubmitted = 0;<a name="line.548"></a>
-<span class="sourceLineNo">549</span>      int unassigned = 0;<a name="line.549"></a>
-<span class="sourceLineNo">550</span>      int tot = 0;<a name="line.550"></a>
-<span class="sourceLineNo">551</span>      boolean found_assigned_task = false;<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.552"></a>
-<span class="sourceLineNo">553</span><a name="line.553"></a>
-<span class="sourceLineNo">554</span>      synchronized (deadWorkersLock) {<a name="line.554"></a>
-<span class="sourceLineNo">555</span>        localDeadWorkers = deadWorkers;<a name="line.555"></a>
-<span class="sourceLineNo">556</span>        deadWorkers = null;<a name="line.556"></a>
-<span class="sourceLineNo">557</span>      }<a name="line.557"></a>
-<span class="sourceLineNo">558</span><a name="line.558"></a>
-<span class="sourceLineNo">559</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>        String path = e.getKey();<a name="line.560"></a>
-<span class="sourceLineNo">561</span>        Task task = e.getValue();<a name="line.561"></a>
-<span class="sourceLineNo">562</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.562"></a>
-<span class="sourceLineNo">563</span>        tot++;<a name="line.563"></a>
-<span class="sourceLineNo">564</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.564"></a>
-<span class="sourceLineNo">565</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.565"></a>
-<span class="sourceLineNo">566</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.566"></a>
-<span class="sourceLineNo">567</span>        // time. If we want progress when there are no region servers then we<a name="line.567"></a>
-<span class="sourceLineNo">568</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.568"></a>
-<span class="sourceLineNo">569</span>        if (task.isUnassigned()) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span>          unassigned++;<a name="line.570"></a>
-<span class="sourceLineNo">571</span>          continue;<a name="line.571"></a>
-<span class="sourceLineNo">572</span>        }<a name="line.572"></a>
-<span class="sourceLineNo">573</span>        found_assigned_task = true;<a name="line.573"></a>
-<span class="sourceLineNo">574</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.575"></a>
-<span class="sourceLineNo">576</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.576"></a>
-<span class="sourceLineNo">577</span>            resubmitted++;<a name="line.577"></a>
-<span class="sourceLineNo">578</span>          } else {<a name="line.578"></a>
-<span class="sourceLineNo">579</span>            handleDeadWorker(cur_worker);<a name="line.579"></a>
-<span class="sourceLineNo">580</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.580"></a>
-<span class="sourceLineNo">581</span>                + ", will retry.");<a name="line.581"></a>
-<span class="sourceLineNo">582</span>          }<a name="line.582"></a>
-<span class="sourceLineNo">583</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.583"></a>
-<span class="sourceLineNo">584</span>          resubmitted++;<a name="line.584"></a>
-<span class="sourceLineNo">585</span>        }<a name="line.585"></a>
-<span class="sourceLineNo">586</span>      }<a name="line.586"></a>
-<span class="sourceLineNo">587</span>      if (tot &gt; 0) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.588"></a>
-<span class="sourceLineNo">589</span>        if (now &gt; lastLog + 5000) {<a name="line.589"></a>
-<span class="sourceLineNo">590</span>          lastLog = now;<a name="line.590"></a>
-<span class="sourceLineNo">591</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.591"></a>
-<span class="sourceLineNo">592</span>        }<a name="line.592"></a>
-<span class="sourceLineNo">593</span>      }<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      if (resubmitted &gt; 0) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.595"></a>
-<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
-<span class="sourceLineNo">597</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.598"></a>
-<span class="sourceLineNo">599</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.599"></a>
-<span class="sourceLineNo">600</span>      // because a. it is very unlikely that every worker had a<a name="line.600"></a>
-<span class="sourceLineNo">601</span>      // transient error when trying to grab the task b. if there are no<a name="line.601"></a>
-<span class="sourceLineNo">602</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.602"></a>
-<span class="sourceLineNo">603</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.603"></a>
-<span class="sourceLineNo">604</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.604"></a>
-<span class="sourceLineNo">605</span>      // that there is always one worker in the system<a name="line.605"></a>
-<span class="sourceLineNo">606</span>      if (tot &gt; 0<a name="line.606"></a>
-<span class="sourceLineNo">607</span>          &amp;&amp; !found_assigned_task<a name="line.607"></a>
-<span class="sourceLineNo">608</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.608"></a>
-<span class="sourceLineNo">609</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.609"></a>
-<span class="sourceLineNo">610</span>          String key = e.getKey();<a name="line.610"></a>
-<span class="sourceLineNo">611</span>          Task task = e.getValue();<a name="line.611"></a>
-<span class="sourceLineNo">612</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.612"></a>
-<span class="sourceLineNo">613</span>          // have been asynchronously assigned. There is no locking required<a name="line.613"></a>
-<span class="sourceLineNo">614</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.614"></a>
-<span class="sourceLineNo">615</span>          // called unnecessarily for a taskpath<a name="line.615"></a>
-<span class="sourceLineNo">616</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.616"></a>
-<span class="sourceLineNo">617</span>            // We just touch the znode to make sure its still there<a name="line.617"></a>
-<span class="sourceLineNo">618</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.618"></a>
-<span class="sourceLineNo">619</span>          }<a name="line.619"></a>
-<span class="sourceLineNo">620</span>        }<a name="line.620"></a>
-<span class="sourceLineNo">621</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.621"></a>
-<span class="sourceLineNo">622</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.623"></a>
-<span class="sourceLineNo">624</span>      }<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      Set&lt;String&gt; failedDeletions =<a name="line.625"></a>
-<span class="sourceLineNo">626</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      // Retry previously failed deletes<a name="line.627"></a>
-<span class="sourceLineNo">628</span>      if (failedDeletions.size() &gt; 0) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.629"></a>
-<span class="sourceLineNo">630</span>        for (String tmpPath : tmpPaths) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>          // deleteNode is an async call<a name="line.631"></a>
-<span class="sourceLineNo">632</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.632"></a>
-<span class="sourceLineNo">633</span>        }<a name="line.633"></a>
-<span class="sourceLineNo">634</span>        failedDeletions.removeAll(tmpPaths);<a name="line.634"></a>
-<span class="sourceLineNo">635</span>      }<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span>  }<a name="line.637"></a>
-<span class="sourceLineNo">638</span><a name="line.638"></a>
-<span class="sourceLineNo">639</span>  public enum ResubmitDirective {<a name="line.639"></a>
-<span class="sourceLineNo">640</span>    CHECK(), FORCE()<a name="line.640"></a>
+<span class="sourceLineNo">548</span>    @Override<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    protected void chore() {<a name="line.549"></a>
+<span class="sourceLineNo">550</span>      if (server.getCoordinatedStateManager() == null) return;<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>      int resubmitted = 0;<a name="line.552"></a>
+<span class="sourceLineNo">553</span>      int unassigned = 0;<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      int tot = 0;<a name="line.554"></a>
+<span class="sourceLineNo">555</span>      boolean found_assigned_task = false;<a name="line.555"></a>
+<span class="sourceLineNo">556</span>      Set&lt;ServerName&gt; localDeadWorkers;<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>      synchronized (deadWorkersLock) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>        localDeadWorkers = deadWorkers;<a name="line.559"></a>
+<span class="sourceLineNo">560</span>        deadWorkers = null;<a name="line.560"></a>
+<span class="sourceLineNo">561</span>      }<a name="line.561"></a>
+<span class="sourceLineNo">562</span><a name="line.562"></a>
+<span class="sourceLineNo">563</span>      for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.563"></a>
+<span class="sourceLineNo">564</span>        String path = e.getKey();<a name="line.564"></a>
+<span class="sourceLineNo">565</span>        Task task = e.getValue();<a name="line.565"></a>
+<span class="sourceLineNo">566</span>        ServerName cur_worker = task.cur_worker_name;<a name="line.566"></a>
+<span class="sourceLineNo">567</span>        tot++;<a name="line.567"></a>
+<span class="sourceLineNo">568</span>        // don't easily resubmit a task which hasn't been picked up yet. It<a name="line.568"></a>
+<span class="sourceLineNo">569</span>        // might be a long while before a SplitLogWorker is free to pick up a<a name="line.569"></a>
+<span class="sourceLineNo">570</span>        // task. This is because a SplitLogWorker picks up a task one at a<a name="line.570"></a>
+<span class="sourceLineNo">571</span>        // time. If we want progress when there are no region servers then we<a name="line.571"></a>
+<span class="sourceLineNo">572</span>        // will have to run a SplitLogWorker thread in the Master.<a name="line.572"></a>
+<span class="sourceLineNo">573</span>        if (task.isUnassigned()) {<a name="line.573"></a>
+<span class="sourceLineNo">574</span>          unassigned++;<a name="line.574"></a>
+<span class="sourceLineNo">575</span>          continue;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>        }<a name="line.576"></a>
+<span class="sourceLineNo">577</span>        found_assigned_task = true;<a name="line.577"></a>
+<span class="sourceLineNo">578</span>        if (localDeadWorkers != null &amp;&amp; localDeadWorkers.contains(cur_worker)) {<a name="line.578"></a>
+<span class="sourceLineNo">579</span>          SplitLogCounters.tot_mgr_resubmit_dead_server_task.increment();<a name="line.579"></a>
+<span class="sourceLineNo">580</span>          if (getSplitLogManagerCoordination().resubmitTask(path, task, FORCE)) {<a name="line.580"></a>
+<span class="sourceLineNo">581</span>            resubmitted++;<a name="line.581"></a>
+<span class="sourceLineNo">582</span>          } else {<a name="line.582"></a>
+<span class="sourceLineNo">583</span>            handleDeadWorker(cur_worker);<a name="line.583"></a>
+<span class="sourceLineNo">584</span>            LOG.warn("Failed to resubmit task " + path + " owned by dead " + cur_worker<a name="line.584"></a>
+<span class="sourceLineNo">585</span>                + ", will retry.");<a name="line.585"></a>
+<span class="sourceLineNo">586</span>          }<a name="line.586"></a>
+<span class="sourceLineNo">587</span>        } else if (getSplitLogManagerCoordination().resubmitTask(path, task, CHECK)) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>          resubmitted++;<a name="line.588"></a>
+<span class="sourceLineNo">589</span>        }<a name="line.589"></a>
+<span class="sourceLineNo">590</span>      }<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      if (tot &gt; 0) {<a name="line.591"></a>
+<span class="sourceLineNo">592</span>        long now = EnvironmentEdgeManager.currentTime();<a name="line.592"></a>
+<span class="sourceLineNo">593</span>        if (now &gt; lastLog + 5000) {<a name="line.593"></a>
+<span class="sourceLineNo">594</span>          lastLog = now;<a name="line.594"></a>
+<span class="sourceLineNo">595</span>          LOG.info("total=" + tot + ", unassigned=" + unassigned + ", tasks=" + tasks);<a name="line.595"></a>
+<span class="sourceLineNo">596</span>        }<a name="line.596"></a>
+<span class="sourceLineNo">597</span>      }<a name="line.597"></a>
+<span class="sourceLineNo">598</span>      if (resubmitted &gt; 0) {<a name="line.598"></a>
+<span class="sourceLineNo">599</span>        LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks");<a name="line.599"></a>
+<span class="sourceLineNo">600</span>      }<a name="line.600"></a>
+<span class="sourceLineNo">601</span>      // If there are pending tasks and all of them have been unassigned for<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      // some time then put up a RESCAN node to ping the workers.<a name="line.602"></a>
+<span class="sourceLineNo">603</span>      // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes<a name="line.603"></a>
+<span class="sourceLineNo">604</span>      // because a. it is very unlikely that every worker had a<a name="line.604"></a>
+<span class="sourceLineNo">605</span>      // transient error when trying to grab the task b. if there are no<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      // workers then all tasks wills stay unassigned indefinitely and the<a name="line.606"></a>
+<span class="sourceLineNo">607</span>      // manager will be indefinitely creating RESCAN nodes. TODO may be the<a name="line.607"></a>
+<span class="sourceLineNo">608</span>      // master should spawn both a manager and a worker thread to guarantee<a name="line.608"></a>
+<span class="sourceLineNo">609</span>      // that there is always one worker in the system<a name="line.609"></a>
+<span class="sourceLineNo">610</span>      if (tot &gt; 0<a name="line.610"></a>
+<span class="sourceLineNo">611</span>          &amp;&amp; !found_assigned_task<a name="line.611"></a>
+<span class="sourceLineNo">612</span>          &amp;&amp; ((EnvironmentEdgeManager.currentTime() - lastTaskCreateTime) &gt; unassignedTimeout)) {<a name="line.612"></a>
+<span class="sourceLineNo">613</span>        for (Map.Entry&lt;String, Task&gt; e : tasks.entrySet()) {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>          String key = e.getKey();<a name="line.614"></a>
+<span class="sourceLineNo">615</span>          Task task = e.getValue();<a name="line.615"></a>
+<span class="sourceLineNo">616</span>          // we have to do task.isUnassigned() check again because tasks might<a name="line.616"></a>
+<span class="sourceLineNo">617</span>          // have been asynchronously assigned. There is no locking required<a name="line.617"></a>
+<span class="sourceLineNo">618</span>          // for these checks ... it is OK even if tryGetDataSetWatch() is<a name="line.618"></a>
+<span class="sourceLineNo">619</span>          // called unnecessarily for a taskpath<a name="line.619"></a>
+<span class="sourceLineNo">620</span>          if (task.isUnassigned() &amp;&amp; (task.status != FAILURE)) {<a name="line.620"></a>
+<span class="sourceLineNo">621</span>            // We just touch the znode to make sure its still there<a name="line.621"></a>
+<span class="sourceLineNo">622</span>            getSplitLogManagerCoordination().checkTaskStillAvailable(key);<a name="line.622"></a>
+<span class="sourceLineNo">623</span>          }<a name="line.623"></a>
+<span class="sourceLineNo">624</span>        }<a name="line.624"></a>
+<span class="sourceLineNo">625</span>        getSplitLogManagerCoordination().checkTasks();<a name="line.625"></a>
+<span class="sourceLineNo">626</span>        SplitLogCounters.tot_mgr_resubmit_unassigned.increment();<a name="line.626"></a>
+<span class="sourceLineNo">627</span>        LOG.debug("resubmitting unassigned task(s) after timeout");<a name="line.627"></a>
+<span class="sourceLineNo">628</span>      }<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      Set&lt;String&gt; failedDeletions =<a name="line.629"></a>
+<span class="sourceLineNo">630</span>        getSplitLogManagerCoordination().getDetails().getFailedDeletions();<a name="line.630"></a>
+<span class="sourceLineNo">631</span>      // Retry previously failed deletes<a name="line.631"></a>
+<span class="sourceLineNo">632</span>      if (failedDeletions.size() &gt; 0) {<a name="line.632"></a>
+<span class="sourceLineNo">633</span>        List&lt;String&gt; tmpPaths = new ArrayList&lt;&gt;(failedDeletions);<a name="line.633"></a>
+<span class="sourceLineNo">634</span>        for (String tmpPath : tmpPaths) {<a name="line.634"></a>
+<span class="sourceLineNo">635</span>          // deleteNode is an async call<a name="line.635"></a>
+<span class="sourceLineNo">636</span>          getSplitLogManagerCoordination().deleteTask(tmpPath);<a name="line.636"></a>
+<span class="sourceLineNo">637</span>        }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>        failedDeletions.removeAll(tmpPaths);<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      }<a name="line.639"></a>
+<span class="sourceLineNo">640</span>    }<a name="line.640"></a>
 <span class="sourceLineNo">641</span>  }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>  public enum TerminationStatus {<a name="line.643"></a>
-<span class="sourceLineNo">644</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.644"></a>
-<span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    final String statusMsg;<a name="line.646"></a>
-<span class="sourceLineNo">647</span><a name="line.647"></a>
-<span class="sourceLineNo">648</span>    TerminationStatus(String msg) {<a name="line.648"></a>
-<span class="sourceLineNo">649</span>      statusMsg = msg;<a name="line.649"></a>
-<span class="sourceLineNo">650</span>    }<a name="line.650"></a>
+<span class="sourceLineNo">643</span>  public enum ResubmitDirective {<a name="line.643"></a>
+<span class="sourceLineNo">644</span>    CHECK(), FORCE()<a name="line.644"></a>
+<span class="sourceLineNo">645</span>  }<a name="line.645"></a>
+<span class="sourceLineNo">646</span><a name="line.646"></a>
+<span class="sourceLineNo">647</span>  public enum TerminationStatus {<a name="line.647"></a>
+<span class="sourceLineNo">648</span>    IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");<a name="line.648"></a>
+<span class="sourceLineNo">649</span><a name="line.649"></a>
+<span class="sourceLineNo">650</span>    final String statusMsg;<a name="line.650"></a>
 <span class="sourceLineNo">651</span><a name="line.651"></a>
-<span class="sourceLineNo">652</span>    @Override<a name="line.652"></a>
-<span class="sourceLineNo">653</span>    public String toString() {<a name="line.653"></a>
-<span class="sourceLineNo">654</span>      return statusMsg;<a name="line.654"></a>
-<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>  }<a name="line.656"></a>
-<span class="sourceLineNo">657</span>}<a name="line.657"></a>
+<span class="sourceLineNo">652</span>    TerminationStatus(String msg) {<a name="line.652"></a>
+<span class="sourceLineNo">653</span>      statusMsg = msg;<a name="line.653"></a>
+<span class="sourceLineNo">654</span>    }<a name="line.654"></a>
+<span class="sourceLineNo">655</span><a name="line.655"></a>
+<span class="sourceLineNo">656</span>    @Override<a name="line.656"></a>
+<span class="sourceLineNo">657</span>    public String toString() {<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      return statusMsg;<a name="line.658"></a>
+<span class="sourceLineNo">659</span>    }<a name="line.659"></a>
+<span class="sourceLineNo">660</span>  }<a name="line.660"></a>
+<span class="sourceLineNo">661</span>}<a name="line.661"></a>