You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2021/02/07 20:18:10 UTC
[hbase-site] branch asf-site updated: Published site at
d6aff6cbae5157b99c3e1c83472c7d3243a131db.
This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hbase-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 2460ecf Published site at d6aff6cbae5157b99c3e1c83472c7d3243a131db.
2460ecf is described below
commit 2460ecf1511391c1e9484637c85ad58c6f601934
Author: jenkins <bu...@apache.org>
AuthorDate: Sun Feb 7 20:18:00 2021 +0000
Published site at d6aff6cbae5157b99c3e1c83472c7d3243a131db.
---
acid-semantics.html | 2 +-
apache_hbase_reference_guide.pdf | 4 +-
book.html | 2 +-
bulk-loads.html | 2 +-
checkstyle-aggregate.html | 4 +-
coc.html | 2 +-
dependencies.html | 2 +-
dependency-convergence.html | 2 +-
dependency-info.html | 2 +-
dependency-management.html | 2 +-
.../apache/hadoop/hbase/class-use/ServerName.html | 22 +
.../hbase/replication/ReplicationTracker.html | 10 +-
...tionTrackerZKImpl.OtherRegionServerWatcher.html | 12 +-
.../replication/ReplicationTrackerZKImpl.html | 34 +-
...eplicationSourceManager.NodeFailoverWorker.html | 12 +-
...ionSourceManager.ReplicationQueueOperation.html | 4 +-
.../regionserver/ReplicationSourceManager.html | 90 +-
.../hbase/replication/ReplicationTracker.html | 55 +-
...tionTrackerZKImpl.OtherRegionServerWatcher.html | 350 ++--
.../replication/ReplicationTrackerZKImpl.html | 350 ++--
.../DumpReplicationQueues.DumpOptions.html | 4 +-
.../DumpReplicationQueues.WarnOnlyAbortable.html | 4 +-
.../DumpReplicationQueues.WarnOnlyStoppable.html | 4 +-
.../regionserver/DumpReplicationQueues.html | 4 +-
...eplicationSourceManager.NodeFailoverWorker.html | 1915 ++++++++++----------
...ionSourceManager.ReplicationQueueOperation.html | 1915 ++++++++++----------
.../regionserver/ReplicationSourceManager.html | 1915 ++++++++++----------
downloads.html | 2 +-
export_control.html | 2 +-
index.html | 2 +-
issue-tracking.html | 2 +-
mail-lists.html | 2 +-
metrics.html | 2 +-
old_news.html | 2 +-
plugin-management.html | 2 +-
plugins.html | 2 +-
poweredbyhbase.html | 2 +-
project-info.html | 2 +-
project-reports.html | 2 +-
project-summary.html | 2 +-
pseudo-distributed.html | 2 +-
replication.html | 2 +-
resources.html | 2 +-
source-repository.html | 2 +-
sponsors.html | 2 +-
supportingprojects.html | 2 +-
team-list.html | 2 +-
...tionTrackerZKImpl.DummyReplicationListener.html | 18 +-
.../TestReplicationTrackerZKImpl.DummyServer.html | 18 +-
.../replication/TestReplicationTrackerZKImpl.html | 18 +-
50 files changed, 3423 insertions(+), 3395 deletions(-)
diff --git a/acid-semantics.html b/acid-semantics.html
index e33b793..6050935 100644
--- a/acid-semantics.html
+++ b/acid-semantics.html
@@ -467,7 +467,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf
index cf8618d..890d2ba 100644
--- a/apache_hbase_reference_guide.pdf
+++ b/apache_hbase_reference_guide.pdf
@@ -5,8 +5,8 @@
/Author (Apache HBase Team)
/Creator (Asciidoctor PDF 1.5.3, based on Prawn 2.2.2)
/Producer (Apache HBase Team)
-/ModDate (D:20210205200631+00'00')
-/CreationDate (D:20210205201836+00'00')
+/ModDate (D:20210207200625+00'00')
+/CreationDate (D:20210207201609+00'00')
>>
endobj
2 0 obj
diff --git a/book.html b/book.html
index 58f806d..1b0a4c3 100644
--- a/book.html
+++ b/book.html
@@ -46091,7 +46091,7 @@ org/apache/hadoop/hbase/security/access/AccessControlClient.revoke:(Lorg/apache/
<div id="footer">
<div id="footer-text">
Version 3.0.0-SNAPSHOT<br>
-Last updated 2021-02-05 20:06:31 UTC
+Last updated 2021-02-07 20:06:25 UTC
</div>
</div>
<link rel="stylesheet" href="./coderay-asciidoctor.css">
diff --git a/bulk-loads.html b/bulk-loads.html
index 2639020..7fd3098 100644
--- a/bulk-loads.html
+++ b/bulk-loads.html
@@ -172,7 +172,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html
index 304fcb9..34ff918 100644
--- a/checkstyle-aggregate.html
+++ b/checkstyle-aggregate.html
@@ -58397,7 +58397,7 @@
<td>javadoc</td>
<td>JavadocTagContinuationIndentation</td>
<td>Line continuation have incorrect indentation level, expected level should be 2.</td>
-<td>1204</td></tr></table></div>
+<td>1203</td></tr></table></div>
<div class="section">
<h3 id="org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.java">org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java</h3>
<table border="0" class="table table-striped">
@@ -75575,7 +75575,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/coc.html b/coc.html
index b3deaeb..ca1a8f6 100644
--- a/coc.html
+++ b/coc.html
@@ -241,7 +241,7 @@ email to <a class="externalLink" href="mailto:private@hbase.apache.org">the priv
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/dependencies.html b/dependencies.html
index 840259a..a1685d0 100644
--- a/dependencies.html
+++ b/dependencies.html
@@ -313,7 +313,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/dependency-convergence.html b/dependency-convergence.html
index 4ad39e0..6c6fe2e 100644
--- a/dependency-convergence.html
+++ b/dependency-convergence.html
@@ -824,7 +824,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/dependency-info.html b/dependency-info.html
index 1744585..88b3543 100644
--- a/dependency-info.html
+++ b/dependency-info.html
@@ -194,7 +194,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/dependency-management.html b/dependency-management.html
index ff85310..2592bd4 100644
--- a/dependency-management.html
+++ b/dependency-management.html
@@ -1019,7 +1019,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/devapidocs/org/apache/hadoop/hbase/class-use/ServerName.html b/devapidocs/org/apache/hadoop/hbase/class-use/ServerName.html
index e5cc40e..600ba6f 100644
--- a/devapidocs/org/apache/hadoop/hbase/class-use/ServerName.html
+++ b/devapidocs/org/apache/hadoop/hbase/class-use/ServerName.html
@@ -5797,6 +5797,10 @@ service.</div>
</tr>
<tr class="altColor">
<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
+<td class="colLast"><span class="typeNameLabel">ReplicationTrackerZKImpl.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#otherRegionServers">otherRegionServers</a></span></code> </td>
+</tr>
+<tr class="rowColor">
+<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
<td class="colLast"><span class="typeNameLabel">HBaseReplicationEndpoint.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.html#sinkServers">sinkServers</a></span></code> </td>
</tr>
</tbody>
@@ -5833,6 +5837,18 @@ service.</div>
</tr>
<tr class="altColor">
<td class="colFirst"><code><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
+<td class="colLast"><span class="typeNameLabel">ReplicationTrackerZKImpl.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#getListOfRegionServers--">getListOfRegionServers</a></span>()</code>
+<div class="block">Return a snapshot of the current region servers.</div>
+</td>
+</tr>
+<tr class="rowColor">
+<td class="colFirst"><code><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
+<td class="colLast"><span class="typeNameLabel">ReplicationTracker.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTracker.html#getListOfRegionServers--">getListOfRegionServers</a></span>()</code>
+<div class="block">Returns a list of other live region servers in the cluster.</div>
+</td>
+</tr>
+<tr class="altColor">
+<td class="colFirst"><code><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
<td class="colLast"><span class="typeNameLabel">ZKReplicationQueueStorage.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.html#getListOfReplicators--">getListOfReplicators</a></span>()</code> </td>
</tr>
<tr class="rowColor">
@@ -5846,6 +5862,12 @@ service.</div>
<td class="colLast"><span class="typeNameLabel">ZKReplicationQueueStorage.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.html#getListOfReplicators0--">getListOfReplicators0</a></span>()</code> </td>
</tr>
<tr class="rowColor">
+<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
+<td class="colLast"><span class="typeNameLabel">ReplicationTrackerZKImpl.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#getRegisteredRegionServers-boolean-">getRegisteredRegionServers</a></span>(boolean watch)</code>
+<div class="block">Get a list of all the other region servers in this cluster and set a watch</div>
+</td>
+</tr>
+<tr class="altColor">
<td class="colFirst"><code>(package private) <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
<td class="colLast"><span class="typeNameLabel">HBaseReplicationEndpoint.</span><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.html#getSinkServers--">getSinkServers</a></span>()</code> </td>
</tr>
diff --git a/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTracker.html b/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTracker.html
index 0bcaf08..d3fb8ce 100644
--- a/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTracker.html
+++ b/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTracker.html
@@ -106,7 +106,7 @@ var activeTableTab = "activeTableTab";
<hr>
<br>
<pre>@InterfaceAudience.Private
-public interface <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html#line.34">ReplicationTracker</a></pre>
+public interface <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html#line.35">ReplicationTracker</a></pre>
<div class="block">This is the interface for a Replication Tracker. A replication tracker provides the facility to
subscribe and track events that reflect a change in replication state. These events are used by
the ReplicationSourceManager to coordinate replication tasks such as addition/deletion of queues
@@ -132,7 +132,7 @@ public interface <a href="../../../../../src-html/org/apache/hadoop/hbase/replic
<th class="colLast" scope="col">Method and Description</th>
</tr>
<tr id="i0" class="altColor">
-<td class="colFirst"><code><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>></code></td>
+<td class="colFirst"><code><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTracker.html#getListOfRegionServers--">getListOfRegionServers</a></span>()</code>
<div class="block">Returns a list of other live region servers in the cluster.</div>
</td>
@@ -168,7 +168,7 @@ public interface <a href="../../../../../src-html/org/apache/hadoop/hbase/replic
<ul class="blockList">
<li class="blockList">
<h4>registerListener</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html#line.40">registerListener</a>(<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a> listener)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html#line.41">registerListener</a>(<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a> listener)</pre>
<div class="block">Register a replication listener to receive replication events.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -182,7 +182,7 @@ public interface <a href="../../../../../src-html/org/apache/hadoop/hbase/replic
<ul class="blockList">
<li class="blockList">
<h4>removeListener</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html#line.42">removeListener</a>(<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a> listener)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html#line.43">removeListener</a>(<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a> listener)</pre>
</li>
</ul>
<a name="getListOfRegionServers--">
@@ -191,7 +191,7 @@ public interface <a href="../../../../../src-html/org/apache/hadoop/hbase/replic
<ul class="blockListLast">
<li class="blockList">
<h4>getListOfRegionServers</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html#line.48">getListOfRegionServers</a>()</pre>
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html#line.49">getListOfRegionServers</a>()</pre>
<div class="block">Returns a list of other live region servers in the cluster.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
diff --git a/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html b/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html
index 960b458..ec93edc 100644
--- a/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html
+++ b/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html
@@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public class <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.91">ReplicationTrackerZKImpl.OtherRegionServerWatcher</a>
+<pre>public class <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.94">ReplicationTrackerZKImpl.OtherRegionServerWatcher</a>
extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKListener</a></pre>
<div class="block">Watcher used to be notified of the other region server's death in the local cluster. It
initiates the process to transfer the queues if it is able to grab the lock.</div>
@@ -231,7 +231,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.htm
<ul class="blockListLast">
<li class="blockList">
<h4>OtherRegionServerWatcher</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.96">OtherRegionServerWatcher</a>(<a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a> watcher)</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.99">OtherRegionServerWatcher</a>(<a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a> watcher)</pre>
<div class="block">Construct a ZooKeeper event listener.</div>
</li>
</ul>
@@ -249,7 +249,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.htm
<ul class="blockList">
<li class="blockList">
<h4>nodeCreated</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.105">nodeCreated</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> path)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.108">nodeCreated</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> path)</pre>
<div class="block">Called when a new node has been created.</div>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
@@ -265,7 +265,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.htm
<ul class="blockList">
<li class="blockList">
<h4>nodeDeleted</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.114">nodeDeleted</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> path)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.117">nodeDeleted</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> path)</pre>
<div class="block">Called when a node has been deleted</div>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
@@ -281,7 +281,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.htm
<ul class="blockList">
<li class="blockList">
<h4>nodeChildrenChanged</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.133">nodeChildrenChanged</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> path)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.136">nodeChildrenChanged</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> path)</pre>
<div class="block">Called when an existing node has a child node added or removed.</div>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
@@ -297,7 +297,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKListener.htm
<ul class="blockListLast">
<li class="blockList">
<h4>refreshListIfRightPath</h4>
-<pre>private boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.140">refreshListIfRightPath</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> path)</pre>
+<pre>private boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html#line.143">refreshListIfRightPath</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> path)</pre>
</li>
</ul>
</li>
diff --git a/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html b/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html
index 85a1432..86bd44f 100644
--- a/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html
+++ b/devapidocs/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html
@@ -114,7 +114,7 @@ var activeTableTab = "activeTableTab";
<hr>
<br>
<pre>@InterfaceAudience.Private
-public class <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.39">ReplicationTrackerZKImpl</a>
+public class <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.42">ReplicationTrackerZKImpl</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a>
implements <a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTracker.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationTracker</a></pre>
<div class="block">This class is a ZooKeeper implementation of the ReplicationTracker interface. This class is
@@ -172,7 +172,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#LOG">LOG</a></span></code> </td>
</tr>
<tr class="rowColor">
-<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>></code></td>
+<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#otherRegionServers">otherRegionServers</a></span></code> </td>
</tr>
<tr class="altColor">
@@ -218,13 +218,13 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<th class="colLast" scope="col">Method and Description</th>
</tr>
<tr id="i0" class="altColor">
-<td class="colFirst"><code><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>></code></td>
+<td class="colFirst"><code><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#getListOfRegionServers--">getListOfRegionServers</a></span>()</code>
<div class="block">Return a snapshot of the current region servers.</div>
</td>
</tr>
<tr id="i1" class="rowColor">
-<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>></code></td>
+<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#getRegisteredRegionServers-boolean-">getRegisteredRegionServers</a></span>(boolean watch)</code>
<div class="block">Get a list of all the other region servers in this cluster and set a watch</div>
</td>
@@ -280,7 +280,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>LOG</h4>
-<pre>private static final org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.41">LOG</a></pre>
+<pre>private static final org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.44">LOG</a></pre>
</li>
</ul>
<a name="zookeeper">
@@ -289,7 +289,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>zookeeper</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.44">zookeeper</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.47">zookeeper</a></pre>
</li>
</ul>
<a name="abortable">
@@ -298,7 +298,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>abortable</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/Abortable.html" title="interface in org.apache.hadoop.hbase">Abortable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.46">abortable</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/Abortable.html" title="interface in org.apache.hadoop.hbase">Abortable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.49">abortable</a></pre>
</li>
</ul>
<a name="stopper">
@@ -307,7 +307,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>stopper</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/Stoppable.html" title="interface in org.apache.hadoop.hbase">Stoppable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.48">stopper</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/Stoppable.html" title="interface in org.apache.hadoop.hbase">Stoppable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.51">stopper</a></pre>
</li>
</ul>
<a name="listeners">
@@ -316,7 +316,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>listeners</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.50">listeners</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.53">listeners</a></pre>
</li>
</ul>
<a name="otherRegionServers">
@@ -325,7 +325,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockListLast">
<li class="blockList">
<h4>otherRegionServers</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.52">otherRegionServers</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.55">otherRegionServers</a></pre>
</li>
</ul>
</li>
@@ -342,7 +342,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockListLast">
<li class="blockList">
<h4>ReplicationTrackerZKImpl</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.54">ReplicationTrackerZKImpl</a>(<a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a> zookeeper,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.57">ReplicationTrackerZKImpl</a>(<a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a> zookeeper,
<a href="../../../../../org/apache/hadoop/hbase/Abortable.html" title="interface in org.apache.hadoop.hbase">Abortable</a> abortable,
<a href="../../../../../org/apache/hadoop/hbase/Stoppable.html" title="interface in org.apache.hadoop.hbase">Stoppable</a> stopper)</pre>
</li>
@@ -361,7 +361,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>registerListener</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.64">registerListener</a>(<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a> listener)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.67">registerListener</a>(<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a> listener)</pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from interface: <code><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTracker.html#registerListener-org.apache.hadoop.hbase.replication.ReplicationListener-">ReplicationTracker</a></code></span></div>
<div class="block">Register a replication listener to receive replication events.</div>
<dl>
@@ -378,7 +378,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>removeListener</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.69">removeListener</a>(<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a> listener)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.72">removeListener</a>(<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationListener</a> listener)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTracker.html#removeListener-org.apache.hadoop.hbase.replication.ReplicationListener-">removeListener</a></code> in interface <code><a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationTracker.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationTracker</a></code></dd>
@@ -391,7 +391,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>getListOfRegionServers</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.77">getListOfRegionServers</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.80">getListOfRegionServers</a>()</pre>
<div class="block">Return a snapshot of the current region servers.</div>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
@@ -407,7 +407,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>getZNodeName</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.153">getZNodeName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fullPath)</pre>
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.156">getZNodeName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fullPath)</pre>
<div class="block">Extracts the znode name of a peer cluster from a ZK path</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -423,7 +423,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockList">
<li class="blockList">
<h4>refreshOtherRegionServersList</h4>
-<pre>private boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.164">refreshOtherRegionServersList</a>(boolean watch)</pre>
+<pre>private boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.167">refreshOtherRegionServersList</a>(boolean watch)</pre>
<div class="block">Reads the list of region servers from ZK and atomically clears our local view of it and
replaces it with the updated list.</div>
<dl>
@@ -439,7 +439,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/replication/Replicati
<ul class="blockListLast">
<li class="blockList">
<h4>getRegisteredRegionServers</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.181">getRegisteredRegionServers</a>(boolean watch)</pre>
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html#line.184">getRegisteredRegionServers</a>(boolean watch)</pre>
<div class="block">Get a list of all the other region servers in this cluster and set a watch</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
diff --git a/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html b/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html
index c544dd6..8583007 100644
--- a/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html
+++ b/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>class <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.865">ReplicationSourceManager.NodeFailoverWorker</a>
+<pre>class <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.864">ReplicationSourceManager.NodeFailoverWorker</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html?is-external=true" title="class or interface in java.lang">Thread</a></pre>
<div class="block">Class responsible to setup new ReplicationSources to take care of the queues from dead region
servers.</div>
@@ -250,7 +250,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockList">
<li class="blockList">
<h4>deadRS</h4>
-<pre>private final <a href="../../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html#line.867">deadRS</a></pre>
+<pre>private final <a href="../../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html#line.866">deadRS</a></pre>
</li>
</ul>
<a name="peersSnapshot">
@@ -259,7 +259,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockListLast">
<li class="blockList">
<h4>peersSnapshot</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="../../../../../../org/apache/hadoop/hbase/replication/ReplicationPeerImpl.html" title="class in org.apache.hadoop.hbase.replication">ReplicationPeerImpl</a>> <a href="../../.. [...]
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="../../../../../../org/apache/hadoop/hbase/replication/ReplicationPeerImpl.html" title="class in org.apache.hadoop.hbase.replication">ReplicationPeerImpl</a>> <a href="../../.. [...]
</li>
</ul>
</li>
@@ -276,7 +276,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockListLast">
<li class="blockList">
<h4>NodeFailoverWorker</h4>
-<pre>public <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html#line.877">NodeFailoverWorker</a>(<a href="../../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a> deadRS)</pre>
+<pre>public <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html#line.876">NodeFailoverWorker</a>(<a href="../../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a> deadRS)</pre>
</li>
</ul>
</li>
@@ -293,7 +293,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockList">
<li class="blockList">
<h4>isOldPeer</h4>
-<pre>private boolean <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html#line.883">isOldPeer</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId,
+<pre>private boolean <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html#line.882">isOldPeer</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId,
<a href="../../../../../../org/apache/hadoop/hbase/replication/ReplicationPeerImpl.html" title="class in org.apache.hadoop.hbase.replication">ReplicationPeerImpl</a> newPeerRef)</pre>
</li>
</ul>
@@ -303,7 +303,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockListLast">
<li class="blockList">
<h4>run</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html#line.889">run</a>()</pre>
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html#line.888">run</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true#run--" title="class or interface in java.lang">run</a></code> in interface <code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true" title="class or interface in java.lang">Runnable</a></code></dd>
diff --git a/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html b/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html
index 9cc3c85..ea637a2 100644
--- a/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html
+++ b/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html
@@ -110,7 +110,7 @@ var activeTableTab = "activeTableTab";
<hr>
<br>
<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/FunctionalInterface.html?is-external=true" title="class or interface in java.lang">@FunctionalInterface</a>
-private static interface <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.601">ReplicationSourceManager.ReplicationQueueOperation</a></pre>
+private static interface <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.600">ReplicationSourceManager.ReplicationQueueOperation</a></pre>
</li>
</ul>
</div>
@@ -154,7 +154,7 @@ private static interface <a href="../../../../../../src-html/org/apache/hadoop/h
<ul class="blockListLast">
<li class="blockList">
<h4>exec</h4>
-<pre>void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html#line.602">exec</a>()
+<pre>void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html#line.601">exec</a>()
throws <a href="../../../../../../org/apache/hadoop/hbase/replication/ReplicationException.html" title="class in org.apache.hadoop.hbase.replication">ReplicationException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
diff --git a/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html b/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html
index 78130f9..b34dbef 100644
--- a/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html
+++ b/devapidocs/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html
@@ -976,7 +976,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>addPeer</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.307">addPeer</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId)
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.306">addPeer</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block"><ol>
<li>Add peer to replicationPeers</li>
@@ -997,7 +997,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>removePeer</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.331">removePeer</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId)</pre>
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.330">removePeer</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId)</pre>
<div class="block"><ol>
<li>Remove peer for replicationPeers</li>
<li>Remove all the recovered sources for the specified id and related replication queues</li>
@@ -1016,7 +1016,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>createSource</h4>
-<pre>private <a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.376">createSource</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface [...]
+<pre>private <a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.375">createSource</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface [...]
<a href="../../../../../../org/apache/hadoop/hbase/replication/ReplicationPeer.html" title="interface in org.apache.hadoop.hbase.replication">ReplicationPeer</a> replicationPeer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
@@ -1037,7 +1037,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>addSource</h4>
-<pre><a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.397">addSource</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">St [...]
+<pre><a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.396">addSource</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">St [...]
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Add a normal source for the given peer on this region server. Meanwhile, add new replication
queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal
@@ -1058,7 +1058,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>drainSources</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.443">drainSources</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId)
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.442">drainSources</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
<a href="../../../../../../org/apache/hadoop/hbase/replication/ReplicationException.html" title="class in org.apache.hadoop.hbase.replication">ReplicationException</a></pre>
<div class="block"><p>
@@ -1088,7 +1088,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>refreshSources</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.507">refreshSources</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId)
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.506">refreshSources</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Close the previous replication sources of this peer id and open new sources to trigger the new
replication state changes or new replication config changes. Here we don't need to change
@@ -1107,7 +1107,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>removeRecoveredSource</h4>
-<pre>private boolean <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.558">removeRecoveredSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> src)</pre>
+<pre>private boolean <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.557">removeRecoveredSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> src)</pre>
<div class="block">Clear the metrics and related replication queue of the specified old source</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1121,7 +1121,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>finishRecoveredSource</h4>
-<pre>void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.569">finishRecoveredSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> src)</pre>
+<pre>void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.568">finishRecoveredSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> src)</pre>
</li>
</ul>
<a name="removeSource-org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface-">
@@ -1130,7 +1130,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>removeSource</h4>
-<pre>void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.583">removeSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> src)</pre>
+<pre>void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.582">removeSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> src)</pre>
<div class="block">Clear the metrics and related replication queue of the specified old source</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1144,7 +1144,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>deleteQueue</h4>
-<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.596">deleteQueue</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> queueId)</pre>
+<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.595">deleteQueue</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> queueId)</pre>
<div class="block">Delete a complete queue of wals associated with a replication source</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1158,7 +1158,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>interruptOrAbortWhenFail</h4>
-<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.609">interruptOrAbortWhenFail</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceManager.ReplicationQueueOperation</a> op)</pre>
+<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.608">interruptOrAbortWhenFail</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceManager.ReplicationQueueOperation</a> op)</pre>
<div class="block">Refresh replication source will terminate the old source first, then the source thread will be
interrupted. Need to handle it instead of abort the region server.</div>
</li>
@@ -1169,7 +1169,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>abortWhenFail</h4>
-<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.628">abortWhenFail</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceManager.ReplicationQueueOperation</a> op)</pre>
+<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.627">abortWhenFail</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceManager.ReplicationQueueOperation</a> op)</pre>
</li>
</ul>
<a name="throwIOExceptionWhenFail-org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.ReplicationQueueOperation-">
@@ -1178,7 +1178,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>throwIOExceptionWhenFail</h4>
-<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.636">throwIOExceptionWhenFail</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceManager.ReplicationQueueOperation</a> op)
+<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.635">throwIOExceptionWhenFail</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceManager.ReplicationQueueOperation</a> op)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -1192,7 +1192,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>abortAndThrowIOExceptionWhenFail</h4>
-<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.644">abortAndThrowIOExceptionWhenFail</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceManager.ReplicationQueueOperation</a> op)
+<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.643">abortAndThrowIOExceptionWhenFail</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceManager.ReplicationQueueOperation</a> op)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -1206,7 +1206,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>logPositionAndCleanOldLogs</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.659">logPositionAndCleanOldLogs</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> source,
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.658">logPositionAndCleanOldLogs</a>(<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> source,
<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/WALEntryBatch.html" title="class in org.apache.hadoop.hbase.replication.regionserver">WALEntryBatch</a> entryBatch)</pre>
<div class="block">This method will log the current position to storage. And also clean old logs from the
replication queue.</div>
@@ -1223,7 +1223,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>cleanOldLogs</h4>
-<pre>void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.674">cleanOldLogs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> log,
+<pre>void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.673">cleanOldLogs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> log,
boolean inclusive,
<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> source)</pre>
<div class="block">Cleans a log file and all older logs from replication queue. Called when we are sure that a log
@@ -1242,7 +1242,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>removeRemoteWALs</h4>
-<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.712">removeRemoteWALs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId,
+<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.711">removeRemoteWALs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> remoteWALDir,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Collection.html?is-external=true" title="class or interface in java.util">Collection</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> wals)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -1258,7 +1258,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>cleanOldLogs</h4>
-<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.732">cleanOldLogs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/NavigableSet.html?is-external=true" title="class or interface in java.util">NavigableSet</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> wals,
+<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.731">cleanOldLogs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/NavigableSet.html?is-external=true" title="class or interface in java.util">NavigableSet</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> wals,
<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> source)</pre>
</li>
</ul>
@@ -1268,7 +1268,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>preLogRoll</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.777">preLogRoll</a>(org.apache.hadoop.fs.Path newLog)
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.776">preLogRoll</a>(org.apache.hadoop.fs.Path newLog)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -1282,7 +1282,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>postLogRoll</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.826">postLogRoll</a>(org.apache.hadoop.fs.Path newLog)
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.825">postLogRoll</a>(org.apache.hadoop.fs.Path newLog)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -1296,7 +1296,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>regionServerRemoved</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.836">regionServerRemoved</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> regionserver)</pre>
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.835">regionServerRemoved</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> regionserver)</pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from interface: <code><a href="../../../../../../org/apache/hadoop/hbase/replication/ReplicationListener.html#regionServerRemoved-java.lang.String-">ReplicationListener</a></code></span></div>
<div class="block">A region server has been removed from the local cluster</div>
<dl>
@@ -1313,7 +1313,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>transferQueues</h4>
-<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.846">transferQueues</a>(<a href="../../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a> deadRS)</pre>
+<pre>private void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.845">transferQueues</a>(<a href="../../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a> deadRS)</pre>
<div class="block">Transfer all the queues of the specified to this region server. First it tries to grab a lock
and if it works it will move the old queues and finally will delete the old queues.
<p>
@@ -1326,7 +1326,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>join</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1020">join</a>()</pre>
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1019">join</a>()</pre>
<div class="block">Terminate the replication on this region server</div>
</li>
</ul>
@@ -1336,7 +1336,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getWALs</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java [...]
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java [...]
<div class="block">Get a copy of the wals of the normal sources on this rs</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1350,7 +1350,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getWalsByIdRecoveredQueues</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String [...]
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String [...]
<div class="block">Get a copy of the wals of the recovered sources on this rs</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1364,7 +1364,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getSources</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a>> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html# [...]
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a>> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html# [...]
<div class="block">Get a list of all the normal sources of this rs</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1378,7 +1378,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getOldSources</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a>> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html# [...]
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a>> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html# [...]
<div class="block">Get a list of all the recovered sources of this rs</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1392,7 +1392,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getSource</h4>
-<pre>public <a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1063">getSource</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in [...]
+<pre>public <a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1062">getSource</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in [...]
<div class="block">Get the normal source for a given peer</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1406,7 +1406,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getAllQueues</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1067">getAllQueues</a>()
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1066">getAllQueues</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -1420,7 +1420,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getSizeOfLatestPath</h4>
-<pre>int <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1077">getSizeOfLatestPath</a>()</pre>
+<pre>int <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1076">getSizeOfLatestPath</a>()</pre>
</li>
</ul>
<a name="getLastestPath--">
@@ -1429,7 +1429,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getLastestPath</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><org.apache.hadoop.fs.Path> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1083">getLastestPath</a>()</pre>
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><org.apache.hadoop.fs.Path> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1082">getLastestPath</a>()</pre>
</li>
</ul>
<a name="getTotalBufferUsed--">
@@ -1438,7 +1438,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getTotalBufferUsed</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1089">getTotalBufferUsed</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1088">getTotalBufferUsed</a>()</pre>
</li>
</ul>
<a name="getTotalBufferLimit--">
@@ -1447,7 +1447,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getTotalBufferLimit</h4>
-<pre>public long <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1097">getTotalBufferLimit</a>()</pre>
+<pre>public long <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1096">getTotalBufferLimit</a>()</pre>
<div class="block">Returns the maximum size in bytes of edits held in memory which are pending replication
across all sources inside this RegionServer.</div>
</li>
@@ -1458,7 +1458,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getOldLogDir</h4>
-<pre>public org.apache.hadoop.fs.Path <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1105">getOldLogDir</a>()</pre>
+<pre>public org.apache.hadoop.fs.Path <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1104">getOldLogDir</a>()</pre>
<div class="block">Get the directory where wals are archived</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1472,7 +1472,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getLogDir</h4>
-<pre>public org.apache.hadoop.fs.Path <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1113">getLogDir</a>()</pre>
+<pre>public org.apache.hadoop.fs.Path <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1112">getLogDir</a>()</pre>
<div class="block">Get the directory where wals are stored by their RSs</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1486,7 +1486,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getFs</h4>
-<pre>public org.apache.hadoop.fs.FileSystem <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1121">getFs</a>()</pre>
+<pre>public org.apache.hadoop.fs.FileSystem <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1120">getFs</a>()</pre>
<div class="block">Get the handle on the local file system</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1500,7 +1500,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getReplicationPeers</h4>
-<pre>public <a href="../../../../../../org/apache/hadoop/hbase/replication/ReplicationPeers.html" title="class in org.apache.hadoop.hbase.replication">ReplicationPeers</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1129">getReplicationPeers</a>()</pre>
+<pre>public <a href="../../../../../../org/apache/hadoop/hbase/replication/ReplicationPeers.html" title="class in org.apache.hadoop.hbase.replication">ReplicationPeers</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1128">getReplicationPeers</a>()</pre>
<div class="block">Get the ReplicationPeers used by this ReplicationSourceManager</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1514,7 +1514,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getStats</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1136">getStats</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1135">getStats</a>()</pre>
<div class="block">Get a string representation of all the sources' metrics</div>
</li>
</ul>
@@ -1524,7 +1524,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>addHFileRefs</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1153">addHFileRefs</a>(<a href="../../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> tableName,
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1152">addHFileRefs</a>(<a href="../../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> tableName,
byte[] family,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a><org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path>> pairs)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -1540,7 +1540,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>cleanUpHFileRefs</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1160">cleanUpHFileRefs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId,
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1159">cleanUpHFileRefs</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> peerId,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>> files)</pre>
</li>
</ul>
@@ -1550,7 +1550,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>activeFailoverTaskCount</h4>
-<pre>int <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1164">activeFailoverTaskCount</a>()</pre>
+<pre>int <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1163">activeFailoverTaskCount</a>()</pre>
</li>
</ul>
<a name="getGlobalMetrics--">
@@ -1559,7 +1559,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>getGlobalMetrics</h4>
-<pre><a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">MetricsReplicationGlobalSourceSource</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1168">getGlobalMetrics</a>()</pre>
+<pre><a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">MetricsReplicationGlobalSourceSource</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1167">getGlobalMetrics</a>()</pre>
</li>
</ul>
<a name="addCatalogReplicationSource-org.apache.hadoop.hbase.client.RegionInfo-">
@@ -1568,7 +1568,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>addCatalogReplicationSource</h4>
-<pre>public <a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1178">addCatalogReplicationSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apa [...]
+<pre>public <a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1177">addCatalogReplicationSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apa [...]
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Add an hbase:meta Catalog replication source. Called on open of an hbase:meta Region.
Create it once only. If exists already, use the existing one.</div>
@@ -1587,7 +1587,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockList">
<li class="blockList">
<h4>removeCatalogReplicationSource</h4>
-<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1193">removeCatalogReplicationSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a> regionInfo)</pre>
+<pre>public void <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1192">removeCatalogReplicationSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a> regionInfo)</pre>
<div class="block">Remove the hbase:meta Catalog replication source.
Called when we close hbase:meta.</div>
<dl>
@@ -1602,7 +1602,7 @@ implements <a href="../../../../../../org/apache/hadoop/hbase/replication/Replic
<ul class="blockListLast">
<li class="blockList">
<h4>createCatalogReplicationSource</h4>
-<pre>private <a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1206">createCatalogReplicationSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org [...]
+<pre>private <a href="../../../../../../org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.html" title="interface in org.apache.hadoop.hbase.replication.regionserver">ReplicationSourceInterface</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html#line.1205">createCatalogReplicationSource</a>(<a href="../../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org [...]
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Create, initialize, and start the Catalog ReplicationSource.
Presumes called one-time only (caller must ensure one-time only call).
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html
index 5780e8b..145bd21 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTracker.html
@@ -28,33 +28,34 @@
<span class="sourceLineNo">020</span><a name="line.20"></a>
<span class="sourceLineNo">021</span>import java.util.List;<a name="line.21"></a>
<span class="sourceLineNo">022</span><a name="line.22"></a>
-<span class="sourceLineNo">023</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.23"></a>
-<span class="sourceLineNo">024</span><a name="line.24"></a>
-<span class="sourceLineNo">025</span>/**<a name="line.25"></a>
-<span class="sourceLineNo">026</span> * This is the interface for a Replication Tracker. A replication tracker provides the facility to<a name="line.26"></a>
-<span class="sourceLineNo">027</span> * subscribe and track events that reflect a change in replication state. These events are used by<a name="line.27"></a>
-<span class="sourceLineNo">028</span> * the ReplicationSourceManager to coordinate replication tasks such as addition/deletion of queues<a name="line.28"></a>
-<span class="sourceLineNo">029</span> * and queue failover. These events are defined in the ReplicationListener interface. If a class<a name="line.29"></a>
-<span class="sourceLineNo">030</span> * would like to listen to replication events it must implement the ReplicationListener interface<a name="line.30"></a>
-<span class="sourceLineNo">031</span> * and register itself with a Replication Tracker.<a name="line.31"></a>
-<span class="sourceLineNo">032</span> */<a name="line.32"></a>
-<span class="sourceLineNo">033</span>@InterfaceAudience.Private<a name="line.33"></a>
-<span class="sourceLineNo">034</span>public interface ReplicationTracker {<a name="line.34"></a>
-<span class="sourceLineNo">035</span><a name="line.35"></a>
-<span class="sourceLineNo">036</span> /**<a name="line.36"></a>
-<span class="sourceLineNo">037</span> * Register a replication listener to receive replication events.<a name="line.37"></a>
-<span class="sourceLineNo">038</span> * @param listener the listener to register<a name="line.38"></a>
-<span class="sourceLineNo">039</span> */<a name="line.39"></a>
-<span class="sourceLineNo">040</span> public void registerListener(ReplicationListener listener);<a name="line.40"></a>
-<span class="sourceLineNo">041</span><a name="line.41"></a>
-<span class="sourceLineNo">042</span> public void removeListener(ReplicationListener listener);<a name="line.42"></a>
-<span class="sourceLineNo">043</span><a name="line.43"></a>
-<span class="sourceLineNo">044</span> /**<a name="line.44"></a>
-<span class="sourceLineNo">045</span> * Returns a list of other live region servers in the cluster.<a name="line.45"></a>
-<span class="sourceLineNo">046</span> * @return List of region servers.<a name="line.46"></a>
-<span class="sourceLineNo">047</span> */<a name="line.47"></a>
-<span class="sourceLineNo">048</span> public List<String> getListOfRegionServers();<a name="line.48"></a>
-<span class="sourceLineNo">049</span>}<a name="line.49"></a>
+<span class="sourceLineNo">023</span>import org.apache.hadoop.hbase.ServerName;<a name="line.23"></a>
+<span class="sourceLineNo">024</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.24"></a>
+<span class="sourceLineNo">025</span><a name="line.25"></a>
+<span class="sourceLineNo">026</span>/**<a name="line.26"></a>
+<span class="sourceLineNo">027</span> * This is the interface for a Replication Tracker. A replication tracker provides the facility to<a name="line.27"></a>
+<span class="sourceLineNo">028</span> * subscribe and track events that reflect a change in replication state. These events are used by<a name="line.28"></a>
+<span class="sourceLineNo">029</span> * the ReplicationSourceManager to coordinate replication tasks such as addition/deletion of queues<a name="line.29"></a>
+<span class="sourceLineNo">030</span> * and queue failover. These events are defined in the ReplicationListener interface. If a class<a name="line.30"></a>
+<span class="sourceLineNo">031</span> * would like to listen to replication events it must implement the ReplicationListener interface<a name="line.31"></a>
+<span class="sourceLineNo">032</span> * and register itself with a Replication Tracker.<a name="line.32"></a>
+<span class="sourceLineNo">033</span> */<a name="line.33"></a>
+<span class="sourceLineNo">034</span>@InterfaceAudience.Private<a name="line.34"></a>
+<span class="sourceLineNo">035</span>public interface ReplicationTracker {<a name="line.35"></a>
+<span class="sourceLineNo">036</span><a name="line.36"></a>
+<span class="sourceLineNo">037</span> /**<a name="line.37"></a>
+<span class="sourceLineNo">038</span> * Register a replication listener to receive replication events.<a name="line.38"></a>
+<span class="sourceLineNo">039</span> * @param listener the listener to register<a name="line.39"></a>
+<span class="sourceLineNo">040</span> */<a name="line.40"></a>
+<span class="sourceLineNo">041</span> void registerListener(ReplicationListener listener);<a name="line.41"></a>
+<span class="sourceLineNo">042</span><a name="line.42"></a>
+<span class="sourceLineNo">043</span> void removeListener(ReplicationListener listener);<a name="line.43"></a>
+<span class="sourceLineNo">044</span><a name="line.44"></a>
+<span class="sourceLineNo">045</span> /**<a name="line.45"></a>
+<span class="sourceLineNo">046</span> * Returns a list of other live region servers in the cluster.<a name="line.46"></a>
+<span class="sourceLineNo">047</span> * @return List of region servers.<a name="line.47"></a>
+<span class="sourceLineNo">048</span> */<a name="line.48"></a>
+<span class="sourceLineNo">049</span> List<ServerName> getListOfRegionServers();<a name="line.49"></a>
+<span class="sourceLineNo">050</span>}<a name="line.50"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html
index c212d30..13ddeea 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.OtherRegionServerWatcher.html
@@ -28,179 +28,183 @@
<span class="sourceLineNo">020</span>import java.util.ArrayList;<a name="line.20"></a>
<span class="sourceLineNo">021</span>import java.util.List;<a name="line.21"></a>
<span class="sourceLineNo">022</span>import java.util.concurrent.CopyOnWriteArrayList;<a name="line.22"></a>
-<span class="sourceLineNo">023</span>import org.apache.hadoop.hbase.Abortable;<a name="line.23"></a>
-<span class="sourceLineNo">024</span>import org.apache.hadoop.hbase.Stoppable;<a name="line.24"></a>
-<span class="sourceLineNo">025</span>import org.apache.hadoop.hbase.zookeeper.ZKListener;<a name="line.25"></a>
-<span class="sourceLineNo">026</span>import org.apache.hadoop.hbase.zookeeper.ZKUtil;<a name="line.26"></a>
-<span class="sourceLineNo">027</span>import org.apache.hadoop.hbase.zookeeper.ZKWatcher;<a name="line.27"></a>
-<span class="sourceLineNo">028</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.28"></a>
-<span class="sourceLineNo">029</span>import org.apache.zookeeper.KeeperException;<a name="line.29"></a>
-<span class="sourceLineNo">030</span>import org.slf4j.Logger;<a name="line.30"></a>
-<span class="sourceLineNo">031</span>import org.slf4j.LoggerFactory;<a name="line.31"></a>
-<span class="sourceLineNo">032</span><a name="line.32"></a>
-<span class="sourceLineNo">033</span>/**<a name="line.33"></a>
-<span class="sourceLineNo">034</span> * This class is a ZooKeeper implementation of the ReplicationTracker interface. This class is<a name="line.34"></a>
-<span class="sourceLineNo">035</span> * responsible for handling replication events that are defined in the ReplicationListener<a name="line.35"></a>
-<span class="sourceLineNo">036</span> * interface.<a name="line.36"></a>
-<span class="sourceLineNo">037</span> */<a name="line.37"></a>
-<span class="sourceLineNo">038</span>@InterfaceAudience.Private<a name="line.38"></a>
-<span class="sourceLineNo">039</span>public class ReplicationTrackerZKImpl implements ReplicationTracker {<a name="line.39"></a>
-<span class="sourceLineNo">040</span><a name="line.40"></a>
-<span class="sourceLineNo">041</span> private static final Logger LOG = LoggerFactory.getLogger(ReplicationTrackerZKImpl.class);<a name="line.41"></a>
-<span class="sourceLineNo">042</span><a name="line.42"></a>
-<span class="sourceLineNo">043</span> // Zookeeper<a name="line.43"></a>
-<span class="sourceLineNo">044</span> private final ZKWatcher zookeeper;<a name="line.44"></a>
-<span class="sourceLineNo">045</span> // Server to abort.<a name="line.45"></a>
-<span class="sourceLineNo">046</span> private final Abortable abortable;<a name="line.46"></a>
-<span class="sourceLineNo">047</span> // All about stopping<a name="line.47"></a>
-<span class="sourceLineNo">048</span> private final Stoppable stopper;<a name="line.48"></a>
-<span class="sourceLineNo">049</span> // listeners to be notified<a name="line.49"></a>
-<span class="sourceLineNo">050</span> private final List<ReplicationListener> listeners = new CopyOnWriteArrayList<>();<a name="line.50"></a>
-<span class="sourceLineNo">051</span> // List of all the other region servers in this cluster<a name="line.51"></a>
-<span class="sourceLineNo">052</span> private final ArrayList<String> otherRegionServers = new ArrayList<>();<a name="line.52"></a>
-<span class="sourceLineNo">053</span><a name="line.53"></a>
-<span class="sourceLineNo">054</span> public ReplicationTrackerZKImpl(ZKWatcher zookeeper, Abortable abortable, Stoppable stopper) {<a name="line.54"></a>
-<span class="sourceLineNo">055</span> this.zookeeper = zookeeper;<a name="line.55"></a>
-<span class="sourceLineNo">056</span> this.abortable = abortable;<a name="line.56"></a>
-<span class="sourceLineNo">057</span> this.stopper = stopper;<a name="line.57"></a>
-<span class="sourceLineNo">058</span> this.zookeeper.registerListener(new OtherRegionServerWatcher(this.zookeeper));<a name="line.58"></a>
-<span class="sourceLineNo">059</span> // watch the changes<a name="line.59"></a>
-<span class="sourceLineNo">060</span> refreshOtherRegionServersList(true);<a name="line.60"></a>
-<span class="sourceLineNo">061</span> }<a name="line.61"></a>
-<span class="sourceLineNo">062</span><a name="line.62"></a>
-<span class="sourceLineNo">063</span> @Override<a name="line.63"></a>
-<span class="sourceLineNo">064</span> public void registerListener(ReplicationListener listener) {<a name="line.64"></a>
-<span class="sourceLineNo">065</span> listeners.add(listener);<a name="line.65"></a>
-<span class="sourceLineNo">066</span> }<a name="line.66"></a>
-<span class="sourceLineNo">067</span><a name="line.67"></a>
-<span class="sourceLineNo">068</span> @Override<a name="line.68"></a>
-<span class="sourceLineNo">069</span> public void removeListener(ReplicationListener listener) {<a name="line.69"></a>
-<span class="sourceLineNo">070</span> listeners.remove(listener);<a name="line.70"></a>
-<span class="sourceLineNo">071</span> }<a name="line.71"></a>
-<span class="sourceLineNo">072</span><a name="line.72"></a>
-<span class="sourceLineNo">073</span> /**<a name="line.73"></a>
-<span class="sourceLineNo">074</span> * Return a snapshot of the current region servers.<a name="line.74"></a>
-<span class="sourceLineNo">075</span> */<a name="line.75"></a>
-<span class="sourceLineNo">076</span> @Override<a name="line.76"></a>
-<span class="sourceLineNo">077</span> public List<String> getListOfRegionServers() {<a name="line.77"></a>
-<span class="sourceLineNo">078</span> refreshOtherRegionServersList(false);<a name="line.78"></a>
-<span class="sourceLineNo">079</span><a name="line.79"></a>
-<span class="sourceLineNo">080</span> List<String> list = null;<a name="line.80"></a>
-<span class="sourceLineNo">081</span> synchronized (otherRegionServers) {<a name="line.81"></a>
-<span class="sourceLineNo">082</span> list = new ArrayList<>(otherRegionServers);<a name="line.82"></a>
-<span class="sourceLineNo">083</span> }<a name="line.83"></a>
-<span class="sourceLineNo">084</span> return list;<a name="line.84"></a>
-<span class="sourceLineNo">085</span> }<a name="line.85"></a>
-<span class="sourceLineNo">086</span><a name="line.86"></a>
-<span class="sourceLineNo">087</span> /**<a name="line.87"></a>
-<span class="sourceLineNo">088</span> * Watcher used to be notified of the other region server's death in the local cluster. It<a name="line.88"></a>
-<span class="sourceLineNo">089</span> * initiates the process to transfer the queues if it is able to grab the lock.<a name="line.89"></a>
-<span class="sourceLineNo">090</span> */<a name="line.90"></a>
-<span class="sourceLineNo">091</span> public class OtherRegionServerWatcher extends ZKListener {<a name="line.91"></a>
-<span class="sourceLineNo">092</span><a name="line.92"></a>
-<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
-<span class="sourceLineNo">094</span> * Construct a ZooKeeper event listener.<a name="line.94"></a>
-<span class="sourceLineNo">095</span> */<a name="line.95"></a>
-<span class="sourceLineNo">096</span> public OtherRegionServerWatcher(ZKWatcher watcher) {<a name="line.96"></a>
-<span class="sourceLineNo">097</span> super(watcher);<a name="line.97"></a>
-<span class="sourceLineNo">098</span> }<a name="line.98"></a>
-<span class="sourceLineNo">099</span><a name="line.99"></a>
-<span class="sourceLineNo">100</span> /**<a name="line.100"></a>
-<span class="sourceLineNo">101</span> * Called when a new node has been created.<a name="line.101"></a>
-<span class="sourceLineNo">102</span> * @param path full path of the new node<a name="line.102"></a>
-<span class="sourceLineNo">103</span> */<a name="line.103"></a>
-<span class="sourceLineNo">104</span> @Override<a name="line.104"></a>
-<span class="sourceLineNo">105</span> public void nodeCreated(String path) {<a name="line.105"></a>
-<span class="sourceLineNo">106</span> refreshListIfRightPath(path);<a name="line.106"></a>
-<span class="sourceLineNo">107</span> }<a name="line.107"></a>
-<span class="sourceLineNo">108</span><a name="line.108"></a>
-<span class="sourceLineNo">109</span> /**<a name="line.109"></a>
-<span class="sourceLineNo">110</span> * Called when a node has been deleted<a name="line.110"></a>
-<span class="sourceLineNo">111</span> * @param path full path of the deleted node<a name="line.111"></a>
-<span class="sourceLineNo">112</span> */<a name="line.112"></a>
-<span class="sourceLineNo">113</span> @Override<a name="line.113"></a>
-<span class="sourceLineNo">114</span> public void nodeDeleted(String path) {<a name="line.114"></a>
-<span class="sourceLineNo">115</span> if (stopper.isStopped()) {<a name="line.115"></a>
-<span class="sourceLineNo">116</span> return;<a name="line.116"></a>
-<span class="sourceLineNo">117</span> }<a name="line.117"></a>
-<span class="sourceLineNo">118</span> boolean cont = refreshListIfRightPath(path);<a name="line.118"></a>
-<span class="sourceLineNo">119</span> if (!cont) {<a name="line.119"></a>
-<span class="sourceLineNo">120</span> return;<a name="line.120"></a>
-<span class="sourceLineNo">121</span> }<a name="line.121"></a>
-<span class="sourceLineNo">122</span> LOG.info(path + " znode expired, triggering replicatorRemoved event");<a name="line.122"></a>
-<span class="sourceLineNo">123</span> for (ReplicationListener rl : listeners) {<a name="line.123"></a>
-<span class="sourceLineNo">124</span> rl.regionServerRemoved(getZNodeName(path));<a name="line.124"></a>
-<span class="sourceLineNo">125</span> }<a name="line.125"></a>
-<span class="sourceLineNo">126</span> }<a name="line.126"></a>
-<span class="sourceLineNo">127</span><a name="line.127"></a>
-<span class="sourceLineNo">128</span> /**<a name="line.128"></a>
-<span class="sourceLineNo">129</span> * Called when an existing node has a child node added or removed.<a name="line.129"></a>
-<span class="sourceLineNo">130</span> * @param path full path of the node whose children have changed<a name="line.130"></a>
-<span class="sourceLineNo">131</span> */<a name="line.131"></a>
-<span class="sourceLineNo">132</span> @Override<a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void nodeChildrenChanged(String path) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> if (stopper.isStopped()) {<a name="line.134"></a>
-<span class="sourceLineNo">135</span> return;<a name="line.135"></a>
-<span class="sourceLineNo">136</span> }<a name="line.136"></a>
-<span class="sourceLineNo">137</span> refreshListIfRightPath(path);<a name="line.137"></a>
-<span class="sourceLineNo">138</span> }<a name="line.138"></a>
-<span class="sourceLineNo">139</span><a name="line.139"></a>
-<span class="sourceLineNo">140</span> private boolean refreshListIfRightPath(String path) {<a name="line.140"></a>
-<span class="sourceLineNo">141</span> if (!path.startsWith(this.watcher.getZNodePaths().rsZNode)) {<a name="line.141"></a>
-<span class="sourceLineNo">142</span> return false;<a name="line.142"></a>
-<span class="sourceLineNo">143</span> }<a name="line.143"></a>
-<span class="sourceLineNo">144</span> return refreshOtherRegionServersList(true);<a name="line.144"></a>
-<span class="sourceLineNo">145</span> }<a name="line.145"></a>
-<span class="sourceLineNo">146</span> }<a name="line.146"></a>
-<span class="sourceLineNo">147</span><a name="line.147"></a>
-<span class="sourceLineNo">148</span> /**<a name="line.148"></a>
-<span class="sourceLineNo">149</span> * Extracts the znode name of a peer cluster from a ZK path<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * @param fullPath Path to extract the id from<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * @return the id or an empty string if path is invalid<a name="line.151"></a>
-<span class="sourceLineNo">152</span> */<a name="line.152"></a>
-<span class="sourceLineNo">153</span> private String getZNodeName(String fullPath) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span> String[] parts = fullPath.split("/");<a name="line.154"></a>
-<span class="sourceLineNo">155</span> return parts.length > 0 ? parts[parts.length - 1] : "";<a name="line.155"></a>
-<span class="sourceLineNo">156</span> }<a name="line.156"></a>
-<span class="sourceLineNo">157</span><a name="line.157"></a>
-<span class="sourceLineNo">158</span> /**<a name="line.158"></a>
-<span class="sourceLineNo">159</span> * Reads the list of region servers from ZK and atomically clears our local view of it and<a name="line.159"></a>
-<span class="sourceLineNo">160</span> * replaces it with the updated list.<a name="line.160"></a>
-<span class="sourceLineNo">161</span> * @return true if the local list of the other region servers was updated with the ZK data (even<a name="line.161"></a>
-<span class="sourceLineNo">162</span> * if it was empty), false if the data was missing in ZK<a name="line.162"></a>
-<span class="sourceLineNo">163</span> */<a name="line.163"></a>
-<span class="sourceLineNo">164</span> private boolean refreshOtherRegionServersList(boolean watch) {<a name="line.164"></a>
-<span class="sourceLineNo">165</span> List<String> newRsList = getRegisteredRegionServers(watch);<a name="line.165"></a>
-<span class="sourceLineNo">166</span> if (newRsList == null) {<a name="line.166"></a>
-<span class="sourceLineNo">167</span> return false;<a name="line.167"></a>
-<span class="sourceLineNo">168</span> } else {<a name="line.168"></a>
-<span class="sourceLineNo">169</span> synchronized (otherRegionServers) {<a name="line.169"></a>
-<span class="sourceLineNo">170</span> otherRegionServers.clear();<a name="line.170"></a>
-<span class="sourceLineNo">171</span> otherRegionServers.addAll(newRsList);<a name="line.171"></a>
-<span class="sourceLineNo">172</span> }<a name="line.172"></a>
-<span class="sourceLineNo">173</span> }<a name="line.173"></a>
-<span class="sourceLineNo">174</span> return true;<a name="line.174"></a>
-<span class="sourceLineNo">175</span> }<a name="line.175"></a>
-<span class="sourceLineNo">176</span><a name="line.176"></a>
-<span class="sourceLineNo">177</span> /**<a name="line.177"></a>
-<span class="sourceLineNo">178</span> * Get a list of all the other region servers in this cluster and set a watch<a name="line.178"></a>
-<span class="sourceLineNo">179</span> * @return a list of server nanes<a name="line.179"></a>
-<span class="sourceLineNo">180</span> */<a name="line.180"></a>
-<span class="sourceLineNo">181</span> private List<String> getRegisteredRegionServers(boolean watch) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span> List<String> result = null;<a name="line.182"></a>
-<span class="sourceLineNo">183</span> try {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> if (watch) {<a name="line.184"></a>
-<span class="sourceLineNo">185</span> result = ZKUtil.listChildrenAndWatchThem(this.zookeeper,<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.zookeeper.getZNodePaths().rsZNode);<a name="line.186"></a>
-<span class="sourceLineNo">187</span> } else {<a name="line.187"></a>
-<span class="sourceLineNo">188</span> result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.zookeeper.getZNodePaths().rsZNode);<a name="line.188"></a>
-<span class="sourceLineNo">189</span> }<a name="line.189"></a>
-<span class="sourceLineNo">190</span> } catch (KeeperException e) {<a name="line.190"></a>
-<span class="sourceLineNo">191</span> this.abortable.abort("Get list of registered region servers", e);<a name="line.191"></a>
-<span class="sourceLineNo">192</span> }<a name="line.192"></a>
-<span class="sourceLineNo">193</span> return result;<a name="line.193"></a>
-<span class="sourceLineNo">194</span> }<a name="line.194"></a>
-<span class="sourceLineNo">195</span>}<a name="line.195"></a>
+<span class="sourceLineNo">023</span>import java.util.stream.Collectors;<a name="line.23"></a>
+<span class="sourceLineNo">024</span><a name="line.24"></a>
+<span class="sourceLineNo">025</span>import org.apache.hadoop.hbase.Abortable;<a name="line.25"></a>
+<span class="sourceLineNo">026</span>import org.apache.hadoop.hbase.ServerName;<a name="line.26"></a>
+<span class="sourceLineNo">027</span>import org.apache.hadoop.hbase.Stoppable;<a name="line.27"></a>
+<span class="sourceLineNo">028</span>import org.apache.hadoop.hbase.zookeeper.ZKListener;<a name="line.28"></a>
+<span class="sourceLineNo">029</span>import org.apache.hadoop.hbase.zookeeper.ZKUtil;<a name="line.29"></a>
+<span class="sourceLineNo">030</span>import org.apache.hadoop.hbase.zookeeper.ZKWatcher;<a name="line.30"></a>
+<span class="sourceLineNo">031</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.31"></a>
+<span class="sourceLineNo">032</span>import org.apache.zookeeper.KeeperException;<a name="line.32"></a>
+<span class="sourceLineNo">033</span>import org.slf4j.Logger;<a name="line.33"></a>
+<span class="sourceLineNo">034</span>import org.slf4j.LoggerFactory;<a name="line.34"></a>
+<span class="sourceLineNo">035</span><a name="line.35"></a>
+<span class="sourceLineNo">036</span>/**<a name="line.36"></a>
+<span class="sourceLineNo">037</span> * This class is a ZooKeeper implementation of the ReplicationTracker interface. This class is<a name="line.37"></a>
+<span class="sourceLineNo">038</span> * responsible for handling replication events that are defined in the ReplicationListener<a name="line.38"></a>
+<span class="sourceLineNo">039</span> * interface.<a name="line.39"></a>
+<span class="sourceLineNo">040</span> */<a name="line.40"></a>
+<span class="sourceLineNo">041</span>@InterfaceAudience.Private<a name="line.41"></a>
+<span class="sourceLineNo">042</span>public class ReplicationTrackerZKImpl implements ReplicationTracker {<a name="line.42"></a>
+<span class="sourceLineNo">043</span><a name="line.43"></a>
+<span class="sourceLineNo">044</span> private static final Logger LOG = LoggerFactory.getLogger(ReplicationTrackerZKImpl.class);<a name="line.44"></a>
+<span class="sourceLineNo">045</span><a name="line.45"></a>
+<span class="sourceLineNo">046</span> // Zookeeper<a name="line.46"></a>
+<span class="sourceLineNo">047</span> private final ZKWatcher zookeeper;<a name="line.47"></a>
+<span class="sourceLineNo">048</span> // Server to abort.<a name="line.48"></a>
+<span class="sourceLineNo">049</span> private final Abortable abortable;<a name="line.49"></a>
+<span class="sourceLineNo">050</span> // All about stopping<a name="line.50"></a>
+<span class="sourceLineNo">051</span> private final Stoppable stopper;<a name="line.51"></a>
+<span class="sourceLineNo">052</span> // listeners to be notified<a name="line.52"></a>
+<span class="sourceLineNo">053</span> private final List<ReplicationListener> listeners = new CopyOnWriteArrayList<>();<a name="line.53"></a>
+<span class="sourceLineNo">054</span> // List of all the other region servers in this cluster<a name="line.54"></a>
+<span class="sourceLineNo">055</span> private final List<ServerName> otherRegionServers = new ArrayList<>();<a name="line.55"></a>
+<span class="sourceLineNo">056</span><a name="line.56"></a>
+<span class="sourceLineNo">057</span> public ReplicationTrackerZKImpl(ZKWatcher zookeeper, Abortable abortable, Stoppable stopper) {<a name="line.57"></a>
+<span class="sourceLineNo">058</span> this.zookeeper = zookeeper;<a name="line.58"></a>
+<span class="sourceLineNo">059</span> this.abortable = abortable;<a name="line.59"></a>
+<span class="sourceLineNo">060</span> this.stopper = stopper;<a name="line.60"></a>
+<span class="sourceLineNo">061</span> this.zookeeper.registerListener(new OtherRegionServerWatcher(this.zookeeper));<a name="line.61"></a>
+<span class="sourceLineNo">062</span> // watch the changes<a name="line.62"></a>
+<span class="sourceLineNo">063</span> refreshOtherRegionServersList(true);<a name="line.63"></a>
+<span class="sourceLineNo">064</span> }<a name="line.64"></a>
+<span class="sourceLineNo">065</span><a name="line.65"></a>
+<span class="sourceLineNo">066</span> @Override<a name="line.66"></a>
+<span class="sourceLineNo">067</span> public void registerListener(ReplicationListener listener) {<a name="line.67"></a>
+<span class="sourceLineNo">068</span> listeners.add(listener);<a name="line.68"></a>
+<span class="sourceLineNo">069</span> }<a name="line.69"></a>
+<span class="sourceLineNo">070</span><a name="line.70"></a>
+<span class="sourceLineNo">071</span> @Override<a name="line.71"></a>
+<span class="sourceLineNo">072</span> public void removeListener(ReplicationListener listener) {<a name="line.72"></a>
+<span class="sourceLineNo">073</span> listeners.remove(listener);<a name="line.73"></a>
+<span class="sourceLineNo">074</span> }<a name="line.74"></a>
+<span class="sourceLineNo">075</span><a name="line.75"></a>
+<span class="sourceLineNo">076</span> /**<a name="line.76"></a>
+<span class="sourceLineNo">077</span> * Return a snapshot of the current region servers.<a name="line.77"></a>
+<span class="sourceLineNo">078</span> */<a name="line.78"></a>
+<span class="sourceLineNo">079</span> @Override<a name="line.79"></a>
+<span class="sourceLineNo">080</span> public List<ServerName> getListOfRegionServers() {<a name="line.80"></a>
+<span class="sourceLineNo">081</span> refreshOtherRegionServersList(false);<a name="line.81"></a>
+<span class="sourceLineNo">082</span><a name="line.82"></a>
+<span class="sourceLineNo">083</span> List<ServerName> list = null;<a name="line.83"></a>
+<span class="sourceLineNo">084</span> synchronized (otherRegionServers) {<a name="line.84"></a>
+<span class="sourceLineNo">085</span> list = new ArrayList<>(otherRegionServers);<a name="line.85"></a>
+<span class="sourceLineNo">086</span> }<a name="line.86"></a>
+<span class="sourceLineNo">087</span> return list;<a name="line.87"></a>
+<span class="sourceLineNo">088</span> }<a name="line.88"></a>
+<span class="sourceLineNo">089</span><a name="line.89"></a>
+<span class="sourceLineNo">090</span> /**<a name="line.90"></a>
+<span class="sourceLineNo">091</span> * Watcher used to be notified of the other region server's death in the local cluster. It<a name="line.91"></a>
+<span class="sourceLineNo">092</span> * initiates the process to transfer the queues if it is able to grab the lock.<a name="line.92"></a>
+<span class="sourceLineNo">093</span> */<a name="line.93"></a>
+<span class="sourceLineNo">094</span> public class OtherRegionServerWatcher extends ZKListener {<a name="line.94"></a>
+<span class="sourceLineNo">095</span><a name="line.95"></a>
+<span class="sourceLineNo">096</span> /**<a name="line.96"></a>
+<span class="sourceLineNo">097</span> * Construct a ZooKeeper event listener.<a name="line.97"></a>
+<span class="sourceLineNo">098</span> */<a name="line.98"></a>
+<span class="sourceLineNo">099</span> public OtherRegionServerWatcher(ZKWatcher watcher) {<a name="line.99"></a>
+<span class="sourceLineNo">100</span> super(watcher);<a name="line.100"></a>
+<span class="sourceLineNo">101</span> }<a name="line.101"></a>
+<span class="sourceLineNo">102</span><a name="line.102"></a>
+<span class="sourceLineNo">103</span> /**<a name="line.103"></a>
+<span class="sourceLineNo">104</span> * Called when a new node has been created.<a name="line.104"></a>
+<span class="sourceLineNo">105</span> * @param path full path of the new node<a name="line.105"></a>
+<span class="sourceLineNo">106</span> */<a name="line.106"></a>
+<span class="sourceLineNo">107</span> @Override<a name="line.107"></a>
+<span class="sourceLineNo">108</span> public void nodeCreated(String path) {<a name="line.108"></a>
+<span class="sourceLineNo">109</span> refreshListIfRightPath(path);<a name="line.109"></a>
+<span class="sourceLineNo">110</span> }<a name="line.110"></a>
+<span class="sourceLineNo">111</span><a name="line.111"></a>
+<span class="sourceLineNo">112</span> /**<a name="line.112"></a>
+<span class="sourceLineNo">113</span> * Called when a node has been deleted<a name="line.113"></a>
+<span class="sourceLineNo">114</span> * @param path full path of the deleted node<a name="line.114"></a>
+<span class="sourceLineNo">115</span> */<a name="line.115"></a>
+<span class="sourceLineNo">116</span> @Override<a name="line.116"></a>
+<span class="sourceLineNo">117</span> public void nodeDeleted(String path) {<a name="line.117"></a>
+<span class="sourceLineNo">118</span> if (stopper.isStopped()) {<a name="line.118"></a>
+<span class="sourceLineNo">119</span> return;<a name="line.119"></a>
+<span class="sourceLineNo">120</span> }<a name="line.120"></a>
+<span class="sourceLineNo">121</span> boolean cont = refreshListIfRightPath(path);<a name="line.121"></a>
+<span class="sourceLineNo">122</span> if (!cont) {<a name="line.122"></a>
+<span class="sourceLineNo">123</span> return;<a name="line.123"></a>
+<span class="sourceLineNo">124</span> }<a name="line.124"></a>
+<span class="sourceLineNo">125</span> LOG.info(path + " znode expired, triggering replicatorRemoved event");<a name="line.125"></a>
+<span class="sourceLineNo">126</span> for (ReplicationListener rl : listeners) {<a name="line.126"></a>
+<span class="sourceLineNo">127</span> rl.regionServerRemoved(getZNodeName(path));<a name="line.127"></a>
+<span class="sourceLineNo">128</span> }<a name="line.128"></a>
+<span class="sourceLineNo">129</span> }<a name="line.129"></a>
+<span class="sourceLineNo">130</span><a name="line.130"></a>
+<span class="sourceLineNo">131</span> /**<a name="line.131"></a>
+<span class="sourceLineNo">132</span> * Called when an existing node has a child node added or removed.<a name="line.132"></a>
+<span class="sourceLineNo">133</span> * @param path full path of the node whose children have changed<a name="line.133"></a>
+<span class="sourceLineNo">134</span> */<a name="line.134"></a>
+<span class="sourceLineNo">135</span> @Override<a name="line.135"></a>
+<span class="sourceLineNo">136</span> public void nodeChildrenChanged(String path) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> if (stopper.isStopped()) {<a name="line.137"></a>
+<span class="sourceLineNo">138</span> return;<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span> refreshListIfRightPath(path);<a name="line.140"></a>
+<span class="sourceLineNo">141</span> }<a name="line.141"></a>
+<span class="sourceLineNo">142</span><a name="line.142"></a>
+<span class="sourceLineNo">143</span> private boolean refreshListIfRightPath(String path) {<a name="line.143"></a>
+<span class="sourceLineNo">144</span> if (!path.startsWith(this.watcher.getZNodePaths().rsZNode)) {<a name="line.144"></a>
+<span class="sourceLineNo">145</span> return false;<a name="line.145"></a>
+<span class="sourceLineNo">146</span> }<a name="line.146"></a>
+<span class="sourceLineNo">147</span> return refreshOtherRegionServersList(true);<a name="line.147"></a>
+<span class="sourceLineNo">148</span> }<a name="line.148"></a>
+<span class="sourceLineNo">149</span> }<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> /**<a name="line.151"></a>
+<span class="sourceLineNo">152</span> * Extracts the znode name of a peer cluster from a ZK path<a name="line.152"></a>
+<span class="sourceLineNo">153</span> * @param fullPath Path to extract the id from<a name="line.153"></a>
+<span class="sourceLineNo">154</span> * @return the id or an empty string if path is invalid<a name="line.154"></a>
+<span class="sourceLineNo">155</span> */<a name="line.155"></a>
+<span class="sourceLineNo">156</span> private String getZNodeName(String fullPath) {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> String[] parts = fullPath.split("/");<a name="line.157"></a>
+<span class="sourceLineNo">158</span> return parts.length > 0 ? parts[parts.length - 1] : "";<a name="line.158"></a>
+<span class="sourceLineNo">159</span> }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span> /**<a name="line.161"></a>
+<span class="sourceLineNo">162</span> * Reads the list of region servers from ZK and atomically clears our local view of it and<a name="line.162"></a>
+<span class="sourceLineNo">163</span> * replaces it with the updated list.<a name="line.163"></a>
+<span class="sourceLineNo">164</span> * @return true if the local list of the other region servers was updated with the ZK data (even<a name="line.164"></a>
+<span class="sourceLineNo">165</span> * if it was empty), false if the data was missing in ZK<a name="line.165"></a>
+<span class="sourceLineNo">166</span> */<a name="line.166"></a>
+<span class="sourceLineNo">167</span> private boolean refreshOtherRegionServersList(boolean watch) {<a name="line.167"></a>
+<span class="sourceLineNo">168</span> List<ServerName> newRsList = getRegisteredRegionServers(watch);<a name="line.168"></a>
+<span class="sourceLineNo">169</span> if (newRsList == null) {<a name="line.169"></a>
+<span class="sourceLineNo">170</span> return false;<a name="line.170"></a>
+<span class="sourceLineNo">171</span> } else {<a name="line.171"></a>
+<span class="sourceLineNo">172</span> synchronized (otherRegionServers) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> otherRegionServers.clear();<a name="line.173"></a>
+<span class="sourceLineNo">174</span> otherRegionServers.addAll(newRsList);<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span> return true;<a name="line.177"></a>
+<span class="sourceLineNo">178</span> }<a name="line.178"></a>
+<span class="sourceLineNo">179</span><a name="line.179"></a>
+<span class="sourceLineNo">180</span> /**<a name="line.180"></a>
+<span class="sourceLineNo">181</span> * Get a list of all the other region servers in this cluster and set a watch<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * @return a list of server nanes<a name="line.182"></a>
+<span class="sourceLineNo">183</span> */<a name="line.183"></a>
+<span class="sourceLineNo">184</span> private List<ServerName> getRegisteredRegionServers(boolean watch) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span> List<String> result = null;<a name="line.185"></a>
+<span class="sourceLineNo">186</span> try {<a name="line.186"></a>
+<span class="sourceLineNo">187</span> if (watch) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span> result = ZKUtil.listChildrenAndWatchThem(this.zookeeper,<a name="line.188"></a>
+<span class="sourceLineNo">189</span> this.zookeeper.getZNodePaths().rsZNode);<a name="line.189"></a>
+<span class="sourceLineNo">190</span> } else {<a name="line.190"></a>
+<span class="sourceLineNo">191</span> result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.zookeeper.getZNodePaths().rsZNode);<a name="line.191"></a>
+<span class="sourceLineNo">192</span> }<a name="line.192"></a>
+<span class="sourceLineNo">193</span> } catch (KeeperException e) {<a name="line.193"></a>
+<span class="sourceLineNo">194</span> this.abortable.abort("Get list of registered region servers", e);<a name="line.194"></a>
+<span class="sourceLineNo">195</span> }<a name="line.195"></a>
+<span class="sourceLineNo">196</span> return result == null ? null :<a name="line.196"></a>
+<span class="sourceLineNo">197</span> result.stream().map(ServerName::parseServerName).collect(Collectors.toList());<a name="line.197"></a>
+<span class="sourceLineNo">198</span> }<a name="line.198"></a>
+<span class="sourceLineNo">199</span>}<a name="line.199"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html
index c212d30..13ddeea 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.html
@@ -28,179 +28,183 @@
<span class="sourceLineNo">020</span>import java.util.ArrayList;<a name="line.20"></a>
<span class="sourceLineNo">021</span>import java.util.List;<a name="line.21"></a>
<span class="sourceLineNo">022</span>import java.util.concurrent.CopyOnWriteArrayList;<a name="line.22"></a>
-<span class="sourceLineNo">023</span>import org.apache.hadoop.hbase.Abortable;<a name="line.23"></a>
-<span class="sourceLineNo">024</span>import org.apache.hadoop.hbase.Stoppable;<a name="line.24"></a>
-<span class="sourceLineNo">025</span>import org.apache.hadoop.hbase.zookeeper.ZKListener;<a name="line.25"></a>
-<span class="sourceLineNo">026</span>import org.apache.hadoop.hbase.zookeeper.ZKUtil;<a name="line.26"></a>
-<span class="sourceLineNo">027</span>import org.apache.hadoop.hbase.zookeeper.ZKWatcher;<a name="line.27"></a>
-<span class="sourceLineNo">028</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.28"></a>
-<span class="sourceLineNo">029</span>import org.apache.zookeeper.KeeperException;<a name="line.29"></a>
-<span class="sourceLineNo">030</span>import org.slf4j.Logger;<a name="line.30"></a>
-<span class="sourceLineNo">031</span>import org.slf4j.LoggerFactory;<a name="line.31"></a>
-<span class="sourceLineNo">032</span><a name="line.32"></a>
-<span class="sourceLineNo">033</span>/**<a name="line.33"></a>
-<span class="sourceLineNo">034</span> * This class is a ZooKeeper implementation of the ReplicationTracker interface. This class is<a name="line.34"></a>
-<span class="sourceLineNo">035</span> * responsible for handling replication events that are defined in the ReplicationListener<a name="line.35"></a>
-<span class="sourceLineNo">036</span> * interface.<a name="line.36"></a>
-<span class="sourceLineNo">037</span> */<a name="line.37"></a>
-<span class="sourceLineNo">038</span>@InterfaceAudience.Private<a name="line.38"></a>
-<span class="sourceLineNo">039</span>public class ReplicationTrackerZKImpl implements ReplicationTracker {<a name="line.39"></a>
-<span class="sourceLineNo">040</span><a name="line.40"></a>
-<span class="sourceLineNo">041</span> private static final Logger LOG = LoggerFactory.getLogger(ReplicationTrackerZKImpl.class);<a name="line.41"></a>
-<span class="sourceLineNo">042</span><a name="line.42"></a>
-<span class="sourceLineNo">043</span> // Zookeeper<a name="line.43"></a>
-<span class="sourceLineNo">044</span> private final ZKWatcher zookeeper;<a name="line.44"></a>
-<span class="sourceLineNo">045</span> // Server to abort.<a name="line.45"></a>
-<span class="sourceLineNo">046</span> private final Abortable abortable;<a name="line.46"></a>
-<span class="sourceLineNo">047</span> // All about stopping<a name="line.47"></a>
-<span class="sourceLineNo">048</span> private final Stoppable stopper;<a name="line.48"></a>
-<span class="sourceLineNo">049</span> // listeners to be notified<a name="line.49"></a>
-<span class="sourceLineNo">050</span> private final List<ReplicationListener> listeners = new CopyOnWriteArrayList<>();<a name="line.50"></a>
-<span class="sourceLineNo">051</span> // List of all the other region servers in this cluster<a name="line.51"></a>
-<span class="sourceLineNo">052</span> private final ArrayList<String> otherRegionServers = new ArrayList<>();<a name="line.52"></a>
-<span class="sourceLineNo">053</span><a name="line.53"></a>
-<span class="sourceLineNo">054</span> public ReplicationTrackerZKImpl(ZKWatcher zookeeper, Abortable abortable, Stoppable stopper) {<a name="line.54"></a>
-<span class="sourceLineNo">055</span> this.zookeeper = zookeeper;<a name="line.55"></a>
-<span class="sourceLineNo">056</span> this.abortable = abortable;<a name="line.56"></a>
-<span class="sourceLineNo">057</span> this.stopper = stopper;<a name="line.57"></a>
-<span class="sourceLineNo">058</span> this.zookeeper.registerListener(new OtherRegionServerWatcher(this.zookeeper));<a name="line.58"></a>
-<span class="sourceLineNo">059</span> // watch the changes<a name="line.59"></a>
-<span class="sourceLineNo">060</span> refreshOtherRegionServersList(true);<a name="line.60"></a>
-<span class="sourceLineNo">061</span> }<a name="line.61"></a>
-<span class="sourceLineNo">062</span><a name="line.62"></a>
-<span class="sourceLineNo">063</span> @Override<a name="line.63"></a>
-<span class="sourceLineNo">064</span> public void registerListener(ReplicationListener listener) {<a name="line.64"></a>
-<span class="sourceLineNo">065</span> listeners.add(listener);<a name="line.65"></a>
-<span class="sourceLineNo">066</span> }<a name="line.66"></a>
-<span class="sourceLineNo">067</span><a name="line.67"></a>
-<span class="sourceLineNo">068</span> @Override<a name="line.68"></a>
-<span class="sourceLineNo">069</span> public void removeListener(ReplicationListener listener) {<a name="line.69"></a>
-<span class="sourceLineNo">070</span> listeners.remove(listener);<a name="line.70"></a>
-<span class="sourceLineNo">071</span> }<a name="line.71"></a>
-<span class="sourceLineNo">072</span><a name="line.72"></a>
-<span class="sourceLineNo">073</span> /**<a name="line.73"></a>
-<span class="sourceLineNo">074</span> * Return a snapshot of the current region servers.<a name="line.74"></a>
-<span class="sourceLineNo">075</span> */<a name="line.75"></a>
-<span class="sourceLineNo">076</span> @Override<a name="line.76"></a>
-<span class="sourceLineNo">077</span> public List<String> getListOfRegionServers() {<a name="line.77"></a>
-<span class="sourceLineNo">078</span> refreshOtherRegionServersList(false);<a name="line.78"></a>
-<span class="sourceLineNo">079</span><a name="line.79"></a>
-<span class="sourceLineNo">080</span> List<String> list = null;<a name="line.80"></a>
-<span class="sourceLineNo">081</span> synchronized (otherRegionServers) {<a name="line.81"></a>
-<span class="sourceLineNo">082</span> list = new ArrayList<>(otherRegionServers);<a name="line.82"></a>
-<span class="sourceLineNo">083</span> }<a name="line.83"></a>
-<span class="sourceLineNo">084</span> return list;<a name="line.84"></a>
-<span class="sourceLineNo">085</span> }<a name="line.85"></a>
-<span class="sourceLineNo">086</span><a name="line.86"></a>
-<span class="sourceLineNo">087</span> /**<a name="line.87"></a>
-<span class="sourceLineNo">088</span> * Watcher used to be notified of the other region server's death in the local cluster. It<a name="line.88"></a>
-<span class="sourceLineNo">089</span> * initiates the process to transfer the queues if it is able to grab the lock.<a name="line.89"></a>
-<span class="sourceLineNo">090</span> */<a name="line.90"></a>
-<span class="sourceLineNo">091</span> public class OtherRegionServerWatcher extends ZKListener {<a name="line.91"></a>
-<span class="sourceLineNo">092</span><a name="line.92"></a>
-<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
-<span class="sourceLineNo">094</span> * Construct a ZooKeeper event listener.<a name="line.94"></a>
-<span class="sourceLineNo">095</span> */<a name="line.95"></a>
-<span class="sourceLineNo">096</span> public OtherRegionServerWatcher(ZKWatcher watcher) {<a name="line.96"></a>
-<span class="sourceLineNo">097</span> super(watcher);<a name="line.97"></a>
-<span class="sourceLineNo">098</span> }<a name="line.98"></a>
-<span class="sourceLineNo">099</span><a name="line.99"></a>
-<span class="sourceLineNo">100</span> /**<a name="line.100"></a>
-<span class="sourceLineNo">101</span> * Called when a new node has been created.<a name="line.101"></a>
-<span class="sourceLineNo">102</span> * @param path full path of the new node<a name="line.102"></a>
-<span class="sourceLineNo">103</span> */<a name="line.103"></a>
-<span class="sourceLineNo">104</span> @Override<a name="line.104"></a>
-<span class="sourceLineNo">105</span> public void nodeCreated(String path) {<a name="line.105"></a>
-<span class="sourceLineNo">106</span> refreshListIfRightPath(path);<a name="line.106"></a>
-<span class="sourceLineNo">107</span> }<a name="line.107"></a>
-<span class="sourceLineNo">108</span><a name="line.108"></a>
-<span class="sourceLineNo">109</span> /**<a name="line.109"></a>
-<span class="sourceLineNo">110</span> * Called when a node has been deleted<a name="line.110"></a>
-<span class="sourceLineNo">111</span> * @param path full path of the deleted node<a name="line.111"></a>
-<span class="sourceLineNo">112</span> */<a name="line.112"></a>
-<span class="sourceLineNo">113</span> @Override<a name="line.113"></a>
-<span class="sourceLineNo">114</span> public void nodeDeleted(String path) {<a name="line.114"></a>
-<span class="sourceLineNo">115</span> if (stopper.isStopped()) {<a name="line.115"></a>
-<span class="sourceLineNo">116</span> return;<a name="line.116"></a>
-<span class="sourceLineNo">117</span> }<a name="line.117"></a>
-<span class="sourceLineNo">118</span> boolean cont = refreshListIfRightPath(path);<a name="line.118"></a>
-<span class="sourceLineNo">119</span> if (!cont) {<a name="line.119"></a>
-<span class="sourceLineNo">120</span> return;<a name="line.120"></a>
-<span class="sourceLineNo">121</span> }<a name="line.121"></a>
-<span class="sourceLineNo">122</span> LOG.info(path + " znode expired, triggering replicatorRemoved event");<a name="line.122"></a>
-<span class="sourceLineNo">123</span> for (ReplicationListener rl : listeners) {<a name="line.123"></a>
-<span class="sourceLineNo">124</span> rl.regionServerRemoved(getZNodeName(path));<a name="line.124"></a>
-<span class="sourceLineNo">125</span> }<a name="line.125"></a>
-<span class="sourceLineNo">126</span> }<a name="line.126"></a>
-<span class="sourceLineNo">127</span><a name="line.127"></a>
-<span class="sourceLineNo">128</span> /**<a name="line.128"></a>
-<span class="sourceLineNo">129</span> * Called when an existing node has a child node added or removed.<a name="line.129"></a>
-<span class="sourceLineNo">130</span> * @param path full path of the node whose children have changed<a name="line.130"></a>
-<span class="sourceLineNo">131</span> */<a name="line.131"></a>
-<span class="sourceLineNo">132</span> @Override<a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void nodeChildrenChanged(String path) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> if (stopper.isStopped()) {<a name="line.134"></a>
-<span class="sourceLineNo">135</span> return;<a name="line.135"></a>
-<span class="sourceLineNo">136</span> }<a name="line.136"></a>
-<span class="sourceLineNo">137</span> refreshListIfRightPath(path);<a name="line.137"></a>
-<span class="sourceLineNo">138</span> }<a name="line.138"></a>
-<span class="sourceLineNo">139</span><a name="line.139"></a>
-<span class="sourceLineNo">140</span> private boolean refreshListIfRightPath(String path) {<a name="line.140"></a>
-<span class="sourceLineNo">141</span> if (!path.startsWith(this.watcher.getZNodePaths().rsZNode)) {<a name="line.141"></a>
-<span class="sourceLineNo">142</span> return false;<a name="line.142"></a>
-<span class="sourceLineNo">143</span> }<a name="line.143"></a>
-<span class="sourceLineNo">144</span> return refreshOtherRegionServersList(true);<a name="line.144"></a>
-<span class="sourceLineNo">145</span> }<a name="line.145"></a>
-<span class="sourceLineNo">146</span> }<a name="line.146"></a>
-<span class="sourceLineNo">147</span><a name="line.147"></a>
-<span class="sourceLineNo">148</span> /**<a name="line.148"></a>
-<span class="sourceLineNo">149</span> * Extracts the znode name of a peer cluster from a ZK path<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * @param fullPath Path to extract the id from<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * @return the id or an empty string if path is invalid<a name="line.151"></a>
-<span class="sourceLineNo">152</span> */<a name="line.152"></a>
-<span class="sourceLineNo">153</span> private String getZNodeName(String fullPath) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span> String[] parts = fullPath.split("/");<a name="line.154"></a>
-<span class="sourceLineNo">155</span> return parts.length > 0 ? parts[parts.length - 1] : "";<a name="line.155"></a>
-<span class="sourceLineNo">156</span> }<a name="line.156"></a>
-<span class="sourceLineNo">157</span><a name="line.157"></a>
-<span class="sourceLineNo">158</span> /**<a name="line.158"></a>
-<span class="sourceLineNo">159</span> * Reads the list of region servers from ZK and atomically clears our local view of it and<a name="line.159"></a>
-<span class="sourceLineNo">160</span> * replaces it with the updated list.<a name="line.160"></a>
-<span class="sourceLineNo">161</span> * @return true if the local list of the other region servers was updated with the ZK data (even<a name="line.161"></a>
-<span class="sourceLineNo">162</span> * if it was empty), false if the data was missing in ZK<a name="line.162"></a>
-<span class="sourceLineNo">163</span> */<a name="line.163"></a>
-<span class="sourceLineNo">164</span> private boolean refreshOtherRegionServersList(boolean watch) {<a name="line.164"></a>
-<span class="sourceLineNo">165</span> List<String> newRsList = getRegisteredRegionServers(watch);<a name="line.165"></a>
-<span class="sourceLineNo">166</span> if (newRsList == null) {<a name="line.166"></a>
-<span class="sourceLineNo">167</span> return false;<a name="line.167"></a>
-<span class="sourceLineNo">168</span> } else {<a name="line.168"></a>
-<span class="sourceLineNo">169</span> synchronized (otherRegionServers) {<a name="line.169"></a>
-<span class="sourceLineNo">170</span> otherRegionServers.clear();<a name="line.170"></a>
-<span class="sourceLineNo">171</span> otherRegionServers.addAll(newRsList);<a name="line.171"></a>
-<span class="sourceLineNo">172</span> }<a name="line.172"></a>
-<span class="sourceLineNo">173</span> }<a name="line.173"></a>
-<span class="sourceLineNo">174</span> return true;<a name="line.174"></a>
-<span class="sourceLineNo">175</span> }<a name="line.175"></a>
-<span class="sourceLineNo">176</span><a name="line.176"></a>
-<span class="sourceLineNo">177</span> /**<a name="line.177"></a>
-<span class="sourceLineNo">178</span> * Get a list of all the other region servers in this cluster and set a watch<a name="line.178"></a>
-<span class="sourceLineNo">179</span> * @return a list of server nanes<a name="line.179"></a>
-<span class="sourceLineNo">180</span> */<a name="line.180"></a>
-<span class="sourceLineNo">181</span> private List<String> getRegisteredRegionServers(boolean watch) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span> List<String> result = null;<a name="line.182"></a>
-<span class="sourceLineNo">183</span> try {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> if (watch) {<a name="line.184"></a>
-<span class="sourceLineNo">185</span> result = ZKUtil.listChildrenAndWatchThem(this.zookeeper,<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.zookeeper.getZNodePaths().rsZNode);<a name="line.186"></a>
-<span class="sourceLineNo">187</span> } else {<a name="line.187"></a>
-<span class="sourceLineNo">188</span> result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.zookeeper.getZNodePaths().rsZNode);<a name="line.188"></a>
-<span class="sourceLineNo">189</span> }<a name="line.189"></a>
-<span class="sourceLineNo">190</span> } catch (KeeperException e) {<a name="line.190"></a>
-<span class="sourceLineNo">191</span> this.abortable.abort("Get list of registered region servers", e);<a name="line.191"></a>
-<span class="sourceLineNo">192</span> }<a name="line.192"></a>
-<span class="sourceLineNo">193</span> return result;<a name="line.193"></a>
-<span class="sourceLineNo">194</span> }<a name="line.194"></a>
-<span class="sourceLineNo">195</span>}<a name="line.195"></a>
+<span class="sourceLineNo">023</span>import java.util.stream.Collectors;<a name="line.23"></a>
+<span class="sourceLineNo">024</span><a name="line.24"></a>
+<span class="sourceLineNo">025</span>import org.apache.hadoop.hbase.Abortable;<a name="line.25"></a>
+<span class="sourceLineNo">026</span>import org.apache.hadoop.hbase.ServerName;<a name="line.26"></a>
+<span class="sourceLineNo">027</span>import org.apache.hadoop.hbase.Stoppable;<a name="line.27"></a>
+<span class="sourceLineNo">028</span>import org.apache.hadoop.hbase.zookeeper.ZKListener;<a name="line.28"></a>
+<span class="sourceLineNo">029</span>import org.apache.hadoop.hbase.zookeeper.ZKUtil;<a name="line.29"></a>
+<span class="sourceLineNo">030</span>import org.apache.hadoop.hbase.zookeeper.ZKWatcher;<a name="line.30"></a>
+<span class="sourceLineNo">031</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.31"></a>
+<span class="sourceLineNo">032</span>import org.apache.zookeeper.KeeperException;<a name="line.32"></a>
+<span class="sourceLineNo">033</span>import org.slf4j.Logger;<a name="line.33"></a>
+<span class="sourceLineNo">034</span>import org.slf4j.LoggerFactory;<a name="line.34"></a>
+<span class="sourceLineNo">035</span><a name="line.35"></a>
+<span class="sourceLineNo">036</span>/**<a name="line.36"></a>
+<span class="sourceLineNo">037</span> * This class is a ZooKeeper implementation of the ReplicationTracker interface. This class is<a name="line.37"></a>
+<span class="sourceLineNo">038</span> * responsible for handling replication events that are defined in the ReplicationListener<a name="line.38"></a>
+<span class="sourceLineNo">039</span> * interface.<a name="line.39"></a>
+<span class="sourceLineNo">040</span> */<a name="line.40"></a>
+<span class="sourceLineNo">041</span>@InterfaceAudience.Private<a name="line.41"></a>
+<span class="sourceLineNo">042</span>public class ReplicationTrackerZKImpl implements ReplicationTracker {<a name="line.42"></a>
+<span class="sourceLineNo">043</span><a name="line.43"></a>
+<span class="sourceLineNo">044</span> private static final Logger LOG = LoggerFactory.getLogger(ReplicationTrackerZKImpl.class);<a name="line.44"></a>
+<span class="sourceLineNo">045</span><a name="line.45"></a>
+<span class="sourceLineNo">046</span> // Zookeeper<a name="line.46"></a>
+<span class="sourceLineNo">047</span> private final ZKWatcher zookeeper;<a name="line.47"></a>
+<span class="sourceLineNo">048</span> // Server to abort.<a name="line.48"></a>
+<span class="sourceLineNo">049</span> private final Abortable abortable;<a name="line.49"></a>
+<span class="sourceLineNo">050</span> // All about stopping<a name="line.50"></a>
+<span class="sourceLineNo">051</span> private final Stoppable stopper;<a name="line.51"></a>
+<span class="sourceLineNo">052</span> // listeners to be notified<a name="line.52"></a>
+<span class="sourceLineNo">053</span> private final List<ReplicationListener> listeners = new CopyOnWriteArrayList<>();<a name="line.53"></a>
+<span class="sourceLineNo">054</span> // List of all the other region servers in this cluster<a name="line.54"></a>
+<span class="sourceLineNo">055</span> private final List<ServerName> otherRegionServers = new ArrayList<>();<a name="line.55"></a>
+<span class="sourceLineNo">056</span><a name="line.56"></a>
+<span class="sourceLineNo">057</span> public ReplicationTrackerZKImpl(ZKWatcher zookeeper, Abortable abortable, Stoppable stopper) {<a name="line.57"></a>
+<span class="sourceLineNo">058</span> this.zookeeper = zookeeper;<a name="line.58"></a>
+<span class="sourceLineNo">059</span> this.abortable = abortable;<a name="line.59"></a>
+<span class="sourceLineNo">060</span> this.stopper = stopper;<a name="line.60"></a>
+<span class="sourceLineNo">061</span> this.zookeeper.registerListener(new OtherRegionServerWatcher(this.zookeeper));<a name="line.61"></a>
+<span class="sourceLineNo">062</span> // watch the changes<a name="line.62"></a>
+<span class="sourceLineNo">063</span> refreshOtherRegionServersList(true);<a name="line.63"></a>
+<span class="sourceLineNo">064</span> }<a name="line.64"></a>
+<span class="sourceLineNo">065</span><a name="line.65"></a>
+<span class="sourceLineNo">066</span> @Override<a name="line.66"></a>
+<span class="sourceLineNo">067</span> public void registerListener(ReplicationListener listener) {<a name="line.67"></a>
+<span class="sourceLineNo">068</span> listeners.add(listener);<a name="line.68"></a>
+<span class="sourceLineNo">069</span> }<a name="line.69"></a>
+<span class="sourceLineNo">070</span><a name="line.70"></a>
+<span class="sourceLineNo">071</span> @Override<a name="line.71"></a>
+<span class="sourceLineNo">072</span> public void removeListener(ReplicationListener listener) {<a name="line.72"></a>
+<span class="sourceLineNo">073</span> listeners.remove(listener);<a name="line.73"></a>
+<span class="sourceLineNo">074</span> }<a name="line.74"></a>
+<span class="sourceLineNo">075</span><a name="line.75"></a>
+<span class="sourceLineNo">076</span> /**<a name="line.76"></a>
+<span class="sourceLineNo">077</span> * Return a snapshot of the current region servers.<a name="line.77"></a>
+<span class="sourceLineNo">078</span> */<a name="line.78"></a>
+<span class="sourceLineNo">079</span> @Override<a name="line.79"></a>
+<span class="sourceLineNo">080</span> public List<ServerName> getListOfRegionServers() {<a name="line.80"></a>
+<span class="sourceLineNo">081</span> refreshOtherRegionServersList(false);<a name="line.81"></a>
+<span class="sourceLineNo">082</span><a name="line.82"></a>
+<span class="sourceLineNo">083</span> List<ServerName> list = null;<a name="line.83"></a>
+<span class="sourceLineNo">084</span> synchronized (otherRegionServers) {<a name="line.84"></a>
+<span class="sourceLineNo">085</span> list = new ArrayList<>(otherRegionServers);<a name="line.85"></a>
+<span class="sourceLineNo">086</span> }<a name="line.86"></a>
+<span class="sourceLineNo">087</span> return list;<a name="line.87"></a>
+<span class="sourceLineNo">088</span> }<a name="line.88"></a>
+<span class="sourceLineNo">089</span><a name="line.89"></a>
+<span class="sourceLineNo">090</span> /**<a name="line.90"></a>
+<span class="sourceLineNo">091</span> * Watcher used to be notified of the other region server's death in the local cluster. It<a name="line.91"></a>
+<span class="sourceLineNo">092</span> * initiates the process to transfer the queues if it is able to grab the lock.<a name="line.92"></a>
+<span class="sourceLineNo">093</span> */<a name="line.93"></a>
+<span class="sourceLineNo">094</span> public class OtherRegionServerWatcher extends ZKListener {<a name="line.94"></a>
+<span class="sourceLineNo">095</span><a name="line.95"></a>
+<span class="sourceLineNo">096</span> /**<a name="line.96"></a>
+<span class="sourceLineNo">097</span> * Construct a ZooKeeper event listener.<a name="line.97"></a>
+<span class="sourceLineNo">098</span> */<a name="line.98"></a>
+<span class="sourceLineNo">099</span> public OtherRegionServerWatcher(ZKWatcher watcher) {<a name="line.99"></a>
+<span class="sourceLineNo">100</span> super(watcher);<a name="line.100"></a>
+<span class="sourceLineNo">101</span> }<a name="line.101"></a>
+<span class="sourceLineNo">102</span><a name="line.102"></a>
+<span class="sourceLineNo">103</span> /**<a name="line.103"></a>
+<span class="sourceLineNo">104</span> * Called when a new node has been created.<a name="line.104"></a>
+<span class="sourceLineNo">105</span> * @param path full path of the new node<a name="line.105"></a>
+<span class="sourceLineNo">106</span> */<a name="line.106"></a>
+<span class="sourceLineNo">107</span> @Override<a name="line.107"></a>
+<span class="sourceLineNo">108</span> public void nodeCreated(String path) {<a name="line.108"></a>
+<span class="sourceLineNo">109</span> refreshListIfRightPath(path);<a name="line.109"></a>
+<span class="sourceLineNo">110</span> }<a name="line.110"></a>
+<span class="sourceLineNo">111</span><a name="line.111"></a>
+<span class="sourceLineNo">112</span> /**<a name="line.112"></a>
+<span class="sourceLineNo">113</span> * Called when a node has been deleted<a name="line.113"></a>
+<span class="sourceLineNo">114</span> * @param path full path of the deleted node<a name="line.114"></a>
+<span class="sourceLineNo">115</span> */<a name="line.115"></a>
+<span class="sourceLineNo">116</span> @Override<a name="line.116"></a>
+<span class="sourceLineNo">117</span> public void nodeDeleted(String path) {<a name="line.117"></a>
+<span class="sourceLineNo">118</span> if (stopper.isStopped()) {<a name="line.118"></a>
+<span class="sourceLineNo">119</span> return;<a name="line.119"></a>
+<span class="sourceLineNo">120</span> }<a name="line.120"></a>
+<span class="sourceLineNo">121</span> boolean cont = refreshListIfRightPath(path);<a name="line.121"></a>
+<span class="sourceLineNo">122</span> if (!cont) {<a name="line.122"></a>
+<span class="sourceLineNo">123</span> return;<a name="line.123"></a>
+<span class="sourceLineNo">124</span> }<a name="line.124"></a>
+<span class="sourceLineNo">125</span> LOG.info(path + " znode expired, triggering replicatorRemoved event");<a name="line.125"></a>
+<span class="sourceLineNo">126</span> for (ReplicationListener rl : listeners) {<a name="line.126"></a>
+<span class="sourceLineNo">127</span> rl.regionServerRemoved(getZNodeName(path));<a name="line.127"></a>
+<span class="sourceLineNo">128</span> }<a name="line.128"></a>
+<span class="sourceLineNo">129</span> }<a name="line.129"></a>
+<span class="sourceLineNo">130</span><a name="line.130"></a>
+<span class="sourceLineNo">131</span> /**<a name="line.131"></a>
+<span class="sourceLineNo">132</span> * Called when an existing node has a child node added or removed.<a name="line.132"></a>
+<span class="sourceLineNo">133</span> * @param path full path of the node whose children have changed<a name="line.133"></a>
+<span class="sourceLineNo">134</span> */<a name="line.134"></a>
+<span class="sourceLineNo">135</span> @Override<a name="line.135"></a>
+<span class="sourceLineNo">136</span> public void nodeChildrenChanged(String path) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> if (stopper.isStopped()) {<a name="line.137"></a>
+<span class="sourceLineNo">138</span> return;<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span> refreshListIfRightPath(path);<a name="line.140"></a>
+<span class="sourceLineNo">141</span> }<a name="line.141"></a>
+<span class="sourceLineNo">142</span><a name="line.142"></a>
+<span class="sourceLineNo">143</span> private boolean refreshListIfRightPath(String path) {<a name="line.143"></a>
+<span class="sourceLineNo">144</span> if (!path.startsWith(this.watcher.getZNodePaths().rsZNode)) {<a name="line.144"></a>
+<span class="sourceLineNo">145</span> return false;<a name="line.145"></a>
+<span class="sourceLineNo">146</span> }<a name="line.146"></a>
+<span class="sourceLineNo">147</span> return refreshOtherRegionServersList(true);<a name="line.147"></a>
+<span class="sourceLineNo">148</span> }<a name="line.148"></a>
+<span class="sourceLineNo">149</span> }<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> /**<a name="line.151"></a>
+<span class="sourceLineNo">152</span> * Extracts the znode name of a peer cluster from a ZK path<a name="line.152"></a>
+<span class="sourceLineNo">153</span> * @param fullPath Path to extract the id from<a name="line.153"></a>
+<span class="sourceLineNo">154</span> * @return the id or an empty string if path is invalid<a name="line.154"></a>
+<span class="sourceLineNo">155</span> */<a name="line.155"></a>
+<span class="sourceLineNo">156</span> private String getZNodeName(String fullPath) {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> String[] parts = fullPath.split("/");<a name="line.157"></a>
+<span class="sourceLineNo">158</span> return parts.length > 0 ? parts[parts.length - 1] : "";<a name="line.158"></a>
+<span class="sourceLineNo">159</span> }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span> /**<a name="line.161"></a>
+<span class="sourceLineNo">162</span> * Reads the list of region servers from ZK and atomically clears our local view of it and<a name="line.162"></a>
+<span class="sourceLineNo">163</span> * replaces it with the updated list.<a name="line.163"></a>
+<span class="sourceLineNo">164</span> * @return true if the local list of the other region servers was updated with the ZK data (even<a name="line.164"></a>
+<span class="sourceLineNo">165</span> * if it was empty), false if the data was missing in ZK<a name="line.165"></a>
+<span class="sourceLineNo">166</span> */<a name="line.166"></a>
+<span class="sourceLineNo">167</span> private boolean refreshOtherRegionServersList(boolean watch) {<a name="line.167"></a>
+<span class="sourceLineNo">168</span> List<ServerName> newRsList = getRegisteredRegionServers(watch);<a name="line.168"></a>
+<span class="sourceLineNo">169</span> if (newRsList == null) {<a name="line.169"></a>
+<span class="sourceLineNo">170</span> return false;<a name="line.170"></a>
+<span class="sourceLineNo">171</span> } else {<a name="line.171"></a>
+<span class="sourceLineNo">172</span> synchronized (otherRegionServers) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> otherRegionServers.clear();<a name="line.173"></a>
+<span class="sourceLineNo">174</span> otherRegionServers.addAll(newRsList);<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span> return true;<a name="line.177"></a>
+<span class="sourceLineNo">178</span> }<a name="line.178"></a>
+<span class="sourceLineNo">179</span><a name="line.179"></a>
+<span class="sourceLineNo">180</span> /**<a name="line.180"></a>
+<span class="sourceLineNo">181</span> * Get a list of all the other region servers in this cluster and set a watch<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * @return a list of server nanes<a name="line.182"></a>
+<span class="sourceLineNo">183</span> */<a name="line.183"></a>
+<span class="sourceLineNo">184</span> private List<ServerName> getRegisteredRegionServers(boolean watch) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span> List<String> result = null;<a name="line.185"></a>
+<span class="sourceLineNo">186</span> try {<a name="line.186"></a>
+<span class="sourceLineNo">187</span> if (watch) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span> result = ZKUtil.listChildrenAndWatchThem(this.zookeeper,<a name="line.188"></a>
+<span class="sourceLineNo">189</span> this.zookeeper.getZNodePaths().rsZNode);<a name="line.189"></a>
+<span class="sourceLineNo">190</span> } else {<a name="line.190"></a>
+<span class="sourceLineNo">191</span> result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.zookeeper.getZNodePaths().rsZNode);<a name="line.191"></a>
+<span class="sourceLineNo">192</span> }<a name="line.192"></a>
+<span class="sourceLineNo">193</span> } catch (KeeperException e) {<a name="line.193"></a>
+<span class="sourceLineNo">194</span> this.abortable.abort("Get list of registered region servers", e);<a name="line.194"></a>
+<span class="sourceLineNo">195</span> }<a name="line.195"></a>
+<span class="sourceLineNo">196</span> return result == null ? null :<a name="line.196"></a>
+<span class="sourceLineNo">197</span> result.stream().map(ServerName::parseServerName).collect(Collectors.toList());<a name="line.197"></a>
+<span class="sourceLineNo">198</span> }<a name="line.198"></a>
+<span class="sourceLineNo">199</span>}<a name="line.199"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.DumpOptions.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.DumpOptions.html
index 5c21ebf..171886f 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.DumpOptions.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.DumpOptions.html
@@ -316,7 +316,7 @@
<span class="sourceLineNo">308</span> queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());<a name="line.308"></a>
<span class="sourceLineNo">309</span> replicationTracker = ReplicationFactory.getReplicationTracker(zkw, new WarnOnlyAbortable(),<a name="line.309"></a>
<span class="sourceLineNo">310</span> new WarnOnlyStoppable());<a name="line.310"></a>
-<span class="sourceLineNo">311</span> Set<String> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());<a name="line.311"></a>
+<span class="sourceLineNo">311</span> Set<ServerName> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());<a name="line.311"></a>
<span class="sourceLineNo">312</span><a name="line.312"></a>
<span class="sourceLineNo">313</span> // Loops each peer on each RS and dumps the queues<a name="line.313"></a>
<span class="sourceLineNo">314</span> List<ServerName> regionservers = queueStorage.getListOfReplicators();<a name="line.314"></a>
@@ -325,7 +325,7 @@
<span class="sourceLineNo">317</span> }<a name="line.317"></a>
<span class="sourceLineNo">318</span> for (ServerName regionserver : regionservers) {<a name="line.318"></a>
<span class="sourceLineNo">319</span> List<String> queueIds = queueStorage.getAllQueues(regionserver);<a name="line.319"></a>
-<span class="sourceLineNo">320</span> if (!liveRegionServers.contains(regionserver.getServerName())) {<a name="line.320"></a>
+<span class="sourceLineNo">320</span> if (!liveRegionServers.contains(regionserver)) {<a name="line.320"></a>
<span class="sourceLineNo">321</span> deadRegionServers.add(regionserver.getServerName());<a name="line.321"></a>
<span class="sourceLineNo">322</span> }<a name="line.322"></a>
<span class="sourceLineNo">323</span> for (String queueId : queueIds) {<a name="line.323"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.WarnOnlyAbortable.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.WarnOnlyAbortable.html
index 5c21ebf..171886f 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.WarnOnlyAbortable.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.WarnOnlyAbortable.html
@@ -316,7 +316,7 @@
<span class="sourceLineNo">308</span> queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());<a name="line.308"></a>
<span class="sourceLineNo">309</span> replicationTracker = ReplicationFactory.getReplicationTracker(zkw, new WarnOnlyAbortable(),<a name="line.309"></a>
<span class="sourceLineNo">310</span> new WarnOnlyStoppable());<a name="line.310"></a>
-<span class="sourceLineNo">311</span> Set<String> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());<a name="line.311"></a>
+<span class="sourceLineNo">311</span> Set<ServerName> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());<a name="line.311"></a>
<span class="sourceLineNo">312</span><a name="line.312"></a>
<span class="sourceLineNo">313</span> // Loops each peer on each RS and dumps the queues<a name="line.313"></a>
<span class="sourceLineNo">314</span> List<ServerName> regionservers = queueStorage.getListOfReplicators();<a name="line.314"></a>
@@ -325,7 +325,7 @@
<span class="sourceLineNo">317</span> }<a name="line.317"></a>
<span class="sourceLineNo">318</span> for (ServerName regionserver : regionservers) {<a name="line.318"></a>
<span class="sourceLineNo">319</span> List<String> queueIds = queueStorage.getAllQueues(regionserver);<a name="line.319"></a>
-<span class="sourceLineNo">320</span> if (!liveRegionServers.contains(regionserver.getServerName())) {<a name="line.320"></a>
+<span class="sourceLineNo">320</span> if (!liveRegionServers.contains(regionserver)) {<a name="line.320"></a>
<span class="sourceLineNo">321</span> deadRegionServers.add(regionserver.getServerName());<a name="line.321"></a>
<span class="sourceLineNo">322</span> }<a name="line.322"></a>
<span class="sourceLineNo">323</span> for (String queueId : queueIds) {<a name="line.323"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.WarnOnlyStoppable.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.WarnOnlyStoppable.html
index 5c21ebf..171886f 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.WarnOnlyStoppable.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.WarnOnlyStoppable.html
@@ -316,7 +316,7 @@
<span class="sourceLineNo">308</span> queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());<a name="line.308"></a>
<span class="sourceLineNo">309</span> replicationTracker = ReplicationFactory.getReplicationTracker(zkw, new WarnOnlyAbortable(),<a name="line.309"></a>
<span class="sourceLineNo">310</span> new WarnOnlyStoppable());<a name="line.310"></a>
-<span class="sourceLineNo">311</span> Set<String> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());<a name="line.311"></a>
+<span class="sourceLineNo">311</span> Set<ServerName> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());<a name="line.311"></a>
<span class="sourceLineNo">312</span><a name="line.312"></a>
<span class="sourceLineNo">313</span> // Loops each peer on each RS and dumps the queues<a name="line.313"></a>
<span class="sourceLineNo">314</span> List<ServerName> regionservers = queueStorage.getListOfReplicators();<a name="line.314"></a>
@@ -325,7 +325,7 @@
<span class="sourceLineNo">317</span> }<a name="line.317"></a>
<span class="sourceLineNo">318</span> for (ServerName regionserver : regionservers) {<a name="line.318"></a>
<span class="sourceLineNo">319</span> List<String> queueIds = queueStorage.getAllQueues(regionserver);<a name="line.319"></a>
-<span class="sourceLineNo">320</span> if (!liveRegionServers.contains(regionserver.getServerName())) {<a name="line.320"></a>
+<span class="sourceLineNo">320</span> if (!liveRegionServers.contains(regionserver)) {<a name="line.320"></a>
<span class="sourceLineNo">321</span> deadRegionServers.add(regionserver.getServerName());<a name="line.321"></a>
<span class="sourceLineNo">322</span> }<a name="line.322"></a>
<span class="sourceLineNo">323</span> for (String queueId : queueIds) {<a name="line.323"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.html
index 5c21ebf..171886f 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.html
@@ -316,7 +316,7 @@
<span class="sourceLineNo">308</span> queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());<a name="line.308"></a>
<span class="sourceLineNo">309</span> replicationTracker = ReplicationFactory.getReplicationTracker(zkw, new WarnOnlyAbortable(),<a name="line.309"></a>
<span class="sourceLineNo">310</span> new WarnOnlyStoppable());<a name="line.310"></a>
-<span class="sourceLineNo">311</span> Set<String> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());<a name="line.311"></a>
+<span class="sourceLineNo">311</span> Set<ServerName> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());<a name="line.311"></a>
<span class="sourceLineNo">312</span><a name="line.312"></a>
<span class="sourceLineNo">313</span> // Loops each peer on each RS and dumps the queues<a name="line.313"></a>
<span class="sourceLineNo">314</span> List<ServerName> regionservers = queueStorage.getListOfReplicators();<a name="line.314"></a>
@@ -325,7 +325,7 @@
<span class="sourceLineNo">317</span> }<a name="line.317"></a>
<span class="sourceLineNo">318</span> for (ServerName regionserver : regionservers) {<a name="line.318"></a>
<span class="sourceLineNo">319</span> List<String> queueIds = queueStorage.getAllQueues(regionserver);<a name="line.319"></a>
-<span class="sourceLineNo">320</span> if (!liveRegionServers.contains(regionserver.getServerName())) {<a name="line.320"></a>
+<span class="sourceLineNo">320</span> if (!liveRegionServers.contains(regionserver)) {<a name="line.320"></a>
<span class="sourceLineNo">321</span> deadRegionServers.add(regionserver.getServerName());<a name="line.321"></a>
<span class="sourceLineNo">322</span> }<a name="line.322"></a>
<span class="sourceLineNo">323</span> for (String queueId : queueIds) {<a name="line.323"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html
index cf69e73..26166d2 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.NodeFailoverWorker.html
@@ -291,964 +291,963 @@
<span class="sourceLineNo">283</span> if (currentReplicators == null || currentReplicators.isEmpty()) {<a name="line.283"></a>
<span class="sourceLineNo">284</span> return;<a name="line.284"></a>
<span class="sourceLineNo">285</span> }<a name="line.285"></a>
-<span class="sourceLineNo">286</span> List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers().stream()<a name="line.286"></a>
-<span class="sourceLineNo">287</span> .map(ServerName::valueOf).collect(Collectors.toList());<a name="line.287"></a>
-<span class="sourceLineNo">288</span> LOG.info(<a name="line.288"></a>
-<span class="sourceLineNo">289</span> "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);<a name="line.289"></a>
-<span class="sourceLineNo">290</span><a name="line.290"></a>
-<span class="sourceLineNo">291</span> // Look if there's anything to process after a restart<a name="line.291"></a>
-<span class="sourceLineNo">292</span> for (ServerName rs : currentReplicators) {<a name="line.292"></a>
-<span class="sourceLineNo">293</span> if (!otherRegionServers.contains(rs)) {<a name="line.293"></a>
-<span class="sourceLineNo">294</span> transferQueues(rs);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> }<a name="line.295"></a>
-<span class="sourceLineNo">296</span> }<a name="line.296"></a>
-<span class="sourceLineNo">297</span> }<a name="line.297"></a>
-<span class="sourceLineNo">298</span><a name="line.298"></a>
-<span class="sourceLineNo">299</span> /**<a name="line.299"></a>
-<span class="sourceLineNo">300</span> * <ol><a name="line.300"></a>
-<span class="sourceLineNo">301</span> * <li>Add peer to replicationPeers</li><a name="line.301"></a>
-<span class="sourceLineNo">302</span> * <li>Add the normal source and related replication queue</li><a name="line.302"></a>
-<span class="sourceLineNo">303</span> * <li>Add HFile Refs</li><a name="line.303"></a>
-<span class="sourceLineNo">304</span> * </ol><a name="line.304"></a>
-<span class="sourceLineNo">305</span> * @param peerId the id of replication peer<a name="line.305"></a>
-<span class="sourceLineNo">306</span> */<a name="line.306"></a>
-<span class="sourceLineNo">307</span> public void addPeer(String peerId) throws IOException {<a name="line.307"></a>
-<span class="sourceLineNo">308</span> boolean added = false;<a name="line.308"></a>
-<span class="sourceLineNo">309</span> try {<a name="line.309"></a>
-<span class="sourceLineNo">310</span> added = this.replicationPeers.addPeer(peerId);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> } catch (ReplicationException e) {<a name="line.311"></a>
-<span class="sourceLineNo">312</span> throw new IOException(e);<a name="line.312"></a>
-<span class="sourceLineNo">313</span> }<a name="line.313"></a>
-<span class="sourceLineNo">314</span> if (added) {<a name="line.314"></a>
-<span class="sourceLineNo">315</span> addSource(peerId);<a name="line.315"></a>
-<span class="sourceLineNo">316</span> if (replicationForBulkLoadDataEnabled) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span> throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));<a name="line.317"></a>
-<span class="sourceLineNo">318</span> }<a name="line.318"></a>
-<span class="sourceLineNo">319</span> }<a name="line.319"></a>
-<span class="sourceLineNo">320</span> }<a name="line.320"></a>
-<span class="sourceLineNo">321</span><a name="line.321"></a>
-<span class="sourceLineNo">322</span> /**<a name="line.322"></a>
-<span class="sourceLineNo">323</span> * <ol><a name="line.323"></a>
-<span class="sourceLineNo">324</span> * <li>Remove peer for replicationPeers</li><a name="line.324"></a>
-<span class="sourceLineNo">325</span> * <li>Remove all the recovered sources for the specified id and related replication queues</li><a name="line.325"></a>
-<span class="sourceLineNo">326</span> * <li>Remove the normal source and related replication queue</li><a name="line.326"></a>
-<span class="sourceLineNo">327</span> * <li>Remove HFile Refs</li><a name="line.327"></a>
-<span class="sourceLineNo">328</span> * </ol><a name="line.328"></a>
-<span class="sourceLineNo">329</span> * @param peerId the id of the replication peer<a name="line.329"></a>
-<span class="sourceLineNo">330</span> */<a name="line.330"></a>
-<span class="sourceLineNo">331</span> public void removePeer(String peerId) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> ReplicationPeer peer = replicationPeers.removePeer(peerId);<a name="line.332"></a>
-<span class="sourceLineNo">333</span> String terminateMessage = "Replication stream was removed by a user";<a name="line.333"></a>
-<span class="sourceLineNo">334</span> List<ReplicationSourceInterface> oldSourcesToDelete = new ArrayList<>();<a name="line.334"></a>
-<span class="sourceLineNo">335</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.335"></a>
-<span class="sourceLineNo">336</span> // see NodeFailoverWorker.run<a name="line.336"></a>
-<span class="sourceLineNo">337</span> synchronized (this.oldsources) {<a name="line.337"></a>
-<span class="sourceLineNo">338</span> // First close all the recovered sources for this peer<a name="line.338"></a>
-<span class="sourceLineNo">339</span> for (ReplicationSourceInterface src : oldsources) {<a name="line.339"></a>
-<span class="sourceLineNo">340</span> if (peerId.equals(src.getPeerId())) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span> oldSourcesToDelete.add(src);<a name="line.341"></a>
-<span class="sourceLineNo">342</span> }<a name="line.342"></a>
-<span class="sourceLineNo">343</span> }<a name="line.343"></a>
-<span class="sourceLineNo">344</span> for (ReplicationSourceInterface src : oldSourcesToDelete) {<a name="line.344"></a>
-<span class="sourceLineNo">345</span> src.terminate(terminateMessage);<a name="line.345"></a>
-<span class="sourceLineNo">346</span> removeRecoveredSource(src);<a name="line.346"></a>
-<span class="sourceLineNo">347</span> }<a name="line.347"></a>
-<span class="sourceLineNo">348</span> }<a name="line.348"></a>
-<span class="sourceLineNo">349</span> LOG.info(<a name="line.349"></a>
-<span class="sourceLineNo">350</span> "Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());<a name="line.350"></a>
-<span class="sourceLineNo">351</span> // Now close the normal source for this peer<a name="line.351"></a>
-<span class="sourceLineNo">352</span> ReplicationSourceInterface srcToRemove = this.sources.get(peerId);<a name="line.352"></a>
-<span class="sourceLineNo">353</span> if (srcToRemove != null) {<a name="line.353"></a>
-<span class="sourceLineNo">354</span> srcToRemove.terminate(terminateMessage);<a name="line.354"></a>
-<span class="sourceLineNo">355</span> removeSource(srcToRemove);<a name="line.355"></a>
-<span class="sourceLineNo">356</span> } else {<a name="line.356"></a>
-<span class="sourceLineNo">357</span> // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup<a name="line.357"></a>
-<span class="sourceLineNo">358</span> // Delete queue from storage and memory and queue id is same with peer id for normal<a name="line.358"></a>
-<span class="sourceLineNo">359</span> // source<a name="line.359"></a>
-<span class="sourceLineNo">360</span> deleteQueue(peerId);<a name="line.360"></a>
-<span class="sourceLineNo">361</span> this.walsById.remove(peerId);<a name="line.361"></a>
-<span class="sourceLineNo">362</span> }<a name="line.362"></a>
-<span class="sourceLineNo">363</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> if (peerConfig.isSyncReplication()) {<a name="line.364"></a>
-<span class="sourceLineNo">365</span> syncReplicationPeerMappingManager.remove(peerId, peerConfig);<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> // Remove HFile Refs<a name="line.367"></a>
-<span class="sourceLineNo">368</span> abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span> /**<a name="line.371"></a>
-<span class="sourceLineNo">372</span> * @return a new 'classic' user-space replication source.<a name="line.372"></a>
-<span class="sourceLineNo">373</span> * @param queueId the id of the replication queue to associate the ReplicationSource with.<a name="line.373"></a>
-<span class="sourceLineNo">374</span> * @see #createCatalogReplicationSource(RegionInfo) for creating a ReplicationSource for meta.<a name="line.374"></a>
-<span class="sourceLineNo">375</span> */<a name="line.375"></a>
-<span class="sourceLineNo">376</span> private ReplicationSourceInterface createSource(String queueId, ReplicationPeer replicationPeer)<a name="line.376"></a>
-<span class="sourceLineNo">377</span> throws IOException {<a name="line.377"></a>
-<span class="sourceLineNo">378</span> ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueId);<a name="line.378"></a>
-<span class="sourceLineNo">379</span> // Init the just created replication source. Pass the default walProvider's wal file length<a name="line.379"></a>
-<span class="sourceLineNo">380</span> // provider. Presumption is we replicate user-space Tables only. For hbase:meta region replica<a name="line.380"></a>
-<span class="sourceLineNo">381</span> // replication, see #createCatalogReplicationSource().<a name="line.381"></a>
-<span class="sourceLineNo">382</span> WALFileLengthProvider walFileLengthProvider =<a name="line.382"></a>
-<span class="sourceLineNo">383</span> this.walFactory.getWALProvider() != null?<a name="line.383"></a>
-<span class="sourceLineNo">384</span> this.walFactory.getWALProvider().getWALFileLengthProvider() : p -> OptionalLong.empty();<a name="line.384"></a>
-<span class="sourceLineNo">385</span> src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId,<a name="line.385"></a>
-<span class="sourceLineNo">386</span> walFileLengthProvider, new MetricsSource(queueId));<a name="line.386"></a>
-<span class="sourceLineNo">387</span> return src;<a name="line.387"></a>
-<span class="sourceLineNo">388</span> }<a name="line.388"></a>
-<span class="sourceLineNo">389</span><a name="line.389"></a>
-<span class="sourceLineNo">390</span> /**<a name="line.390"></a>
-<span class="sourceLineNo">391</span> * Add a normal source for the given peer on this region server. Meanwhile, add new replication<a name="line.391"></a>
-<span class="sourceLineNo">392</span> * queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal<a name="line.392"></a>
-<span class="sourceLineNo">393</span> * group and do replication<a name="line.393"></a>
-<span class="sourceLineNo">394</span> * @param peerId the id of the replication peer<a name="line.394"></a>
-<span class="sourceLineNo">395</span> * @return the source that was created<a name="line.395"></a>
-<span class="sourceLineNo">396</span> */<a name="line.396"></a>
-<span class="sourceLineNo">397</span> ReplicationSourceInterface addSource(String peerId) throws IOException {<a name="line.397"></a>
-<span class="sourceLineNo">398</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.398"></a>
-<span class="sourceLineNo">399</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.399"></a>
-<span class="sourceLineNo">400</span> // synchronized on latestPaths to avoid missing the new log<a name="line.400"></a>
-<span class="sourceLineNo">401</span> synchronized (this.latestPaths) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> this.sources.put(peerId, src);<a name="line.402"></a>
-<span class="sourceLineNo">403</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.403"></a>
-<span class="sourceLineNo">404</span> this.walsById.put(peerId, walsByGroup);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> // Add the latest wal to that source's queue<a name="line.405"></a>
-<span class="sourceLineNo">406</span> if (!latestPaths.isEmpty()) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span> for (Map.Entry<String, Path> walPrefixAndPath : latestPaths.entrySet()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> Path walPath = walPrefixAndPath.getValue();<a name="line.408"></a>
-<span class="sourceLineNo">409</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.409"></a>
-<span class="sourceLineNo">410</span> wals.add(walPath.getName());<a name="line.410"></a>
-<span class="sourceLineNo">411</span> walsByGroup.put(walPrefixAndPath.getKey(), wals);<a name="line.411"></a>
-<span class="sourceLineNo">412</span> // Abort RS and throw exception to make add peer failed<a name="line.412"></a>
-<span class="sourceLineNo">413</span> abortAndThrowIOExceptionWhenFail(<a name="line.413"></a>
-<span class="sourceLineNo">414</span> () -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName()));<a name="line.414"></a>
-<span class="sourceLineNo">415</span> src.enqueueLog(walPath);<a name="line.415"></a>
-<span class="sourceLineNo">416</span> LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());<a name="line.416"></a>
-<span class="sourceLineNo">417</span> }<a name="line.417"></a>
-<span class="sourceLineNo">418</span> }<a name="line.418"></a>
-<span class="sourceLineNo">419</span> }<a name="line.419"></a>
-<span class="sourceLineNo">420</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.420"></a>
-<span class="sourceLineNo">421</span> if (peerConfig.isSyncReplication()) {<a name="line.421"></a>
-<span class="sourceLineNo">422</span> syncReplicationPeerMappingManager.add(peer.getId(), peerConfig);<a name="line.422"></a>
-<span class="sourceLineNo">423</span> }<a name="line.423"></a>
-<span class="sourceLineNo">424</span> src.startup();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> return src;<a name="line.425"></a>
-<span class="sourceLineNo">426</span> }<a name="line.426"></a>
-<span class="sourceLineNo">427</span><a name="line.427"></a>
-<span class="sourceLineNo">428</span> /**<a name="line.428"></a>
-<span class="sourceLineNo">429</span> * <p><a name="line.429"></a>
-<span class="sourceLineNo">430</span> * This is used when we transit a sync replication peer to {@link SyncReplicationState#STANDBY}.<a name="line.430"></a>
-<span class="sourceLineNo">431</span> * </p><a name="line.431"></a>
-<span class="sourceLineNo">432</span> * <p><a name="line.432"></a>
-<span class="sourceLineNo">433</span> * When transiting to {@link SyncReplicationState#STANDBY}, we can remove all the pending wal<a name="line.433"></a>
-<span class="sourceLineNo">434</span> * files for a replication peer as we do not need to replicate them any more. And this is<a name="line.434"></a>
-<span class="sourceLineNo">435</span> * necessary, otherwise when we transit back to {@link SyncReplicationState#DOWNGRADE_ACTIVE}<a name="line.435"></a>
-<span class="sourceLineNo">436</span> * later, the stale data will be replicated again and cause inconsistency.<a name="line.436"></a>
-<span class="sourceLineNo">437</span> * </p><a name="line.437"></a>
-<span class="sourceLineNo">438</span> * <p><a name="line.438"></a>
-<span class="sourceLineNo">439</span> * See HBASE-20426 for more details.<a name="line.439"></a>
-<span class="sourceLineNo">440</span> * </p><a name="line.440"></a>
-<span class="sourceLineNo">441</span> * @param peerId the id of the sync replication peer<a name="line.441"></a>
-<span class="sourceLineNo">442</span> */<a name="line.442"></a>
-<span class="sourceLineNo">443</span> public void drainSources(String peerId) throws IOException, ReplicationException {<a name="line.443"></a>
-<span class="sourceLineNo">444</span> String terminateMessage = "Sync replication peer " + peerId +<a name="line.444"></a>
-<span class="sourceLineNo">445</span> " is transiting to STANDBY. Will close the previous replication source and open a new one";<a name="line.445"></a>
-<span class="sourceLineNo">446</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.446"></a>
-<span class="sourceLineNo">447</span> assert peer.getPeerConfig().isSyncReplication();<a name="line.447"></a>
-<span class="sourceLineNo">448</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.448"></a>
-<span class="sourceLineNo">449</span> // synchronized here to avoid race with preLogRoll where we add new log to source and also<a name="line.449"></a>
-<span class="sourceLineNo">450</span> // walsById.<a name="line.450"></a>
-<span class="sourceLineNo">451</span> ReplicationSourceInterface toRemove;<a name="line.451"></a>
-<span class="sourceLineNo">452</span> Map<String, NavigableSet<String>> wals = new HashMap<>();<a name="line.452"></a>
-<span class="sourceLineNo">453</span> synchronized (latestPaths) {<a name="line.453"></a>
-<span class="sourceLineNo">454</span> toRemove = sources.put(peerId, src);<a name="line.454"></a>
-<span class="sourceLineNo">455</span> if (toRemove != null) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.456"></a>
-<span class="sourceLineNo">457</span> toRemove.terminate(terminateMessage);<a name="line.457"></a>
-<span class="sourceLineNo">458</span> toRemove.getSourceMetrics().clear();<a name="line.458"></a>
-<span class="sourceLineNo">459</span> }<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // Here we make a copy of all the remaining wal files and then delete them from the<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // replication queue storage after releasing the lock. It is not safe to just remove the old<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // map from walsById since later we may fail to delete them from the replication queue<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // storage, and when we retry next time, we can not know the wal files that need to be deleted<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // from the replication queue storage.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> walsById.get(peerId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));<a name="line.465"></a>
-<span class="sourceLineNo">466</span> }<a name="line.466"></a>
-<span class="sourceLineNo">467</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.467"></a>
-<span class="sourceLineNo">468</span> src.startup();<a name="line.468"></a>
-<span class="sourceLineNo">469</span> for (NavigableSet<String> walsByGroup : wals.values()) {<a name="line.469"></a>
-<span class="sourceLineNo">470</span> for (String wal : walsByGroup) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> queueStorage.removeWAL(server.getServerName(), peerId, wal);<a name="line.471"></a>
-<span class="sourceLineNo">472</span> }<a name="line.472"></a>
-<span class="sourceLineNo">473</span> }<a name="line.473"></a>
-<span class="sourceLineNo">474</span> synchronized (walsById) {<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Map<String, NavigableSet<String>> oldWals = walsById.get(peerId);<a name="line.475"></a>
-<span class="sourceLineNo">476</span> wals.forEach((k, v) -> {<a name="line.476"></a>
-<span class="sourceLineNo">477</span> NavigableSet<String> walsByGroup = oldWals.get(k);<a name="line.477"></a>
-<span class="sourceLineNo">478</span> if (walsByGroup != null) {<a name="line.478"></a>
-<span class="sourceLineNo">479</span> walsByGroup.removeAll(v);<a name="line.479"></a>
-<span class="sourceLineNo">480</span> }<a name="line.480"></a>
-<span class="sourceLineNo">481</span> });<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // synchronized on oldsources to avoid race with NodeFailoverWorker. Since NodeFailoverWorker is<a name="line.483"></a>
-<span class="sourceLineNo">484</span> // a background task, we will delete the file from replication queue storage under the lock to<a name="line.484"></a>
-<span class="sourceLineNo">485</span> // simplify the logic.<a name="line.485"></a>
-<span class="sourceLineNo">486</span> synchronized (this.oldsources) {<a name="line.486"></a>
-<span class="sourceLineNo">487</span> for (Iterator<ReplicationSourceInterface> iter = oldsources.iterator(); iter.hasNext();) {<a name="line.487"></a>
-<span class="sourceLineNo">488</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.488"></a>
-<span class="sourceLineNo">489</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.489"></a>
-<span class="sourceLineNo">490</span> String queueId = oldSource.getQueueId();<a name="line.490"></a>
-<span class="sourceLineNo">491</span> oldSource.terminate(terminateMessage);<a name="line.491"></a>
-<span class="sourceLineNo">492</span> oldSource.getSourceMetrics().clear();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> queueStorage.removeQueue(server.getServerName(), queueId);<a name="line.493"></a>
-<span class="sourceLineNo">494</span> walsByIdRecoveredQueues.remove(queueId);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> iter.remove();<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span> }<a name="line.497"></a>
-<span class="sourceLineNo">498</span> }<a name="line.498"></a>
-<span class="sourceLineNo">499</span> }<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span> /**<a name="line.501"></a>
-<span class="sourceLineNo">502</span> * Close the previous replication sources of this peer id and open new sources to trigger the new<a name="line.502"></a>
-<span class="sourceLineNo">503</span> * replication state changes or new replication config changes. Here we don't need to change<a name="line.503"></a>
-<span class="sourceLineNo">504</span> * replication queue storage and only to enqueue all logs to the new replication source<a name="line.504"></a>
-<span class="sourceLineNo">505</span> * @param peerId the id of the replication peer<a name="line.505"></a>
-<span class="sourceLineNo">506</span> */<a name="line.506"></a>
-<span class="sourceLineNo">507</span> public void refreshSources(String peerId) throws IOException {<a name="line.507"></a>
-<span class="sourceLineNo">508</span> String terminateMessage = "Peer " + peerId +<a name="line.508"></a>
-<span class="sourceLineNo">509</span> " state or config changed. Will close the previous replication source and open a new one";<a name="line.509"></a>
-<span class="sourceLineNo">510</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.510"></a>
-<span class="sourceLineNo">511</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.511"></a>
-<span class="sourceLineNo">512</span> // synchronized on latestPaths to avoid missing the new log<a name="line.512"></a>
-<span class="sourceLineNo">513</span> synchronized (this.latestPaths) {<a name="line.513"></a>
-<span class="sourceLineNo">514</span> ReplicationSourceInterface toRemove = this.sources.put(peerId, src);<a name="line.514"></a>
-<span class="sourceLineNo">515</span> if (toRemove != null) {<a name="line.515"></a>
-<span class="sourceLineNo">516</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.516"></a>
-<span class="sourceLineNo">517</span> // Do not clear metrics<a name="line.517"></a>
-<span class="sourceLineNo">518</span> toRemove.terminate(terminateMessage, null, false);<a name="line.518"></a>
-<span class="sourceLineNo">519</span> }<a name="line.519"></a>
-<span class="sourceLineNo">520</span> for (NavigableSet<String> walsByGroup : walsById.get(peerId).values()) {<a name="line.520"></a>
-<span class="sourceLineNo">521</span> walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));<a name="line.521"></a>
-<span class="sourceLineNo">522</span> }<a name="line.522"></a>
-<span class="sourceLineNo">523</span> }<a name="line.523"></a>
-<span class="sourceLineNo">524</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.524"></a>
-<span class="sourceLineNo">525</span> src.startup();<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> List<ReplicationSourceInterface> toStartup = new ArrayList<>();<a name="line.527"></a>
-<span class="sourceLineNo">528</span> // synchronized on oldsources to avoid race with NodeFailoverWorker<a name="line.528"></a>
-<span class="sourceLineNo">529</span> synchronized (this.oldsources) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span> List<String> previousQueueIds = new ArrayList<>();<a name="line.530"></a>
-<span class="sourceLineNo">531</span> for (Iterator<ReplicationSourceInterface> iter = this.oldsources.iterator(); iter<a name="line.531"></a>
-<span class="sourceLineNo">532</span> .hasNext();) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.534"></a>
-<span class="sourceLineNo">535</span> previousQueueIds.add(oldSource.getQueueId());<a name="line.535"></a>
-<span class="sourceLineNo">536</span> oldSource.terminate(terminateMessage);<a name="line.536"></a>
-<span class="sourceLineNo">537</span> iter.remove();<a name="line.537"></a>
-<span class="sourceLineNo">538</span> }<a name="line.538"></a>
-<span class="sourceLineNo">539</span> }<a name="line.539"></a>
-<span class="sourceLineNo">540</span> for (String queueId : previousQueueIds) {<a name="line.540"></a>
-<span class="sourceLineNo">541</span> ReplicationSourceInterface recoveredReplicationSource = createSource(queueId, peer);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> this.oldsources.add(recoveredReplicationSource);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {<a name="line.543"></a>
-<span class="sourceLineNo">544</span> walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span> toStartup.add(recoveredReplicationSource);<a name="line.546"></a>
-<span class="sourceLineNo">547</span> }<a name="line.547"></a>
-<span class="sourceLineNo">548</span> }<a name="line.548"></a>
-<span class="sourceLineNo">549</span> for (ReplicationSourceInterface replicationSource : toStartup) {<a name="line.549"></a>
-<span class="sourceLineNo">550</span> replicationSource.startup();<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span> }<a name="line.552"></a>
-<span class="sourceLineNo">553</span><a name="line.553"></a>
-<span class="sourceLineNo">554</span> /**<a name="line.554"></a>
-<span class="sourceLineNo">555</span> * Clear the metrics and related replication queue of the specified old source<a name="line.555"></a>
-<span class="sourceLineNo">556</span> * @param src source to clear<a name="line.556"></a>
-<span class="sourceLineNo">557</span> */<a name="line.557"></a>
-<span class="sourceLineNo">558</span> private boolean removeRecoveredSource(ReplicationSourceInterface src) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> if (!this.oldsources.remove(src)) {<a name="line.559"></a>
-<span class="sourceLineNo">560</span> return false;<a name="line.560"></a>
-<span class="sourceLineNo">561</span> }<a name="line.561"></a>
-<span class="sourceLineNo">562</span> LOG.info("Done with the recovered queue {}", src.getQueueId());<a name="line.562"></a>
-<span class="sourceLineNo">563</span> // Delete queue from storage and memory<a name="line.563"></a>
-<span class="sourceLineNo">564</span> deleteQueue(src.getQueueId());<a name="line.564"></a>
-<span class="sourceLineNo">565</span> this.walsByIdRecoveredQueues.remove(src.getQueueId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> return true;<a name="line.566"></a>
-<span class="sourceLineNo">567</span> }<a name="line.567"></a>
-<span class="sourceLineNo">568</span><a name="line.568"></a>
-<span class="sourceLineNo">569</span> void finishRecoveredSource(ReplicationSourceInterface src) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span> synchronized (oldsources) {<a name="line.570"></a>
-<span class="sourceLineNo">571</span> if (!removeRecoveredSource(src)) {<a name="line.571"></a>
-<span class="sourceLineNo">572</span> return;<a name="line.572"></a>
-<span class="sourceLineNo">573</span> }<a name="line.573"></a>
-<span class="sourceLineNo">574</span> }<a name="line.574"></a>
-<span class="sourceLineNo">575</span> LOG.info("Finished recovering queue {} with the following stats: {}", src.getQueueId(),<a name="line.575"></a>
-<span class="sourceLineNo">576</span> src.getStats());<a name="line.576"></a>
-<span class="sourceLineNo">577</span> }<a name="line.577"></a>
-<span class="sourceLineNo">578</span><a name="line.578"></a>
-<span class="sourceLineNo">579</span> /**<a name="line.579"></a>
-<span class="sourceLineNo">580</span> * Clear the metrics and related replication queue of the specified old source<a name="line.580"></a>
-<span class="sourceLineNo">581</span> * @param src source to clear<a name="line.581"></a>
-<span class="sourceLineNo">582</span> */<a name="line.582"></a>
-<span class="sourceLineNo">583</span> void removeSource(ReplicationSourceInterface src) {<a name="line.583"></a>
-<span class="sourceLineNo">584</span> LOG.info("Done with the queue " + src.getQueueId());<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.sources.remove(src.getPeerId());<a name="line.585"></a>
-<span class="sourceLineNo">586</span> // Delete queue from storage and memory<a name="line.586"></a>
-<span class="sourceLineNo">587</span> deleteQueue(src.getQueueId());<a name="line.587"></a>
-<span class="sourceLineNo">588</span> this.walsById.remove(src.getQueueId());<a name="line.588"></a>
-<span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span> }<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> /**<a name="line.592"></a>
-<span class="sourceLineNo">593</span> * Delete a complete queue of wals associated with a replication source<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * @param queueId the id of replication queue to delete<a name="line.594"></a>
-<span class="sourceLineNo">595</span> */<a name="line.595"></a>
-<span class="sourceLineNo">596</span> private void deleteQueue(String queueId) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span> abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.597"></a>
-<span class="sourceLineNo">598</span> }<a name="line.598"></a>
-<span class="sourceLineNo">599</span><a name="line.599"></a>
-<span class="sourceLineNo">600</span> @FunctionalInterface<a name="line.600"></a>
-<span class="sourceLineNo">601</span> private interface ReplicationQueueOperation {<a name="line.601"></a>
-<span class="sourceLineNo">602</span> void exec() throws ReplicationException;<a name="line.602"></a>
-<span class="sourceLineNo">603</span> }<a name="line.603"></a>
-<span class="sourceLineNo">604</span><a name="line.604"></a>
-<span class="sourceLineNo">605</span> /**<a name="line.605"></a>
-<span class="sourceLineNo">606</span> * Refresh replication source will terminate the old source first, then the source thread will be<a name="line.606"></a>
-<span class="sourceLineNo">607</span> * interrupted. Need to handle it instead of abort the region server.<a name="line.607"></a>
-<span class="sourceLineNo">608</span> */<a name="line.608"></a>
-<span class="sourceLineNo">609</span> private void interruptOrAbortWhenFail(ReplicationQueueOperation op) {<a name="line.609"></a>
-<span class="sourceLineNo">610</span> try {<a name="line.610"></a>
-<span class="sourceLineNo">611</span> op.exec();<a name="line.611"></a>
-<span class="sourceLineNo">612</span> } catch (ReplicationException e) {<a name="line.612"></a>
-<span class="sourceLineNo">613</span> if (e.getCause() != null && e.getCause() instanceof KeeperException.SystemErrorException<a name="line.613"></a>
-<span class="sourceLineNo">614</span> && e.getCause().getCause() != null && e.getCause()<a name="line.614"></a>
-<span class="sourceLineNo">615</span> .getCause() instanceof InterruptedException) {<a name="line.615"></a>
-<span class="sourceLineNo">616</span> // ReplicationRuntimeException(a RuntimeException) is thrown out here. The reason is<a name="line.616"></a>
-<span class="sourceLineNo">617</span> // that thread is interrupted deep down in the stack, it should pass the following<a name="line.617"></a>
-<span class="sourceLineNo">618</span> // processing logic and propagate to the most top layer which can handle this exception<a name="line.618"></a>
-<span class="sourceLineNo">619</span> // properly. In this specific case, the top layer is ReplicationSourceShipper#run().<a name="line.619"></a>
-<span class="sourceLineNo">620</span> throw new ReplicationRuntimeException(<a name="line.620"></a>
-<span class="sourceLineNo">621</span> "Thread is interrupted, the replication source may be terminated",<a name="line.621"></a>
-<span class="sourceLineNo">622</span> e.getCause().getCause());<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span> server.abort("Failed to operate on replication queue", e);<a name="line.624"></a>
-<span class="sourceLineNo">625</span> }<a name="line.625"></a>
-<span class="sourceLineNo">626</span> }<a name="line.626"></a>
-<span class="sourceLineNo">627</span><a name="line.627"></a>
-<span class="sourceLineNo">628</span> private void abortWhenFail(ReplicationQueueOperation op) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> op.exec();<a name="line.630"></a>
-<span class="sourceLineNo">631</span> } catch (ReplicationException e) {<a name="line.631"></a>
-<span class="sourceLineNo">632</span> server.abort("Failed to operate on replication queue", e);<a name="line.632"></a>
-<span class="sourceLineNo">633</span> }<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span><a name="line.635"></a>
-<span class="sourceLineNo">636</span> private void throwIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.636"></a>
-<span class="sourceLineNo">637</span> try {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> op.exec();<a name="line.638"></a>
-<span class="sourceLineNo">639</span> } catch (ReplicationException e) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> throw new IOException(e);<a name="line.640"></a>
-<span class="sourceLineNo">641</span> }<a name="line.641"></a>
-<span class="sourceLineNo">642</span> }<a name="line.642"></a>
-<span class="sourceLineNo">643</span><a name="line.643"></a>
-<span class="sourceLineNo">644</span> private void abortAndThrowIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> try {<a name="line.645"></a>
-<span class="sourceLineNo">646</span> op.exec();<a name="line.646"></a>
-<span class="sourceLineNo">647</span> } catch (ReplicationException e) {<a name="line.647"></a>
-<span class="sourceLineNo">648</span> server.abort("Failed to operate on replication queue", e);<a name="line.648"></a>
-<span class="sourceLineNo">649</span> throw new IOException(e);<a name="line.649"></a>
-<span class="sourceLineNo">650</span> }<a name="line.650"></a>
-<span class="sourceLineNo">651</span> }<a name="line.651"></a>
-<span class="sourceLineNo">652</span><a name="line.652"></a>
-<span class="sourceLineNo">653</span> /**<a name="line.653"></a>
-<span class="sourceLineNo">654</span> * This method will log the current position to storage. And also clean old logs from the<a name="line.654"></a>
-<span class="sourceLineNo">655</span> * replication queue.<a name="line.655"></a>
-<span class="sourceLineNo">656</span> * @param source the replication source<a name="line.656"></a>
-<span class="sourceLineNo">657</span> * @param entryBatch the wal entry batch we just shipped<a name="line.657"></a>
-<span class="sourceLineNo">658</span> */<a name="line.658"></a>
-<span class="sourceLineNo">659</span> public void logPositionAndCleanOldLogs(ReplicationSourceInterface source,<a name="line.659"></a>
-<span class="sourceLineNo">660</span> WALEntryBatch entryBatch) {<a name="line.660"></a>
-<span class="sourceLineNo">661</span> String fileName = entryBatch.getLastWalPath().getName();<a name="line.661"></a>
-<span class="sourceLineNo">662</span> interruptOrAbortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(),<a name="line.662"></a>
-<span class="sourceLineNo">663</span> source.getQueueId(), fileName, entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds()));<a name="line.663"></a>
-<span class="sourceLineNo">664</span> cleanOldLogs(fileName, entryBatch.isEndOfFile(), source);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> }<a name="line.665"></a>
-<span class="sourceLineNo">666</span><a name="line.666"></a>
-<span class="sourceLineNo">667</span> /**<a name="line.667"></a>
-<span class="sourceLineNo">668</span> * Cleans a log file and all older logs from replication queue. Called when we are sure that a log<a name="line.668"></a>
-<span class="sourceLineNo">669</span> * file is closed and has no more entries.<a name="line.669"></a>
-<span class="sourceLineNo">670</span> * @param log Path to the log<a name="line.670"></a>
-<span class="sourceLineNo">671</span> * @param inclusive whether we should also remove the given log file<a name="line.671"></a>
-<span class="sourceLineNo">672</span> * @param source the replication source<a name="line.672"></a>
-<span class="sourceLineNo">673</span> */<a name="line.673"></a>
-<span class="sourceLineNo">674</span> void cleanOldLogs(String log, boolean inclusive, ReplicationSourceInterface source) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(log);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> if (source.isRecovered()) {<a name="line.676"></a>
-<span class="sourceLineNo">677</span> NavigableSet<String> wals = walsByIdRecoveredQueues.get(source.getQueueId()).get(logPrefix);<a name="line.677"></a>
-<span class="sourceLineNo">678</span> if (wals != null) {<a name="line.678"></a>
-<span class="sourceLineNo">679</span> NavigableSet<String> walsToRemove = wals.headSet(log, inclusive);<a name="line.679"></a>
-<span class="sourceLineNo">680</span> if (walsToRemove.isEmpty()) {<a name="line.680"></a>
-<span class="sourceLineNo">681</span> return;<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> cleanOldLogs(walsToRemove, source);<a name="line.683"></a>
-<span class="sourceLineNo">684</span> walsToRemove.clear();<a name="line.684"></a>
-<span class="sourceLineNo">685</span> }<a name="line.685"></a>
-<span class="sourceLineNo">686</span> } else {<a name="line.686"></a>
-<span class="sourceLineNo">687</span> NavigableSet<String> wals;<a name="line.687"></a>
-<span class="sourceLineNo">688</span> NavigableSet<String> walsToRemove;<a name="line.688"></a>
-<span class="sourceLineNo">689</span> // synchronized on walsById to avoid race with preLogRoll<a name="line.689"></a>
-<span class="sourceLineNo">690</span> synchronized (this.walsById) {<a name="line.690"></a>
-<span class="sourceLineNo">691</span> wals = walsById.get(source.getQueueId()).get(logPrefix);<a name="line.691"></a>
-<span class="sourceLineNo">692</span> if (wals == null) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span> return;<a name="line.693"></a>
-<span class="sourceLineNo">694</span> }<a name="line.694"></a>
-<span class="sourceLineNo">695</span> walsToRemove = wals.headSet(log, inclusive);<a name="line.695"></a>
-<span class="sourceLineNo">696</span> if (walsToRemove.isEmpty()) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span> return;<a name="line.697"></a>
-<span class="sourceLineNo">698</span> }<a name="line.698"></a>
-<span class="sourceLineNo">699</span> walsToRemove = new TreeSet<>(walsToRemove);<a name="line.699"></a>
-<span class="sourceLineNo">700</span> }<a name="line.700"></a>
-<span class="sourceLineNo">701</span> // cleanOldLogs may spend some time, especially for sync replication where we may want to<a name="line.701"></a>
-<span class="sourceLineNo">702</span> // remove remote wals as the remote cluster may have already been down, so we do it outside<a name="line.702"></a>
-<span class="sourceLineNo">703</span> // the lock to avoid block preLogRoll<a name="line.703"></a>
-<span class="sourceLineNo">704</span> cleanOldLogs(walsToRemove, source);<a name="line.704"></a>
-<span class="sourceLineNo">705</span> // now let's remove the files in the set<a name="line.705"></a>
-<span class="sourceLineNo">706</span> synchronized (this.walsById) {<a name="line.706"></a>
-<span class="sourceLineNo">707</span> wals.removeAll(walsToRemove);<a name="line.707"></a>
-<span class="sourceLineNo">708</span> }<a name="line.708"></a>
-<span class="sourceLineNo">709</span> }<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> private void removeRemoteWALs(String peerId, String remoteWALDir, Collection<String> wals)<a name="line.712"></a>
-<span class="sourceLineNo">713</span> throws IOException {<a name="line.713"></a>
-<span class="sourceLineNo">714</span> Path remoteWALDirForPeer = ReplicationUtils.getPeerRemoteWALDir(remoteWALDir, peerId);<a name="line.714"></a>
-<span class="sourceLineNo">715</span> FileSystem fs = ReplicationUtils.getRemoteWALFileSystem(conf, remoteWALDir);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> for (String wal : wals) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> Path walFile = new Path(remoteWALDirForPeer, wal);<a name="line.717"></a>
-<span class="sourceLineNo">718</span> try {<a name="line.718"></a>
-<span class="sourceLineNo">719</span> if (!fs.delete(walFile, false) && fs.exists(walFile)) {<a name="line.719"></a>
-<span class="sourceLineNo">720</span> throw new IOException("Can not delete " + walFile);<a name="line.720"></a>
-<span class="sourceLineNo">721</span> }<a name="line.721"></a>
-<span class="sourceLineNo">722</span> } catch (FileNotFoundException e) {<a name="line.722"></a>
-<span class="sourceLineNo">723</span> // Just ignore since this means the file has already been deleted.<a name="line.723"></a>
-<span class="sourceLineNo">724</span> // The javadoc of the FileSystem.delete methods does not specify the behavior of deleting an<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // inexistent file, so here we deal with both, i.e, check the return value of the<a name="line.725"></a>
-<span class="sourceLineNo">726</span> // FileSystem.delete, and also catch FNFE.<a name="line.726"></a>
-<span class="sourceLineNo">727</span> LOG.debug("The remote wal {} has already been deleted?", walFile, e);<a name="line.727"></a>
-<span class="sourceLineNo">728</span> }<a name="line.728"></a>
-<span class="sourceLineNo">729</span> }<a name="line.729"></a>
-<span class="sourceLineNo">730</span> }<a name="line.730"></a>
-<span class="sourceLineNo">731</span><a name="line.731"></a>
-<span class="sourceLineNo">732</span> private void cleanOldLogs(NavigableSet<String> wals, ReplicationSourceInterface source) {<a name="line.732"></a>
-<span class="sourceLineNo">733</span> LOG.debug("Removing {} logs in the list: {}", wals.size(), wals);<a name="line.733"></a>
-<span class="sourceLineNo">734</span> // The intention here is that, we want to delete the remote wal files ASAP as it may effect the<a name="line.734"></a>
-<span class="sourceLineNo">735</span> // failover time if you want to transit the remote cluster from S to A. And the infinite retry<a name="line.735"></a>
-<span class="sourceLineNo">736</span> // is not a problem, as if we can not contact with the remote HDFS cluster, then usually we can<a name="line.736"></a>
-<span class="sourceLineNo">737</span> // not contact with the HBase cluster either, so the replication will be blocked either.<a name="line.737"></a>
-<span class="sourceLineNo">738</span> if (source.isSyncReplication()) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span> String peerId = source.getPeerId();<a name="line.739"></a>
-<span class="sourceLineNo">740</span> String remoteWALDir = source.getPeer().getPeerConfig().getRemoteWALDir();<a name="line.740"></a>
-<span class="sourceLineNo">741</span> // Filter out the wals need to be removed from the remote directory. Its name should be the<a name="line.741"></a>
-<span class="sourceLineNo">742</span> // special format, and also, the peer id in its name should match the peer id for the<a name="line.742"></a>
-<span class="sourceLineNo">743</span> // replication source.<a name="line.743"></a>
-<span class="sourceLineNo">744</span> List<String> remoteWals = wals.stream().filter(w -> SyncReplicationWALProvider<a name="line.744"></a>
-<span class="sourceLineNo">745</span> .getSyncReplicationPeerIdFromWALName(w).map(peerId::equals).orElse(false))<a name="line.745"></a>
-<span class="sourceLineNo">746</span> .collect(Collectors.toList());<a name="line.746"></a>
-<span class="sourceLineNo">747</span> LOG.debug("Removing {} logs from remote dir {} in the list: {}", remoteWals.size(),<a name="line.747"></a>
-<span class="sourceLineNo">748</span> remoteWALDir, remoteWals);<a name="line.748"></a>
-<span class="sourceLineNo">749</span> if (!remoteWals.isEmpty()) {<a name="line.749"></a>
-<span class="sourceLineNo">750</span> for (int sleepMultiplier = 0;;) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> try {<a name="line.751"></a>
-<span class="sourceLineNo">752</span> removeRemoteWALs(peerId, remoteWALDir, remoteWals);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> break;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> LOG.warn("Failed to delete remote wals from remote dir {} for peer {}", remoteWALDir,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> peerId);<a name="line.756"></a>
-<span class="sourceLineNo">757</span> }<a name="line.757"></a>
-<span class="sourceLineNo">758</span> if (!source.isSourceActive()) {<a name="line.758"></a>
-<span class="sourceLineNo">759</span> // skip the following operations<a name="line.759"></a>
-<span class="sourceLineNo">760</span> return;<a name="line.760"></a>
-<span class="sourceLineNo">761</span> }<a name="line.761"></a>
-<span class="sourceLineNo">762</span> if (ReplicationUtils.sleepForRetries("Failed to delete remote wals", sleepForRetries,<a name="line.762"></a>
-<span class="sourceLineNo">763</span> sleepMultiplier, maxRetriesMultiplier)) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> sleepMultiplier++;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> }<a name="line.766"></a>
-<span class="sourceLineNo">767</span> }<a name="line.767"></a>
-<span class="sourceLineNo">768</span> }<a name="line.768"></a>
-<span class="sourceLineNo">769</span> String queueId = source.getQueueId();<a name="line.769"></a>
-<span class="sourceLineNo">770</span> for (String wal : wals) {<a name="line.770"></a>
-<span class="sourceLineNo">771</span> interruptOrAbortWhenFail(<a name="line.771"></a>
-<span class="sourceLineNo">772</span> () -> this.queueStorage.removeWAL(server.getServerName(), queueId, wal));<a name="line.772"></a>
-<span class="sourceLineNo">773</span> }<a name="line.773"></a>
-<span class="sourceLineNo">774</span> }<a name="line.774"></a>
-<span class="sourceLineNo">775</span><a name="line.775"></a>
-<span class="sourceLineNo">776</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.776"></a>
-<span class="sourceLineNo">777</span> public void preLogRoll(Path newLog) throws IOException {<a name="line.777"></a>
-<span class="sourceLineNo">778</span> String logName = newLog.getName();<a name="line.778"></a>
-<span class="sourceLineNo">779</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);<a name="line.779"></a>
-<span class="sourceLineNo">780</span> // synchronized on latestPaths to avoid the new open source miss the new log<a name="line.780"></a>
-<span class="sourceLineNo">781</span> synchronized (this.latestPaths) {<a name="line.781"></a>
-<span class="sourceLineNo">782</span> // Add log to queue storage<a name="line.782"></a>
-<span class="sourceLineNo">783</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span> // If record log to queue storage failed, abort RS and throw exception to make log roll<a name="line.784"></a>
-<span class="sourceLineNo">785</span> // failed<a name="line.785"></a>
-<span class="sourceLineNo">786</span> abortAndThrowIOExceptionWhenFail(<a name="line.786"></a>
-<span class="sourceLineNo">787</span> () -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));<a name="line.787"></a>
-<span class="sourceLineNo">788</span> }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span> // synchronized on walsById to avoid race with cleanOldLogs<a name="line.790"></a>
-<span class="sourceLineNo">791</span> synchronized (this.walsById) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span> // Update walsById map<a name="line.792"></a>
-<span class="sourceLineNo">793</span> for (Map.Entry<String, Map<String, NavigableSet<String>>> entry : this.walsById<a name="line.793"></a>
-<span class="sourceLineNo">794</span> .entrySet()) {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> String peerId = entry.getKey();<a name="line.795"></a>
-<span class="sourceLineNo">796</span> Map<String, NavigableSet<String>> walsByPrefix = entry.getValue();<a name="line.796"></a>
-<span class="sourceLineNo">797</span> boolean existingPrefix = false;<a name="line.797"></a>
-<span class="sourceLineNo">798</span> for (Map.Entry<String, NavigableSet<String>> walsEntry : walsByPrefix.entrySet()) {<a name="line.798"></a>
-<span class="sourceLineNo">799</span> SortedSet<String> wals = walsEntry.getValue();<a name="line.799"></a>
-<span class="sourceLineNo">800</span> if (this.sources.isEmpty()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span> // If there's no slaves, don't need to keep the old wals since<a name="line.801"></a>
-<span class="sourceLineNo">802</span> // we only consider the last one when a new slave comes in<a name="line.802"></a>
-<span class="sourceLineNo">803</span> wals.clear();<a name="line.803"></a>
-<span class="sourceLineNo">804</span> }<a name="line.804"></a>
-<span class="sourceLineNo">805</span> if (logPrefix.equals(walsEntry.getKey())) {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> wals.add(logName);<a name="line.806"></a>
-<span class="sourceLineNo">807</span> existingPrefix = true;<a name="line.807"></a>
-<span class="sourceLineNo">808</span> }<a name="line.808"></a>
-<span class="sourceLineNo">809</span> }<a name="line.809"></a>
-<span class="sourceLineNo">810</span> if (!existingPrefix) {<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // The new log belongs to a new group, add it into this peer<a name="line.811"></a>
-<span class="sourceLineNo">812</span> LOG.debug("Start tracking logs for wal group {} for peer {}", logPrefix, peerId);<a name="line.812"></a>
-<span class="sourceLineNo">813</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.813"></a>
-<span class="sourceLineNo">814</span> wals.add(logName);<a name="line.814"></a>
-<span class="sourceLineNo">815</span> walsByPrefix.put(logPrefix, wals);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span> }<a name="line.817"></a>
-<span class="sourceLineNo">818</span> }<a name="line.818"></a>
-<span class="sourceLineNo">819</span><a name="line.819"></a>
-<span class="sourceLineNo">820</span> // Add to latestPaths<a name="line.820"></a>
-<span class="sourceLineNo">821</span> latestPaths.put(logPrefix, newLog);<a name="line.821"></a>
-<span class="sourceLineNo">822</span> }<a name="line.822"></a>
-<span class="sourceLineNo">823</span> }<a name="line.823"></a>
-<span class="sourceLineNo">824</span><a name="line.824"></a>
-<span class="sourceLineNo">825</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.825"></a>
-<span class="sourceLineNo">826</span> public void postLogRoll(Path newLog) throws IOException {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> // This only updates the sources we own, not the recovered ones<a name="line.827"></a>
-<span class="sourceLineNo">828</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.828"></a>
-<span class="sourceLineNo">829</span> source.enqueueLog(newLog);<a name="line.829"></a>
-<span class="sourceLineNo">830</span> LOG.trace("Enqueued {} to source {} while performing postLogRoll operation.",<a name="line.830"></a>
-<span class="sourceLineNo">831</span> newLog, source.getQueueId());<a name="line.831"></a>
-<span class="sourceLineNo">832</span> }<a name="line.832"></a>
-<span class="sourceLineNo">833</span> }<a name="line.833"></a>
-<span class="sourceLineNo">834</span><a name="line.834"></a>
-<span class="sourceLineNo">835</span> @Override<a name="line.835"></a>
-<span class="sourceLineNo">836</span> public void regionServerRemoved(String regionserver) {<a name="line.836"></a>
-<span class="sourceLineNo">837</span> transferQueues(ServerName.valueOf(regionserver));<a name="line.837"></a>
-<span class="sourceLineNo">838</span> }<a name="line.838"></a>
-<span class="sourceLineNo">839</span><a name="line.839"></a>
-<span class="sourceLineNo">840</span> /**<a name="line.840"></a>
-<span class="sourceLineNo">841</span> * Transfer all the queues of the specified to this region server. First it tries to grab a lock<a name="line.841"></a>
-<span class="sourceLineNo">842</span> * and if it works it will move the old queues and finally will delete the old queues.<a name="line.842"></a>
-<span class="sourceLineNo">843</span> * <p><a name="line.843"></a>
-<span class="sourceLineNo">844</span> * It creates one old source for any type of source of the old rs.<a name="line.844"></a>
-<span class="sourceLineNo">845</span> */<a name="line.845"></a>
-<span class="sourceLineNo">846</span> private void transferQueues(ServerName deadRS) {<a name="line.846"></a>
-<span class="sourceLineNo">847</span> if (server.getServerName().equals(deadRS)) {<a name="line.847"></a>
-<span class="sourceLineNo">848</span> // it's just us, give up<a name="line.848"></a>
-<span class="sourceLineNo">849</span> return;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> }<a name="line.850"></a>
-<span class="sourceLineNo">851</span> NodeFailoverWorker transfer = new NodeFailoverWorker(deadRS);<a name="line.851"></a>
-<span class="sourceLineNo">852</span> try {<a name="line.852"></a>
-<span class="sourceLineNo">853</span> this.executor.execute(transfer);<a name="line.853"></a>
-<span class="sourceLineNo">854</span> } catch (RejectedExecutionException ex) {<a name="line.854"></a>
-<span class="sourceLineNo">855</span> CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)<a name="line.855"></a>
-<span class="sourceLineNo">856</span> .getGlobalSource().incrFailedRecoveryQueue();<a name="line.856"></a>
-<span class="sourceLineNo">857</span> LOG.info("Cancelling the transfer of " + deadRS + " because of " + ex.getMessage());<a name="line.857"></a>
-<span class="sourceLineNo">858</span> }<a name="line.858"></a>
-<span class="sourceLineNo">859</span> }<a name="line.859"></a>
-<span class="sourceLineNo">860</span><a name="line.860"></a>
-<span class="sourceLineNo">861</span> /**<a name="line.861"></a>
-<span class="sourceLineNo">862</span> * Class responsible to setup new ReplicationSources to take care of the queues from dead region<a name="line.862"></a>
-<span class="sourceLineNo">863</span> * servers.<a name="line.863"></a>
-<span class="sourceLineNo">864</span> */<a name="line.864"></a>
-<span class="sourceLineNo">865</span> class NodeFailoverWorker extends Thread {<a name="line.865"></a>
-<span class="sourceLineNo">866</span><a name="line.866"></a>
-<span class="sourceLineNo">867</span> private final ServerName deadRS;<a name="line.867"></a>
-<span class="sourceLineNo">868</span> // After claim the queues from dead region server, the NodeFailoverWorker will skip to start<a name="line.868"></a>
-<span class="sourceLineNo">869</span> // the RecoveredReplicationSource if the peer has been removed. but there's possible that<a name="line.869"></a>
-<span class="sourceLineNo">870</span> // remove a peer with peerId = 2 and add a peer with peerId = 2 again during the<a name="line.870"></a>
-<span class="sourceLineNo">871</span> // NodeFailoverWorker. So we need a deep copied <peerId, peer> map to decide whether we<a name="line.871"></a>
-<span class="sourceLineNo">872</span> // should start the RecoveredReplicationSource. If the latest peer is not the old peer when<a name="line.872"></a>
-<span class="sourceLineNo">873</span> // NodeFailoverWorker begin, we should skip to start the RecoveredReplicationSource, Otherwise<a name="line.873"></a>
-<span class="sourceLineNo">874</span> // the rs will abort (See HBASE-20475).<a name="line.874"></a>
-<span class="sourceLineNo">875</span> private final Map<String, ReplicationPeerImpl> peersSnapshot;<a name="line.875"></a>
-<span class="sourceLineNo">876</span><a name="line.876"></a>
-<span class="sourceLineNo">877</span> public NodeFailoverWorker(ServerName deadRS) {<a name="line.877"></a>
-<span class="sourceLineNo">878</span> super("Failover-for-" + deadRS);<a name="line.878"></a>
-<span class="sourceLineNo">879</span> this.deadRS = deadRS;<a name="line.879"></a>
-<span class="sourceLineNo">880</span> peersSnapshot = new HashMap<>(replicationPeers.getPeerCache());<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span><a name="line.882"></a>
-<span class="sourceLineNo">883</span> private boolean isOldPeer(String peerId, ReplicationPeerImpl newPeerRef) {<a name="line.883"></a>
-<span class="sourceLineNo">884</span> ReplicationPeerImpl oldPeerRef = peersSnapshot.get(peerId);<a name="line.884"></a>
-<span class="sourceLineNo">885</span> return oldPeerRef != null && oldPeerRef == newPeerRef;<a name="line.885"></a>
-<span class="sourceLineNo">886</span> }<a name="line.886"></a>
-<span class="sourceLineNo">887</span><a name="line.887"></a>
-<span class="sourceLineNo">888</span> @Override<a name="line.888"></a>
-<span class="sourceLineNo">889</span> public void run() {<a name="line.889"></a>
-<span class="sourceLineNo">890</span> // Wait a bit before transferring the queues, we may be shutting down.<a name="line.890"></a>
-<span class="sourceLineNo">891</span> // This sleep may not be enough in some cases.<a name="line.891"></a>
-<span class="sourceLineNo">892</span> try {<a name="line.892"></a>
-<span class="sourceLineNo">893</span> Thread.sleep(sleepBeforeFailover +<a name="line.893"></a>
-<span class="sourceLineNo">894</span> (long) (ThreadLocalRandom.current().nextFloat() * sleepBeforeFailover));<a name="line.894"></a>
-<span class="sourceLineNo">895</span> } catch (InterruptedException e) {<a name="line.895"></a>
-<span class="sourceLineNo">896</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.896"></a>
-<span class="sourceLineNo">897</span> Thread.currentThread().interrupt();<a name="line.897"></a>
-<span class="sourceLineNo">898</span> }<a name="line.898"></a>
-<span class="sourceLineNo">899</span> // We try to lock that rs' queue directory<a name="line.899"></a>
-<span class="sourceLineNo">900</span> if (server.isStopped()) {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> LOG.info("Not transferring queue since we are shutting down");<a name="line.901"></a>
-<span class="sourceLineNo">902</span> return;<a name="line.902"></a>
-<span class="sourceLineNo">903</span> }<a name="line.903"></a>
-<span class="sourceLineNo">904</span> Map<String, Set<String>> newQueues = new HashMap<>();<a name="line.904"></a>
-<span class="sourceLineNo">905</span> try {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> List<String> queues = queueStorage.getAllQueues(deadRS);<a name="line.906"></a>
-<span class="sourceLineNo">907</span> while (!queues.isEmpty()) {<a name="line.907"></a>
-<span class="sourceLineNo">908</span> Pair<String, SortedSet<String>> peer = queueStorage.claimQueue(deadRS,<a name="line.908"></a>
-<span class="sourceLineNo">909</span> queues.get(ThreadLocalRandom.current().nextInt(queues.size())), server.getServerName());<a name="line.909"></a>
-<span class="sourceLineNo">910</span> long sleep = sleepBeforeFailover / 2;<a name="line.910"></a>
-<span class="sourceLineNo">911</span> if (!peer.getSecond().isEmpty()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> newQueues.put(peer.getFirst(), peer.getSecond());<a name="line.912"></a>
-<span class="sourceLineNo">913</span> sleep = sleepBeforeFailover;<a name="line.913"></a>
-<span class="sourceLineNo">914</span> }<a name="line.914"></a>
-<span class="sourceLineNo">915</span> try {<a name="line.915"></a>
-<span class="sourceLineNo">916</span> Thread.sleep(sleep);<a name="line.916"></a>
-<span class="sourceLineNo">917</span> } catch (InterruptedException e) {<a name="line.917"></a>
-<span class="sourceLineNo">918</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.918"></a>
-<span class="sourceLineNo">919</span> Thread.currentThread().interrupt();<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span> queues = queueStorage.getAllQueues(deadRS);<a name="line.921"></a>
-<span class="sourceLineNo">922</span> }<a name="line.922"></a>
-<span class="sourceLineNo">923</span> if (queues.isEmpty()) {<a name="line.923"></a>
-<span class="sourceLineNo">924</span> queueStorage.removeReplicatorIfQueueIsEmpty(deadRS);<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span> } catch (ReplicationException e) {<a name="line.926"></a>
-<span class="sourceLineNo">927</span> LOG.error(String.format("ReplicationException: cannot claim dead region (%s)'s " +<a name="line.927"></a>
-<span class="sourceLineNo">928</span> "replication queue. Znode : (%s)" +<a name="line.928"></a>
-<span class="sourceLineNo">929</span> " Possible solution: check if znode size exceeds jute.maxBuffer value. " +<a name="line.929"></a>
-<span class="sourceLineNo">930</span> " If so, increase it for both client and server side." + e), deadRS,<a name="line.930"></a>
-<span class="sourceLineNo">931</span> queueStorage.getRsNode(deadRS));<a name="line.931"></a>
-<span class="sourceLineNo">932</span> server.abort("Failed to claim queue from dead regionserver.", e);<a name="line.932"></a>
-<span class="sourceLineNo">933</span> return;<a name="line.933"></a>
-<span class="sourceLineNo">934</span> }<a name="line.934"></a>
-<span class="sourceLineNo">935</span> // Copying over the failed queue is completed.<a name="line.935"></a>
-<span class="sourceLineNo">936</span> if (newQueues.isEmpty()) {<a name="line.936"></a>
-<span class="sourceLineNo">937</span> // We either didn't get the lock or the failed region server didn't have any outstanding<a name="line.937"></a>
-<span class="sourceLineNo">938</span> // WALs to replicate, so we are done.<a name="line.938"></a>
-<span class="sourceLineNo">939</span> return;<a name="line.939"></a>
-<span class="sourceLineNo">940</span> }<a name="line.940"></a>
-<span class="sourceLineNo">941</span><a name="line.941"></a>
-<span class="sourceLineNo">942</span> for (Map.Entry<String, Set<String>> entry : newQueues.entrySet()) {<a name="line.942"></a>
-<span class="sourceLineNo">943</span> String queueId = entry.getKey();<a name="line.943"></a>
-<span class="sourceLineNo">944</span> Set<String> walsSet = entry.getValue();<a name="line.944"></a>
-<span class="sourceLineNo">945</span> try {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> // there is not an actual peer defined corresponding to peerId for the failover.<a name="line.946"></a>
-<span class="sourceLineNo">947</span> ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);<a name="line.947"></a>
-<span class="sourceLineNo">948</span> String actualPeerId = replicationQueueInfo.getPeerId();<a name="line.948"></a>
-<span class="sourceLineNo">949</span><a name="line.949"></a>
-<span class="sourceLineNo">950</span> ReplicationPeerImpl peer = replicationPeers.getPeer(actualPeerId);<a name="line.950"></a>
-<span class="sourceLineNo">951</span> if (peer == null || !isOldPeer(actualPeerId, peer)) {<a name="line.951"></a>
-<span class="sourceLineNo">952</span> LOG.warn("Skipping failover for peer {} of node {}, peer is null", actualPeerId,<a name="line.952"></a>
-<span class="sourceLineNo">953</span> deadRS);<a name="line.953"></a>
-<span class="sourceLineNo">954</span> abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.954"></a>
-<span class="sourceLineNo">955</span> continue;<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> if (server instanceof ReplicationSyncUp.DummyServer<a name="line.957"></a>
-<span class="sourceLineNo">958</span> && peer.getPeerState().equals(PeerState.DISABLED)) {<a name="line.958"></a>
-<span class="sourceLineNo">959</span> LOG.warn("Peer {} is disabled. ReplicationSyncUp tool will skip "<a name="line.959"></a>
-<span class="sourceLineNo">960</span> + "replicating data to this peer.",<a name="line.960"></a>
-<span class="sourceLineNo">961</span> actualPeerId);<a name="line.961"></a>
-<span class="sourceLineNo">962</span> continue;<a name="line.962"></a>
-<span class="sourceLineNo">963</span> }<a name="line.963"></a>
-<span class="sourceLineNo">964</span><a name="line.964"></a>
-<span class="sourceLineNo">965</span> ReplicationSourceInterface src = createSource(queueId, peer);<a name="line.965"></a>
-<span class="sourceLineNo">966</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.966"></a>
-<span class="sourceLineNo">967</span> synchronized (oldsources) {<a name="line.967"></a>
-<span class="sourceLineNo">968</span> peer = replicationPeers.getPeer(src.getPeerId());<a name="line.968"></a>
-<span class="sourceLineNo">969</span> if (peer == null || !isOldPeer(src.getPeerId(), peer)) {<a name="line.969"></a>
-<span class="sourceLineNo">970</span> src.terminate("Recovered queue doesn't belong to any current peer");<a name="line.970"></a>
-<span class="sourceLineNo">971</span> deleteQueue(queueId);<a name="line.971"></a>
-<span class="sourceLineNo">972</span> continue;<a name="line.972"></a>
-<span class="sourceLineNo">973</span> }<a name="line.973"></a>
-<span class="sourceLineNo">974</span> // Do not setup recovered queue if a sync replication peer is in STANDBY state, or is<a name="line.974"></a>
-<span class="sourceLineNo">975</span> // transiting to STANDBY state. The only exception is we are in STANDBY state and<a name="line.975"></a>
-<span class="sourceLineNo">976</span> // transiting to DA, under this state we will replay the remote WAL and they need to be<a name="line.976"></a>
-<span class="sourceLineNo">977</span> // replicated back.<a name="line.977"></a>
-<span class="sourceLineNo">978</span> if (peer.getPeerConfig().isSyncReplication()) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> Pair<SyncReplicationState, SyncReplicationState> stateAndNewState =<a name="line.979"></a>
-<span class="sourceLineNo">980</span> peer.getSyncReplicationStateAndNewState();<a name="line.980"></a>
-<span class="sourceLineNo">981</span> if ((stateAndNewState.getFirst().equals(SyncReplicationState.STANDBY) &&<a name="line.981"></a>
-<span class="sourceLineNo">982</span> stateAndNewState.getSecond().equals(SyncReplicationState.NONE)) ||<a name="line.982"></a>
-<span class="sourceLineNo">983</span> stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)) {<a name="line.983"></a>
-<span class="sourceLineNo">984</span> src.terminate("Sync replication peer is in STANDBY state");<a name="line.984"></a>
-<span class="sourceLineNo">985</span> deleteQueue(queueId);<a name="line.985"></a>
-<span class="sourceLineNo">986</span> continue;<a name="line.986"></a>
-<span class="sourceLineNo">987</span> }<a name="line.987"></a>
-<span class="sourceLineNo">988</span> }<a name="line.988"></a>
-<span class="sourceLineNo">989</span> // track sources in walsByIdRecoveredQueues<a name="line.989"></a>
-<span class="sourceLineNo">990</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.990"></a>
-<span class="sourceLineNo">991</span> walsByIdRecoveredQueues.put(queueId, walsByGroup);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> for (String wal : walsSet) {<a name="line.992"></a>
-<span class="sourceLineNo">993</span> String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal);<a name="line.993"></a>
-<span class="sourceLineNo">994</span> NavigableSet<String> wals = walsByGroup.get(walPrefix);<a name="line.994"></a>
-<span class="sourceLineNo">995</span> if (wals == null) {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> wals = new TreeSet<>();<a name="line.996"></a>
-<span class="sourceLineNo">997</span> walsByGroup.put(walPrefix, wals);<a name="line.997"></a>
-<span class="sourceLineNo">998</span> }<a name="line.998"></a>
-<span class="sourceLineNo">999</span> wals.add(wal);<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> }<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> oldsources.add(src);<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span> LOG.info("Added source for recovered queue {}", src.getQueueId());<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> for (String wal : walsSet) {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> LOG.trace("Enqueueing log from recovered queue for source: " + src.getQueueId());<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> src.enqueueLog(new Path(oldLogDir, wal));<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span> }<a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> src.startup();<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> }<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> } catch (IOException e) {<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> // TODO manage it<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span> LOG.error("Failed creating a source", e);<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> }<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> }<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> }<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span><a name="line.1016"></a>
-<span class="sourceLineNo">1017</span> /**<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> * Terminate the replication on this region server<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> */<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> public void join() {<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.executor.shutdown();<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> source.terminate("Region server is closing");<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span> }<a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> }<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span><a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> /**<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> * Get a copy of the wals of the normal sources on this rs<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> * @return a sorted set of wal names<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> */<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> public Map<String, Map<String, NavigableSet<String>>> getWALs() {<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> return Collections.unmodifiableMap(walsById);<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> /**<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> * Get a copy of the wals of the recovered sources on this rs<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> * @return a sorted set of wal names<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> */<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> Map<String, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> return Collections.unmodifiableMap(walsByIdRecoveredQueues);<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> }<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span><a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> /**<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> * Get a list of all the normal sources of this rs<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> * @return list of all normal sources<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> */<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> public List<ReplicationSourceInterface> getSources() {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> return new ArrayList<>(this.sources.values());<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> }<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span><a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> /**<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> * Get a list of all the recovered sources of this rs<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> * @return list of all recovered sources<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> */<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span> public List<ReplicationSourceInterface> getOldSources() {<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> return this.oldsources;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> }<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span><a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> /**<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> * Get the normal source for a given peer<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> * @return the normal source for the give peer if it exists, otherwise null.<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> */<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> public ReplicationSourceInterface getSource(String peerId) {<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span> return this.sources.get(peerId);<a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> }<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span><a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> List<String> getAllQueues() throws IOException {<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span> List<String> allQueues = Collections.emptyList();<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> try {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> allQueues = queueStorage.getAllQueues(server.getServerName());<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> } catch (ReplicationException e) {<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> throw new IOException(e);<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> return allQueues;<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> int getSizeOfLatestPath() {<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> synchronized (latestPaths) {<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> return latestPaths.size();<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> }<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> }<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> Set<Path> getLastestPath() {<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> synchronized (latestPaths) {<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span> return Sets.newHashSet(latestPaths.values());<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> }<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> }<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span><a name="line.1088"></a>
-<span class="sourceLineNo">1089</span> public AtomicLong getTotalBufferUsed() {<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span> return totalBufferUsed;<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span> }<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span><a name="line.1092"></a>
-<span class="sourceLineNo">1093</span> /**<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span> * Returns the maximum size in bytes of edits held in memory which are pending replication<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span> * across all sources inside this RegionServer.<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span> */<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span> public long getTotalBufferLimit() {<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span> return totalBufferLimit;<a name="line.1098"></a>
-<span class="sourceLineNo">1099</span> }<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span><a name="line.1100"></a>
-<span class="sourceLineNo">1101</span> /**<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span> * Get the directory where wals are archived<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span> * @return the directory where wals are archived<a name="line.1103"></a>
-<span class="sourceLineNo">1104</span> */<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span> public Path getOldLogDir() {<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span> return this.oldLogDir;<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span> }<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span><a name="line.1108"></a>
-<span class="sourceLineNo">1109</span> /**<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span> * Get the directory where wals are stored by their RSs<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span> * @return the directory where wals are stored by their RSs<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span> */<a name="line.1112"></a>
-<span class="sourceLineNo">1113</span> public Path getLogDir() {<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span> return this.logDir;<a name="line.1114"></a>
-<span class="sourceLineNo">1115</span> }<a name="line.1115"></a>
-<span class="sourceLineNo">1116</span><a name="line.1116"></a>
-<span class="sourceLineNo">1117</span> /**<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span> * Get the handle on the local file system<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span> * @return Handle on the local file system<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span> */<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span> public FileSystem getFs() {<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span> return this.fs;<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span> }<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span><a name="line.1124"></a>
-<span class="sourceLineNo">1125</span> /**<a name="line.1125"></a>
-<span class="sourceLineNo">1126</span> * Get the ReplicationPeers used by this ReplicationSourceManager<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span> * @return the ReplicationPeers used by this ReplicationSourceManager<a name="line.1127"></a>
-<span class="sourceLineNo">1128</span> */<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span> public ReplicationPeers getReplicationPeers() {<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span> return this.replicationPeers;<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span> }<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span><a name="line.1132"></a>
-<span class="sourceLineNo">1133</span> /**<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span> * Get a string representation of all the sources' metrics<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span> */<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span> public String getStats() {<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span> StringBuilder stats = new StringBuilder();<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span> // Print stats that apply across all Replication Sources<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span> stats.append("Global stats: ");<a name="line.1139"></a>
-<span class="sourceLineNo">1140</span> stats.append("WAL Edits Buffer Used=").append(getTotalBufferUsed().get()).append("B, Limit=")<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span> .append(getTotalBufferLimit()).append("B\n");<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span> stats.append("Normal source for cluster " + source.getPeerId() + ": ");<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span> stats.append(source.getStats() + "\n");<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span> }<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span> for (ReplicationSourceInterface oldSource : oldsources) {<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span> stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerId() + ": ");<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span> stats.append(oldSource.getStats() + "\n");<a name="line.1148"></a>
-<span class="sourceLineNo">1149</span> }<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span> return stats.toString();<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span> }<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span><a name="line.1152"></a>
-<span class="sourceLineNo">1153</span> public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span> throws IOException {<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span> throwIOExceptionWhenFail(() -> source.addHFileRefs(tableName, family, pairs));<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span> }<a name="line.1157"></a>
-<span class="sourceLineNo">1158</span> }<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span><a name="line.1159"></a>
-<span class="sourceLineNo">1160</span> public void cleanUpHFileRefs(String peerId, List<String> files) {<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span> interruptOrAbortWhenFail(() -> this.queueStorage.removeHFileRefs(peerId, files));<a name="line.1161"></a>
-<span class="sourceLineNo">1162</span> }<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span><a name="line.1163"></a>
-<span class="sourceLineNo">1164</span> int activeFailoverTaskCount() {<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span> return executor.getActiveCount();<a name="line.1165"></a>
-<span class="sourceLineNo">1166</span> }<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span><a name="line.1167"></a>
-<span class="sourceLineNo">1168</span> MetricsReplicationGlobalSourceSource getGlobalMetrics() {<a name="line.1168"></a>
-<span class="sourceLineNo">1169</span> return this.globalMetrics;<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span> }<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span><a name="line.1171"></a>
-<span class="sourceLineNo">1172</span> /**<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span> * Add an hbase:meta Catalog replication source. Called on open of an hbase:meta Region.<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span> * Create it once only. If exists already, use the existing one.<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span> * @see #removeCatalogReplicationSource(RegionInfo)<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span> * @see #addSource(String) This is specialization on the addSource method.<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span> */<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span> public ReplicationSourceInterface addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span> throws IOException {<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span> // Poor-man's putIfAbsent<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span> synchronized (this.catalogReplicationSource) {<a name="line.1181"></a>
-<span class="sourceLineNo">1182</span> ReplicationSourceInterface rs = this.catalogReplicationSource.get();<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span> return rs != null ? rs :<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span> this.catalogReplicationSource.getAndSet(createCatalogReplicationSource(regionInfo));<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span> }<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span> }<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span><a name="line.1187"></a>
-<span class="sourceLineNo">1188</span> /**<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span> * Remove the hbase:meta Catalog replication source.<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span> * Called when we close hbase:meta.<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span> * @see #addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span> */<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span> public void removeCatalogReplicationSource(RegionInfo regionInfo) {<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span> // Nothing to do. Leave any CatalogReplicationSource in place in case an hbase:meta Region<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span> // comes back to this server.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span> }<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span><a name="line.1197"></a>
-<span class="sourceLineNo">1198</span> /**<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span> * Create, initialize, and start the Catalog ReplicationSource.<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span> * Presumes called one-time only (caller must ensure one-time only call).<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span> * This ReplicationSource is NOT created via {@link ReplicationSourceFactory}.<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span> * @see #addSource(String) This is a specialization of the addSource call.<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span> * @see #catalogReplicationSource for a note on this ReplicationSource's lifecycle (and more on<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span> * why the special handling).<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span> */<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span> private ReplicationSourceInterface createCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span> throws IOException {<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span> // Instantiate meta walProvider. Instantiated here or over in the #warmupRegion call made by the<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span> // Master on a 'move' operation. Need to do extra work if we did NOT instantiate the provider.<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span> WALProvider walProvider = this.walFactory.getMetaWALProvider();<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span> boolean instantiate = walProvider == null;<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span> if (instantiate) {<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span> walProvider = this.walFactory.getMetaProvider();<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span> }<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span> // Here we do a specialization on what {@link ReplicationSourceFactory} does. There is no need<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span> // for persisting offset into WALs up in zookeeper (via ReplicationQueueInfo) as the catalog<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span> // read replicas feature that makes use of the source does a reset on a crash of the WAL<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span> // source process. See "4.1 Skip maintaining zookeeper replication queue (offsets/WALs)" in the<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span> // design doc attached to HBASE-18070 'Enable memstore replication for meta replica' for detail.<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span> CatalogReplicationSourcePeer peer = new CatalogReplicationSourcePeer(this.conf,<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span> this.clusterId.toString(), "meta_" + ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER);<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span> final ReplicationSourceInterface crs = new CatalogReplicationSource();<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span> crs.init(conf, fs, this, new NoopReplicationQueueStorage(), peer, server, peer.getId(),<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span> clusterId, walProvider.getWALFileLengthProvider(), new MetricsSource(peer.getId()));<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span> // Add listener on the provider so we can pick up the WAL to replicate on roll.<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span> WALActionsListener listener = new WALActionsListener() {<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span> @Override public void postLogRoll(Path oldPath, Path newPath) throws IOException {<a name="line.1227"></a>
-<span class="sourceLineNo">1228</span> crs.enqueueLog(newPath);<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span> }<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span> };<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span> walProvider.addWALActionsListener(listener);<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span> if (!instantiate) {<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span> // If we did not instantiate provider, need to add our listener on already-created WAL<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span> // instance too (listeners are passed by provider to WAL instance on creation but if provider<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span> // created already, our listener add above is missed). And add the current WAL file to the<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span> // Replication Source so it can start replicating it.<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span> WAL wal = walProvider.getWAL(regionInfo);<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span> wal.registerWALActionsListener(listener);<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span> crs.enqueueLog(((AbstractFSWAL)wal).getCurrentFileName());<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span> }<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span> return crs.startup();<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span> }<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>}<a name="line.1243"></a>
+<span class="sourceLineNo">286</span> List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers();<a name="line.286"></a>
+<span class="sourceLineNo">287</span> LOG.info(<a name="line.287"></a>
+<span class="sourceLineNo">288</span> "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);<a name="line.288"></a>
+<span class="sourceLineNo">289</span><a name="line.289"></a>
+<span class="sourceLineNo">290</span> // Look if there's anything to process after a restart<a name="line.290"></a>
+<span class="sourceLineNo">291</span> for (ServerName rs : currentReplicators) {<a name="line.291"></a>
+<span class="sourceLineNo">292</span> if (!otherRegionServers.contains(rs)) {<a name="line.292"></a>
+<span class="sourceLineNo">293</span> transferQueues(rs);<a name="line.293"></a>
+<span class="sourceLineNo">294</span> }<a name="line.294"></a>
+<span class="sourceLineNo">295</span> }<a name="line.295"></a>
+<span class="sourceLineNo">296</span> }<a name="line.296"></a>
+<span class="sourceLineNo">297</span><a name="line.297"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * <ol><a name="line.299"></a>
+<span class="sourceLineNo">300</span> * <li>Add peer to replicationPeers</li><a name="line.300"></a>
+<span class="sourceLineNo">301</span> * <li>Add the normal source and related replication queue</li><a name="line.301"></a>
+<span class="sourceLineNo">302</span> * <li>Add HFile Refs</li><a name="line.302"></a>
+<span class="sourceLineNo">303</span> * </ol><a name="line.303"></a>
+<span class="sourceLineNo">304</span> * @param peerId the id of replication peer<a name="line.304"></a>
+<span class="sourceLineNo">305</span> */<a name="line.305"></a>
+<span class="sourceLineNo">306</span> public void addPeer(String peerId) throws IOException {<a name="line.306"></a>
+<span class="sourceLineNo">307</span> boolean added = false;<a name="line.307"></a>
+<span class="sourceLineNo">308</span> try {<a name="line.308"></a>
+<span class="sourceLineNo">309</span> added = this.replicationPeers.addPeer(peerId);<a name="line.309"></a>
+<span class="sourceLineNo">310</span> } catch (ReplicationException e) {<a name="line.310"></a>
+<span class="sourceLineNo">311</span> throw new IOException(e);<a name="line.311"></a>
+<span class="sourceLineNo">312</span> }<a name="line.312"></a>
+<span class="sourceLineNo">313</span> if (added) {<a name="line.313"></a>
+<span class="sourceLineNo">314</span> addSource(peerId);<a name="line.314"></a>
+<span class="sourceLineNo">315</span> if (replicationForBulkLoadDataEnabled) {<a name="line.315"></a>
+<span class="sourceLineNo">316</span> throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));<a name="line.316"></a>
+<span class="sourceLineNo">317</span> }<a name="line.317"></a>
+<span class="sourceLineNo">318</span> }<a name="line.318"></a>
+<span class="sourceLineNo">319</span> }<a name="line.319"></a>
+<span class="sourceLineNo">320</span><a name="line.320"></a>
+<span class="sourceLineNo">321</span> /**<a name="line.321"></a>
+<span class="sourceLineNo">322</span> * <ol><a name="line.322"></a>
+<span class="sourceLineNo">323</span> * <li>Remove peer for replicationPeers</li><a name="line.323"></a>
+<span class="sourceLineNo">324</span> * <li>Remove all the recovered sources for the specified id and related replication queues</li><a name="line.324"></a>
+<span class="sourceLineNo">325</span> * <li>Remove the normal source and related replication queue</li><a name="line.325"></a>
+<span class="sourceLineNo">326</span> * <li>Remove HFile Refs</li><a name="line.326"></a>
+<span class="sourceLineNo">327</span> * </ol><a name="line.327"></a>
+<span class="sourceLineNo">328</span> * @param peerId the id of the replication peer<a name="line.328"></a>
+<span class="sourceLineNo">329</span> */<a name="line.329"></a>
+<span class="sourceLineNo">330</span> public void removePeer(String peerId) {<a name="line.330"></a>
+<span class="sourceLineNo">331</span> ReplicationPeer peer = replicationPeers.removePeer(peerId);<a name="line.331"></a>
+<span class="sourceLineNo">332</span> String terminateMessage = "Replication stream was removed by a user";<a name="line.332"></a>
+<span class="sourceLineNo">333</span> List<ReplicationSourceInterface> oldSourcesToDelete = new ArrayList<>();<a name="line.333"></a>
+<span class="sourceLineNo">334</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.334"></a>
+<span class="sourceLineNo">335</span> // see NodeFailoverWorker.run<a name="line.335"></a>
+<span class="sourceLineNo">336</span> synchronized (this.oldsources) {<a name="line.336"></a>
+<span class="sourceLineNo">337</span> // First close all the recovered sources for this peer<a name="line.337"></a>
+<span class="sourceLineNo">338</span> for (ReplicationSourceInterface src : oldsources) {<a name="line.338"></a>
+<span class="sourceLineNo">339</span> if (peerId.equals(src.getPeerId())) {<a name="line.339"></a>
+<span class="sourceLineNo">340</span> oldSourcesToDelete.add(src);<a name="line.340"></a>
+<span class="sourceLineNo">341</span> }<a name="line.341"></a>
+<span class="sourceLineNo">342</span> }<a name="line.342"></a>
+<span class="sourceLineNo">343</span> for (ReplicationSourceInterface src : oldSourcesToDelete) {<a name="line.343"></a>
+<span class="sourceLineNo">344</span> src.terminate(terminateMessage);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> removeRecoveredSource(src);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> }<a name="line.346"></a>
+<span class="sourceLineNo">347</span> }<a name="line.347"></a>
+<span class="sourceLineNo">348</span> LOG.info(<a name="line.348"></a>
+<span class="sourceLineNo">349</span> "Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());<a name="line.349"></a>
+<span class="sourceLineNo">350</span> // Now close the normal source for this peer<a name="line.350"></a>
+<span class="sourceLineNo">351</span> ReplicationSourceInterface srcToRemove = this.sources.get(peerId);<a name="line.351"></a>
+<span class="sourceLineNo">352</span> if (srcToRemove != null) {<a name="line.352"></a>
+<span class="sourceLineNo">353</span> srcToRemove.terminate(terminateMessage);<a name="line.353"></a>
+<span class="sourceLineNo">354</span> removeSource(srcToRemove);<a name="line.354"></a>
+<span class="sourceLineNo">355</span> } else {<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // Delete queue from storage and memory and queue id is same with peer id for normal<a name="line.357"></a>
+<span class="sourceLineNo">358</span> // source<a name="line.358"></a>
+<span class="sourceLineNo">359</span> deleteQueue(peerId);<a name="line.359"></a>
+<span class="sourceLineNo">360</span> this.walsById.remove(peerId);<a name="line.360"></a>
+<span class="sourceLineNo">361</span> }<a name="line.361"></a>
+<span class="sourceLineNo">362</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.362"></a>
+<span class="sourceLineNo">363</span> if (peerConfig.isSyncReplication()) {<a name="line.363"></a>
+<span class="sourceLineNo">364</span> syncReplicationPeerMappingManager.remove(peerId, peerConfig);<a name="line.364"></a>
+<span class="sourceLineNo">365</span> }<a name="line.365"></a>
+<span class="sourceLineNo">366</span> // Remove HFile Refs<a name="line.366"></a>
+<span class="sourceLineNo">367</span> abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));<a name="line.367"></a>
+<span class="sourceLineNo">368</span> }<a name="line.368"></a>
+<span class="sourceLineNo">369</span><a name="line.369"></a>
+<span class="sourceLineNo">370</span> /**<a name="line.370"></a>
+<span class="sourceLineNo">371</span> * @return a new 'classic' user-space replication source.<a name="line.371"></a>
+<span class="sourceLineNo">372</span> * @param queueId the id of the replication queue to associate the ReplicationSource with.<a name="line.372"></a>
+<span class="sourceLineNo">373</span> * @see #createCatalogReplicationSource(RegionInfo) for creating a ReplicationSource for meta.<a name="line.373"></a>
+<span class="sourceLineNo">374</span> */<a name="line.374"></a>
+<span class="sourceLineNo">375</span> private ReplicationSourceInterface createSource(String queueId, ReplicationPeer replicationPeer)<a name="line.375"></a>
+<span class="sourceLineNo">376</span> throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueId);<a name="line.377"></a>
+<span class="sourceLineNo">378</span> // Init the just created replication source. Pass the default walProvider's wal file length<a name="line.378"></a>
+<span class="sourceLineNo">379</span> // provider. Presumption is we replicate user-space Tables only. For hbase:meta region replica<a name="line.379"></a>
+<span class="sourceLineNo">380</span> // replication, see #createCatalogReplicationSource().<a name="line.380"></a>
+<span class="sourceLineNo">381</span> WALFileLengthProvider walFileLengthProvider =<a name="line.381"></a>
+<span class="sourceLineNo">382</span> this.walFactory.getWALProvider() != null?<a name="line.382"></a>
+<span class="sourceLineNo">383</span> this.walFactory.getWALProvider().getWALFileLengthProvider() : p -> OptionalLong.empty();<a name="line.383"></a>
+<span class="sourceLineNo">384</span> src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId,<a name="line.384"></a>
+<span class="sourceLineNo">385</span> walFileLengthProvider, new MetricsSource(queueId));<a name="line.385"></a>
+<span class="sourceLineNo">386</span> return src;<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> /**<a name="line.389"></a>
+<span class="sourceLineNo">390</span> * Add a normal source for the given peer on this region server. Meanwhile, add new replication<a name="line.390"></a>
+<span class="sourceLineNo">391</span> * queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal<a name="line.391"></a>
+<span class="sourceLineNo">392</span> * group and do replication<a name="line.392"></a>
+<span class="sourceLineNo">393</span> * @param peerId the id of the replication peer<a name="line.393"></a>
+<span class="sourceLineNo">394</span> * @return the source that was created<a name="line.394"></a>
+<span class="sourceLineNo">395</span> */<a name="line.395"></a>
+<span class="sourceLineNo">396</span> ReplicationSourceInterface addSource(String peerId) throws IOException {<a name="line.396"></a>
+<span class="sourceLineNo">397</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.397"></a>
+<span class="sourceLineNo">398</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.398"></a>
+<span class="sourceLineNo">399</span> // synchronized on latestPaths to avoid missing the new log<a name="line.399"></a>
+<span class="sourceLineNo">400</span> synchronized (this.latestPaths) {<a name="line.400"></a>
+<span class="sourceLineNo">401</span> this.sources.put(peerId, src);<a name="line.401"></a>
+<span class="sourceLineNo">402</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.402"></a>
+<span class="sourceLineNo">403</span> this.walsById.put(peerId, walsByGroup);<a name="line.403"></a>
+<span class="sourceLineNo">404</span> // Add the latest wal to that source's queue<a name="line.404"></a>
+<span class="sourceLineNo">405</span> if (!latestPaths.isEmpty()) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span> for (Map.Entry<String, Path> walPrefixAndPath : latestPaths.entrySet()) {<a name="line.406"></a>
+<span class="sourceLineNo">407</span> Path walPath = walPrefixAndPath.getValue();<a name="line.407"></a>
+<span class="sourceLineNo">408</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.408"></a>
+<span class="sourceLineNo">409</span> wals.add(walPath.getName());<a name="line.409"></a>
+<span class="sourceLineNo">410</span> walsByGroup.put(walPrefixAndPath.getKey(), wals);<a name="line.410"></a>
+<span class="sourceLineNo">411</span> // Abort RS and throw exception to make add peer failed<a name="line.411"></a>
+<span class="sourceLineNo">412</span> abortAndThrowIOExceptionWhenFail(<a name="line.412"></a>
+<span class="sourceLineNo">413</span> () -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName()));<a name="line.413"></a>
+<span class="sourceLineNo">414</span> src.enqueueLog(walPath);<a name="line.414"></a>
+<span class="sourceLineNo">415</span> LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());<a name="line.415"></a>
+<span class="sourceLineNo">416</span> }<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.419"></a>
+<span class="sourceLineNo">420</span> if (peerConfig.isSyncReplication()) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span> syncReplicationPeerMappingManager.add(peer.getId(), peerConfig);<a name="line.421"></a>
+<span class="sourceLineNo">422</span> }<a name="line.422"></a>
+<span class="sourceLineNo">423</span> src.startup();<a name="line.423"></a>
+<span class="sourceLineNo">424</span> return src;<a name="line.424"></a>
+<span class="sourceLineNo">425</span> }<a name="line.425"></a>
+<span class="sourceLineNo">426</span><a name="line.426"></a>
+<span class="sourceLineNo">427</span> /**<a name="line.427"></a>
+<span class="sourceLineNo">428</span> * <p><a name="line.428"></a>
+<span class="sourceLineNo">429</span> * This is used when we transit a sync replication peer to {@link SyncReplicationState#STANDBY}.<a name="line.429"></a>
+<span class="sourceLineNo">430</span> * </p><a name="line.430"></a>
+<span class="sourceLineNo">431</span> * <p><a name="line.431"></a>
+<span class="sourceLineNo">432</span> * When transiting to {@link SyncReplicationState#STANDBY}, we can remove all the pending wal<a name="line.432"></a>
+<span class="sourceLineNo">433</span> * files for a replication peer as we do not need to replicate them any more. And this is<a name="line.433"></a>
+<span class="sourceLineNo">434</span> * necessary, otherwise when we transit back to {@link SyncReplicationState#DOWNGRADE_ACTIVE}<a name="line.434"></a>
+<span class="sourceLineNo">435</span> * later, the stale data will be replicated again and cause inconsistency.<a name="line.435"></a>
+<span class="sourceLineNo">436</span> * </p><a name="line.436"></a>
+<span class="sourceLineNo">437</span> * <p><a name="line.437"></a>
+<span class="sourceLineNo">438</span> * See HBASE-20426 for more details.<a name="line.438"></a>
+<span class="sourceLineNo">439</span> * </p><a name="line.439"></a>
+<span class="sourceLineNo">440</span> * @param peerId the id of the sync replication peer<a name="line.440"></a>
+<span class="sourceLineNo">441</span> */<a name="line.441"></a>
+<span class="sourceLineNo">442</span> public void drainSources(String peerId) throws IOException, ReplicationException {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> String terminateMessage = "Sync replication peer " + peerId +<a name="line.443"></a>
+<span class="sourceLineNo">444</span> " is transiting to STANDBY. Will close the previous replication source and open a new one";<a name="line.444"></a>
+<span class="sourceLineNo">445</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.445"></a>
+<span class="sourceLineNo">446</span> assert peer.getPeerConfig().isSyncReplication();<a name="line.446"></a>
+<span class="sourceLineNo">447</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.447"></a>
+<span class="sourceLineNo">448</span> // synchronized here to avoid race with preLogRoll where we add new log to source and also<a name="line.448"></a>
+<span class="sourceLineNo">449</span> // walsById.<a name="line.449"></a>
+<span class="sourceLineNo">450</span> ReplicationSourceInterface toRemove;<a name="line.450"></a>
+<span class="sourceLineNo">451</span> Map<String, NavigableSet<String>> wals = new HashMap<>();<a name="line.451"></a>
+<span class="sourceLineNo">452</span> synchronized (latestPaths) {<a name="line.452"></a>
+<span class="sourceLineNo">453</span> toRemove = sources.put(peerId, src);<a name="line.453"></a>
+<span class="sourceLineNo">454</span> if (toRemove != null) {<a name="line.454"></a>
+<span class="sourceLineNo">455</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.455"></a>
+<span class="sourceLineNo">456</span> toRemove.terminate(terminateMessage);<a name="line.456"></a>
+<span class="sourceLineNo">457</span> toRemove.getSourceMetrics().clear();<a name="line.457"></a>
+<span class="sourceLineNo">458</span> }<a name="line.458"></a>
+<span class="sourceLineNo">459</span> // Here we make a copy of all the remaining wal files and then delete them from the<a name="line.459"></a>
+<span class="sourceLineNo">460</span> // replication queue storage after releasing the lock. It is not safe to just remove the old<a name="line.460"></a>
+<span class="sourceLineNo">461</span> // map from walsById since later we may fail to delete them from the replication queue<a name="line.461"></a>
+<span class="sourceLineNo">462</span> // storage, and when we retry next time, we can not know the wal files that need to be deleted<a name="line.462"></a>
+<span class="sourceLineNo">463</span> // from the replication queue storage.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> walsById.get(peerId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));<a name="line.464"></a>
+<span class="sourceLineNo">465</span> }<a name="line.465"></a>
+<span class="sourceLineNo">466</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.466"></a>
+<span class="sourceLineNo">467</span> src.startup();<a name="line.467"></a>
+<span class="sourceLineNo">468</span> for (NavigableSet<String> walsByGroup : wals.values()) {<a name="line.468"></a>
+<span class="sourceLineNo">469</span> for (String wal : walsByGroup) {<a name="line.469"></a>
+<span class="sourceLineNo">470</span> queueStorage.removeWAL(server.getServerName(), peerId, wal);<a name="line.470"></a>
+<span class="sourceLineNo">471</span> }<a name="line.471"></a>
+<span class="sourceLineNo">472</span> }<a name="line.472"></a>
+<span class="sourceLineNo">473</span> synchronized (walsById) {<a name="line.473"></a>
+<span class="sourceLineNo">474</span> Map<String, NavigableSet<String>> oldWals = walsById.get(peerId);<a name="line.474"></a>
+<span class="sourceLineNo">475</span> wals.forEach((k, v) -> {<a name="line.475"></a>
+<span class="sourceLineNo">476</span> NavigableSet<String> walsByGroup = oldWals.get(k);<a name="line.476"></a>
+<span class="sourceLineNo">477</span> if (walsByGroup != null) {<a name="line.477"></a>
+<span class="sourceLineNo">478</span> walsByGroup.removeAll(v);<a name="line.478"></a>
+<span class="sourceLineNo">479</span> }<a name="line.479"></a>
+<span class="sourceLineNo">480</span> });<a name="line.480"></a>
+<span class="sourceLineNo">481</span> }<a name="line.481"></a>
+<span class="sourceLineNo">482</span> // synchronized on oldsources to avoid race with NodeFailoverWorker. Since NodeFailoverWorker is<a name="line.482"></a>
+<span class="sourceLineNo">483</span> // a background task, we will delete the file from replication queue storage under the lock to<a name="line.483"></a>
+<span class="sourceLineNo">484</span> // simplify the logic.<a name="line.484"></a>
+<span class="sourceLineNo">485</span> synchronized (this.oldsources) {<a name="line.485"></a>
+<span class="sourceLineNo">486</span> for (Iterator<ReplicationSourceInterface> iter = oldsources.iterator(); iter.hasNext();) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.487"></a>
+<span class="sourceLineNo">488</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.488"></a>
+<span class="sourceLineNo">489</span> String queueId = oldSource.getQueueId();<a name="line.489"></a>
+<span class="sourceLineNo">490</span> oldSource.terminate(terminateMessage);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> oldSource.getSourceMetrics().clear();<a name="line.491"></a>
+<span class="sourceLineNo">492</span> queueStorage.removeQueue(server.getServerName(), queueId);<a name="line.492"></a>
+<span class="sourceLineNo">493</span> walsByIdRecoveredQueues.remove(queueId);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> iter.remove();<a name="line.494"></a>
+<span class="sourceLineNo">495</span> }<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span> }<a name="line.498"></a>
+<span class="sourceLineNo">499</span><a name="line.499"></a>
+<span class="sourceLineNo">500</span> /**<a name="line.500"></a>
+<span class="sourceLineNo">501</span> * Close the previous replication sources of this peer id and open new sources to trigger the new<a name="line.501"></a>
+<span class="sourceLineNo">502</span> * replication state changes or new replication config changes. Here we don't need to change<a name="line.502"></a>
+<span class="sourceLineNo">503</span> * replication queue storage and only to enqueue all logs to the new replication source<a name="line.503"></a>
+<span class="sourceLineNo">504</span> * @param peerId the id of the replication peer<a name="line.504"></a>
+<span class="sourceLineNo">505</span> */<a name="line.505"></a>
+<span class="sourceLineNo">506</span> public void refreshSources(String peerId) throws IOException {<a name="line.506"></a>
+<span class="sourceLineNo">507</span> String terminateMessage = "Peer " + peerId +<a name="line.507"></a>
+<span class="sourceLineNo">508</span> " state or config changed. Will close the previous replication source and open a new one";<a name="line.508"></a>
+<span class="sourceLineNo">509</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.509"></a>
+<span class="sourceLineNo">510</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.510"></a>
+<span class="sourceLineNo">511</span> // synchronized on latestPaths to avoid missing the new log<a name="line.511"></a>
+<span class="sourceLineNo">512</span> synchronized (this.latestPaths) {<a name="line.512"></a>
+<span class="sourceLineNo">513</span> ReplicationSourceInterface toRemove = this.sources.put(peerId, src);<a name="line.513"></a>
+<span class="sourceLineNo">514</span> if (toRemove != null) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.515"></a>
+<span class="sourceLineNo">516</span> // Do not clear metrics<a name="line.516"></a>
+<span class="sourceLineNo">517</span> toRemove.terminate(terminateMessage, null, false);<a name="line.517"></a>
+<span class="sourceLineNo">518</span> }<a name="line.518"></a>
+<span class="sourceLineNo">519</span> for (NavigableSet<String> walsByGroup : walsById.get(peerId).values()) {<a name="line.519"></a>
+<span class="sourceLineNo">520</span> walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));<a name="line.520"></a>
+<span class="sourceLineNo">521</span> }<a name="line.521"></a>
+<span class="sourceLineNo">522</span> }<a name="line.522"></a>
+<span class="sourceLineNo">523</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.523"></a>
+<span class="sourceLineNo">524</span> src.startup();<a name="line.524"></a>
+<span class="sourceLineNo">525</span><a name="line.525"></a>
+<span class="sourceLineNo">526</span> List<ReplicationSourceInterface> toStartup = new ArrayList<>();<a name="line.526"></a>
+<span class="sourceLineNo">527</span> // synchronized on oldsources to avoid race with NodeFailoverWorker<a name="line.527"></a>
+<span class="sourceLineNo">528</span> synchronized (this.oldsources) {<a name="line.528"></a>
+<span class="sourceLineNo">529</span> List<String> previousQueueIds = new ArrayList<>();<a name="line.529"></a>
+<span class="sourceLineNo">530</span> for (Iterator<ReplicationSourceInterface> iter = this.oldsources.iterator(); iter<a name="line.530"></a>
+<span class="sourceLineNo">531</span> .hasNext();) {<a name="line.531"></a>
+<span class="sourceLineNo">532</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.532"></a>
+<span class="sourceLineNo">533</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.533"></a>
+<span class="sourceLineNo">534</span> previousQueueIds.add(oldSource.getQueueId());<a name="line.534"></a>
+<span class="sourceLineNo">535</span> oldSource.terminate(terminateMessage);<a name="line.535"></a>
+<span class="sourceLineNo">536</span> iter.remove();<a name="line.536"></a>
+<span class="sourceLineNo">537</span> }<a name="line.537"></a>
+<span class="sourceLineNo">538</span> }<a name="line.538"></a>
+<span class="sourceLineNo">539</span> for (String queueId : previousQueueIds) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span> ReplicationSourceInterface recoveredReplicationSource = createSource(queueId, peer);<a name="line.540"></a>
+<span class="sourceLineNo">541</span> this.oldsources.add(recoveredReplicationSource);<a name="line.541"></a>
+<span class="sourceLineNo">542</span> for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span> toStartup.add(recoveredReplicationSource);<a name="line.545"></a>
+<span class="sourceLineNo">546</span> }<a name="line.546"></a>
+<span class="sourceLineNo">547</span> }<a name="line.547"></a>
+<span class="sourceLineNo">548</span> for (ReplicationSourceInterface replicationSource : toStartup) {<a name="line.548"></a>
+<span class="sourceLineNo">549</span> replicationSource.startup();<a name="line.549"></a>
+<span class="sourceLineNo">550</span> }<a name="line.550"></a>
+<span class="sourceLineNo">551</span> }<a name="line.551"></a>
+<span class="sourceLineNo">552</span><a name="line.552"></a>
+<span class="sourceLineNo">553</span> /**<a name="line.553"></a>
+<span class="sourceLineNo">554</span> * Clear the metrics and related replication queue of the specified old source<a name="line.554"></a>
+<span class="sourceLineNo">555</span> * @param src source to clear<a name="line.555"></a>
+<span class="sourceLineNo">556</span> */<a name="line.556"></a>
+<span class="sourceLineNo">557</span> private boolean removeRecoveredSource(ReplicationSourceInterface src) {<a name="line.557"></a>
+<span class="sourceLineNo">558</span> if (!this.oldsources.remove(src)) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span> return false;<a name="line.559"></a>
+<span class="sourceLineNo">560</span> }<a name="line.560"></a>
+<span class="sourceLineNo">561</span> LOG.info("Done with the recovered queue {}", src.getQueueId());<a name="line.561"></a>
+<span class="sourceLineNo">562</span> // Delete queue from storage and memory<a name="line.562"></a>
+<span class="sourceLineNo">563</span> deleteQueue(src.getQueueId());<a name="line.563"></a>
+<span class="sourceLineNo">564</span> this.walsByIdRecoveredQueues.remove(src.getQueueId());<a name="line.564"></a>
+<span class="sourceLineNo">565</span> return true;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> }<a name="line.566"></a>
+<span class="sourceLineNo">567</span><a name="line.567"></a>
+<span class="sourceLineNo">568</span> void finishRecoveredSource(ReplicationSourceInterface src) {<a name="line.568"></a>
+<span class="sourceLineNo">569</span> synchronized (oldsources) {<a name="line.569"></a>
+<span class="sourceLineNo">570</span> if (!removeRecoveredSource(src)) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span> return;<a name="line.571"></a>
+<span class="sourceLineNo">572</span> }<a name="line.572"></a>
+<span class="sourceLineNo">573</span> }<a name="line.573"></a>
+<span class="sourceLineNo">574</span> LOG.info("Finished recovering queue {} with the following stats: {}", src.getQueueId(),<a name="line.574"></a>
+<span class="sourceLineNo">575</span> src.getStats());<a name="line.575"></a>
+<span class="sourceLineNo">576</span> }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span> /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span> * Clear the metrics and related replication queue of the specified old source<a name="line.579"></a>
+<span class="sourceLineNo">580</span> * @param src source to clear<a name="line.580"></a>
+<span class="sourceLineNo">581</span> */<a name="line.581"></a>
+<span class="sourceLineNo">582</span> void removeSource(ReplicationSourceInterface src) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.info("Done with the queue " + src.getQueueId());<a name="line.583"></a>
+<span class="sourceLineNo">584</span> this.sources.remove(src.getPeerId());<a name="line.584"></a>
+<span class="sourceLineNo">585</span> // Delete queue from storage and memory<a name="line.585"></a>
+<span class="sourceLineNo">586</span> deleteQueue(src.getQueueId());<a name="line.586"></a>
+<span class="sourceLineNo">587</span> this.walsById.remove(src.getQueueId());<a name="line.587"></a>
+<span class="sourceLineNo">588</span><a name="line.588"></a>
+<span class="sourceLineNo">589</span> }<a name="line.589"></a>
+<span class="sourceLineNo">590</span><a name="line.590"></a>
+<span class="sourceLineNo">591</span> /**<a name="line.591"></a>
+<span class="sourceLineNo">592</span> * Delete a complete queue of wals associated with a replication source<a name="line.592"></a>
+<span class="sourceLineNo">593</span> * @param queueId the id of replication queue to delete<a name="line.593"></a>
+<span class="sourceLineNo">594</span> */<a name="line.594"></a>
+<span class="sourceLineNo">595</span> private void deleteQueue(String queueId) {<a name="line.595"></a>
+<span class="sourceLineNo">596</span> abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.596"></a>
+<span class="sourceLineNo">597</span> }<a name="line.597"></a>
+<span class="sourceLineNo">598</span><a name="line.598"></a>
+<span class="sourceLineNo">599</span> @FunctionalInterface<a name="line.599"></a>
+<span class="sourceLineNo">600</span> private interface ReplicationQueueOperation {<a name="line.600"></a>
+<span class="sourceLineNo">601</span> void exec() throws ReplicationException;<a name="line.601"></a>
+<span class="sourceLineNo">602</span> }<a name="line.602"></a>
+<span class="sourceLineNo">603</span><a name="line.603"></a>
+<span class="sourceLineNo">604</span> /**<a name="line.604"></a>
+<span class="sourceLineNo">605</span> * Refresh replication source will terminate the old source first, then the source thread will be<a name="line.605"></a>
+<span class="sourceLineNo">606</span> * interrupted. Need to handle it instead of abort the region server.<a name="line.606"></a>
+<span class="sourceLineNo">607</span> */<a name="line.607"></a>
+<span class="sourceLineNo">608</span> private void interruptOrAbortWhenFail(ReplicationQueueOperation op) {<a name="line.608"></a>
+<span class="sourceLineNo">609</span> try {<a name="line.609"></a>
+<span class="sourceLineNo">610</span> op.exec();<a name="line.610"></a>
+<span class="sourceLineNo">611</span> } catch (ReplicationException e) {<a name="line.611"></a>
+<span class="sourceLineNo">612</span> if (e.getCause() != null && e.getCause() instanceof KeeperException.SystemErrorException<a name="line.612"></a>
+<span class="sourceLineNo">613</span> && e.getCause().getCause() != null && e.getCause()<a name="line.613"></a>
+<span class="sourceLineNo">614</span> .getCause() instanceof InterruptedException) {<a name="line.614"></a>
+<span class="sourceLineNo">615</span> // ReplicationRuntimeException(a RuntimeException) is thrown out here. The reason is<a name="line.615"></a>
+<span class="sourceLineNo">616</span> // that thread is interrupted deep down in the stack, it should pass the following<a name="line.616"></a>
+<span class="sourceLineNo">617</span> // processing logic and propagate to the most top layer which can handle this exception<a name="line.617"></a>
+<span class="sourceLineNo">618</span> // properly. In this specific case, the top layer is ReplicationSourceShipper#run().<a name="line.618"></a>
+<span class="sourceLineNo">619</span> throw new ReplicationRuntimeException(<a name="line.619"></a>
+<span class="sourceLineNo">620</span> "Thread is interrupted, the replication source may be terminated",<a name="line.620"></a>
+<span class="sourceLineNo">621</span> e.getCause().getCause());<a name="line.621"></a>
+<span class="sourceLineNo">622</span> }<a name="line.622"></a>
+<span class="sourceLineNo">623</span> server.abort("Failed to operate on replication queue", e);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span> }<a name="line.625"></a>
+<span class="sourceLineNo">626</span><a name="line.626"></a>
+<span class="sourceLineNo">627</span> private void abortWhenFail(ReplicationQueueOperation op) {<a name="line.627"></a>
+<span class="sourceLineNo">628</span> try {<a name="line.628"></a>
+<span class="sourceLineNo">629</span> op.exec();<a name="line.629"></a>
+<span class="sourceLineNo">630</span> } catch (ReplicationException e) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> server.abort("Failed to operate on replication queue", e);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> }<a name="line.632"></a>
+<span class="sourceLineNo">633</span> }<a name="line.633"></a>
+<span class="sourceLineNo">634</span><a name="line.634"></a>
+<span class="sourceLineNo">635</span> private void throwIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.635"></a>
+<span class="sourceLineNo">636</span> try {<a name="line.636"></a>
+<span class="sourceLineNo">637</span> op.exec();<a name="line.637"></a>
+<span class="sourceLineNo">638</span> } catch (ReplicationException e) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> throw new IOException(e);<a name="line.639"></a>
+<span class="sourceLineNo">640</span> }<a name="line.640"></a>
+<span class="sourceLineNo">641</span> }<a name="line.641"></a>
+<span class="sourceLineNo">642</span><a name="line.642"></a>
+<span class="sourceLineNo">643</span> private void abortAndThrowIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.643"></a>
+<span class="sourceLineNo">644</span> try {<a name="line.644"></a>
+<span class="sourceLineNo">645</span> op.exec();<a name="line.645"></a>
+<span class="sourceLineNo">646</span> } catch (ReplicationException e) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span> server.abort("Failed to operate on replication queue", e);<a name="line.647"></a>
+<span class="sourceLineNo">648</span> throw new IOException(e);<a name="line.648"></a>
+<span class="sourceLineNo">649</span> }<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * This method will log the current position to storage. And also clean old logs from the<a name="line.653"></a>
+<span class="sourceLineNo">654</span> * replication queue.<a name="line.654"></a>
+<span class="sourceLineNo">655</span> * @param source the replication source<a name="line.655"></a>
+<span class="sourceLineNo">656</span> * @param entryBatch the wal entry batch we just shipped<a name="line.656"></a>
+<span class="sourceLineNo">657</span> */<a name="line.657"></a>
+<span class="sourceLineNo">658</span> public void logPositionAndCleanOldLogs(ReplicationSourceInterface source,<a name="line.658"></a>
+<span class="sourceLineNo">659</span> WALEntryBatch entryBatch) {<a name="line.659"></a>
+<span class="sourceLineNo">660</span> String fileName = entryBatch.getLastWalPath().getName();<a name="line.660"></a>
+<span class="sourceLineNo">661</span> interruptOrAbortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(),<a name="line.661"></a>
+<span class="sourceLineNo">662</span> source.getQueueId(), fileName, entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds()));<a name="line.662"></a>
+<span class="sourceLineNo">663</span> cleanOldLogs(fileName, entryBatch.isEndOfFile(), source);<a name="line.663"></a>
+<span class="sourceLineNo">664</span> }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span> /**<a name="line.666"></a>
+<span class="sourceLineNo">667</span> * Cleans a log file and all older logs from replication queue. Called when we are sure that a log<a name="line.667"></a>
+<span class="sourceLineNo">668</span> * file is closed and has no more entries.<a name="line.668"></a>
+<span class="sourceLineNo">669</span> * @param log Path to the log<a name="line.669"></a>
+<span class="sourceLineNo">670</span> * @param inclusive whether we should also remove the given log file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> * @param source the replication source<a name="line.671"></a>
+<span class="sourceLineNo">672</span> */<a name="line.672"></a>
+<span class="sourceLineNo">673</span> void cleanOldLogs(String log, boolean inclusive, ReplicationSourceInterface source) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(log);<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (source.isRecovered()) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> NavigableSet<String> wals = walsByIdRecoveredQueues.get(source.getQueueId()).get(logPrefix);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> if (wals != null) {<a name="line.677"></a>
+<span class="sourceLineNo">678</span> NavigableSet<String> walsToRemove = wals.headSet(log, inclusive);<a name="line.678"></a>
+<span class="sourceLineNo">679</span> if (walsToRemove.isEmpty()) {<a name="line.679"></a>
+<span class="sourceLineNo">680</span> return;<a name="line.680"></a>
+<span class="sourceLineNo">681</span> }<a name="line.681"></a>
+<span class="sourceLineNo">682</span> cleanOldLogs(walsToRemove, source);<a name="line.682"></a>
+<span class="sourceLineNo">683</span> walsToRemove.clear();<a name="line.683"></a>
+<span class="sourceLineNo">684</span> }<a name="line.684"></a>
+<span class="sourceLineNo">685</span> } else {<a name="line.685"></a>
+<span class="sourceLineNo">686</span> NavigableSet<String> wals;<a name="line.686"></a>
+<span class="sourceLineNo">687</span> NavigableSet<String> walsToRemove;<a name="line.687"></a>
+<span class="sourceLineNo">688</span> // synchronized on walsById to avoid race with preLogRoll<a name="line.688"></a>
+<span class="sourceLineNo">689</span> synchronized (this.walsById) {<a name="line.689"></a>
+<span class="sourceLineNo">690</span> wals = walsById.get(source.getQueueId()).get(logPrefix);<a name="line.690"></a>
+<span class="sourceLineNo">691</span> if (wals == null) {<a name="line.691"></a>
+<span class="sourceLineNo">692</span> return;<a name="line.692"></a>
+<span class="sourceLineNo">693</span> }<a name="line.693"></a>
+<span class="sourceLineNo">694</span> walsToRemove = wals.headSet(log, inclusive);<a name="line.694"></a>
+<span class="sourceLineNo">695</span> if (walsToRemove.isEmpty()) {<a name="line.695"></a>
+<span class="sourceLineNo">696</span> return;<a name="line.696"></a>
+<span class="sourceLineNo">697</span> }<a name="line.697"></a>
+<span class="sourceLineNo">698</span> walsToRemove = new TreeSet<>(walsToRemove);<a name="line.698"></a>
+<span class="sourceLineNo">699</span> }<a name="line.699"></a>
+<span class="sourceLineNo">700</span> // cleanOldLogs may spend some time, especially for sync replication where we may want to<a name="line.700"></a>
+<span class="sourceLineNo">701</span> // remove remote wals as the remote cluster may have already been down, so we do it outside<a name="line.701"></a>
+<span class="sourceLineNo">702</span> // the lock to avoid block preLogRoll<a name="line.702"></a>
+<span class="sourceLineNo">703</span> cleanOldLogs(walsToRemove, source);<a name="line.703"></a>
+<span class="sourceLineNo">704</span> // now let's remove the files in the set<a name="line.704"></a>
+<span class="sourceLineNo">705</span> synchronized (this.walsById) {<a name="line.705"></a>
+<span class="sourceLineNo">706</span> wals.removeAll(walsToRemove);<a name="line.706"></a>
+<span class="sourceLineNo">707</span> }<a name="line.707"></a>
+<span class="sourceLineNo">708</span> }<a name="line.708"></a>
+<span class="sourceLineNo">709</span> }<a name="line.709"></a>
+<span class="sourceLineNo">710</span><a name="line.710"></a>
+<span class="sourceLineNo">711</span> private void removeRemoteWALs(String peerId, String remoteWALDir, Collection<String> wals)<a name="line.711"></a>
+<span class="sourceLineNo">712</span> throws IOException {<a name="line.712"></a>
+<span class="sourceLineNo">713</span> Path remoteWALDirForPeer = ReplicationUtils.getPeerRemoteWALDir(remoteWALDir, peerId);<a name="line.713"></a>
+<span class="sourceLineNo">714</span> FileSystem fs = ReplicationUtils.getRemoteWALFileSystem(conf, remoteWALDir);<a name="line.714"></a>
+<span class="sourceLineNo">715</span> for (String wal : wals) {<a name="line.715"></a>
+<span class="sourceLineNo">716</span> Path walFile = new Path(remoteWALDirForPeer, wal);<a name="line.716"></a>
+<span class="sourceLineNo">717</span> try {<a name="line.717"></a>
+<span class="sourceLineNo">718</span> if (!fs.delete(walFile, false) && fs.exists(walFile)) {<a name="line.718"></a>
+<span class="sourceLineNo">719</span> throw new IOException("Can not delete " + walFile);<a name="line.719"></a>
+<span class="sourceLineNo">720</span> }<a name="line.720"></a>
+<span class="sourceLineNo">721</span> } catch (FileNotFoundException e) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span> // Just ignore since this means the file has already been deleted.<a name="line.722"></a>
+<span class="sourceLineNo">723</span> // The javadoc of the FileSystem.delete methods does not specify the behavior of deleting an<a name="line.723"></a>
+<span class="sourceLineNo">724</span> // inexistent file, so here we deal with both, i.e, check the return value of the<a name="line.724"></a>
+<span class="sourceLineNo">725</span> // FileSystem.delete, and also catch FNFE.<a name="line.725"></a>
+<span class="sourceLineNo">726</span> LOG.debug("The remote wal {} has already been deleted?", walFile, e);<a name="line.726"></a>
+<span class="sourceLineNo">727</span> }<a name="line.727"></a>
+<span class="sourceLineNo">728</span> }<a name="line.728"></a>
+<span class="sourceLineNo">729</span> }<a name="line.729"></a>
+<span class="sourceLineNo">730</span><a name="line.730"></a>
+<span class="sourceLineNo">731</span> private void cleanOldLogs(NavigableSet<String> wals, ReplicationSourceInterface source) {<a name="line.731"></a>
+<span class="sourceLineNo">732</span> LOG.debug("Removing {} logs in the list: {}", wals.size(), wals);<a name="line.732"></a>
+<span class="sourceLineNo">733</span> // The intention here is that, we want to delete the remote wal files ASAP as it may effect the<a name="line.733"></a>
+<span class="sourceLineNo">734</span> // failover time if you want to transit the remote cluster from S to A. And the infinite retry<a name="line.734"></a>
+<span class="sourceLineNo">735</span> // is not a problem, as if we can not contact with the remote HDFS cluster, then usually we can<a name="line.735"></a>
+<span class="sourceLineNo">736</span> // not contact with the HBase cluster either, so the replication will be blocked either.<a name="line.736"></a>
+<span class="sourceLineNo">737</span> if (source.isSyncReplication()) {<a name="line.737"></a>
+<span class="sourceLineNo">738</span> String peerId = source.getPeerId();<a name="line.738"></a>
+<span class="sourceLineNo">739</span> String remoteWALDir = source.getPeer().getPeerConfig().getRemoteWALDir();<a name="line.739"></a>
+<span class="sourceLineNo">740</span> // Filter out the wals need to be removed from the remote directory. Its name should be the<a name="line.740"></a>
+<span class="sourceLineNo">741</span> // special format, and also, the peer id in its name should match the peer id for the<a name="line.741"></a>
+<span class="sourceLineNo">742</span> // replication source.<a name="line.742"></a>
+<span class="sourceLineNo">743</span> List<String> remoteWals = wals.stream().filter(w -> SyncReplicationWALProvider<a name="line.743"></a>
+<span class="sourceLineNo">744</span> .getSyncReplicationPeerIdFromWALName(w).map(peerId::equals).orElse(false))<a name="line.744"></a>
+<span class="sourceLineNo">745</span> .collect(Collectors.toList());<a name="line.745"></a>
+<span class="sourceLineNo">746</span> LOG.debug("Removing {} logs from remote dir {} in the list: {}", remoteWals.size(),<a name="line.746"></a>
+<span class="sourceLineNo">747</span> remoteWALDir, remoteWals);<a name="line.747"></a>
+<span class="sourceLineNo">748</span> if (!remoteWals.isEmpty()) {<a name="line.748"></a>
+<span class="sourceLineNo">749</span> for (int sleepMultiplier = 0;;) {<a name="line.749"></a>
+<span class="sourceLineNo">750</span> try {<a name="line.750"></a>
+<span class="sourceLineNo">751</span> removeRemoteWALs(peerId, remoteWALDir, remoteWals);<a name="line.751"></a>
+<span class="sourceLineNo">752</span> break;<a name="line.752"></a>
+<span class="sourceLineNo">753</span> } catch (IOException e) {<a name="line.753"></a>
+<span class="sourceLineNo">754</span> LOG.warn("Failed to delete remote wals from remote dir {} for peer {}", remoteWALDir,<a name="line.754"></a>
+<span class="sourceLineNo">755</span> peerId);<a name="line.755"></a>
+<span class="sourceLineNo">756</span> }<a name="line.756"></a>
+<span class="sourceLineNo">757</span> if (!source.isSourceActive()) {<a name="line.757"></a>
+<span class="sourceLineNo">758</span> // skip the following operations<a name="line.758"></a>
+<span class="sourceLineNo">759</span> return;<a name="line.759"></a>
+<span class="sourceLineNo">760</span> }<a name="line.760"></a>
+<span class="sourceLineNo">761</span> if (ReplicationUtils.sleepForRetries("Failed to delete remote wals", sleepForRetries,<a name="line.761"></a>
+<span class="sourceLineNo">762</span> sleepMultiplier, maxRetriesMultiplier)) {<a name="line.762"></a>
+<span class="sourceLineNo">763</span> sleepMultiplier++;<a name="line.763"></a>
+<span class="sourceLineNo">764</span> }<a name="line.764"></a>
+<span class="sourceLineNo">765</span> }<a name="line.765"></a>
+<span class="sourceLineNo">766</span> }<a name="line.766"></a>
+<span class="sourceLineNo">767</span> }<a name="line.767"></a>
+<span class="sourceLineNo">768</span> String queueId = source.getQueueId();<a name="line.768"></a>
+<span class="sourceLineNo">769</span> for (String wal : wals) {<a name="line.769"></a>
+<span class="sourceLineNo">770</span> interruptOrAbortWhenFail(<a name="line.770"></a>
+<span class="sourceLineNo">771</span> () -> this.queueStorage.removeWAL(server.getServerName(), queueId, wal));<a name="line.771"></a>
+<span class="sourceLineNo">772</span> }<a name="line.772"></a>
+<span class="sourceLineNo">773</span> }<a name="line.773"></a>
+<span class="sourceLineNo">774</span><a name="line.774"></a>
+<span class="sourceLineNo">775</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.775"></a>
+<span class="sourceLineNo">776</span> public void preLogRoll(Path newLog) throws IOException {<a name="line.776"></a>
+<span class="sourceLineNo">777</span> String logName = newLog.getName();<a name="line.777"></a>
+<span class="sourceLineNo">778</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);<a name="line.778"></a>
+<span class="sourceLineNo">779</span> // synchronized on latestPaths to avoid the new open source miss the new log<a name="line.779"></a>
+<span class="sourceLineNo">780</span> synchronized (this.latestPaths) {<a name="line.780"></a>
+<span class="sourceLineNo">781</span> // Add log to queue storage<a name="line.781"></a>
+<span class="sourceLineNo">782</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.782"></a>
+<span class="sourceLineNo">783</span> // If record log to queue storage failed, abort RS and throw exception to make log roll<a name="line.783"></a>
+<span class="sourceLineNo">784</span> // failed<a name="line.784"></a>
+<span class="sourceLineNo">785</span> abortAndThrowIOExceptionWhenFail(<a name="line.785"></a>
+<span class="sourceLineNo">786</span> () -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));<a name="line.786"></a>
+<span class="sourceLineNo">787</span> }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span> // synchronized on walsById to avoid race with cleanOldLogs<a name="line.789"></a>
+<span class="sourceLineNo">790</span> synchronized (this.walsById) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span> // Update walsById map<a name="line.791"></a>
+<span class="sourceLineNo">792</span> for (Map.Entry<String, Map<String, NavigableSet<String>>> entry : this.walsById<a name="line.792"></a>
+<span class="sourceLineNo">793</span> .entrySet()) {<a name="line.793"></a>
+<span class="sourceLineNo">794</span> String peerId = entry.getKey();<a name="line.794"></a>
+<span class="sourceLineNo">795</span> Map<String, NavigableSet<String>> walsByPrefix = entry.getValue();<a name="line.795"></a>
+<span class="sourceLineNo">796</span> boolean existingPrefix = false;<a name="line.796"></a>
+<span class="sourceLineNo">797</span> for (Map.Entry<String, NavigableSet<String>> walsEntry : walsByPrefix.entrySet()) {<a name="line.797"></a>
+<span class="sourceLineNo">798</span> SortedSet<String> wals = walsEntry.getValue();<a name="line.798"></a>
+<span class="sourceLineNo">799</span> if (this.sources.isEmpty()) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span> // If there's no slaves, don't need to keep the old wals since<a name="line.800"></a>
+<span class="sourceLineNo">801</span> // we only consider the last one when a new slave comes in<a name="line.801"></a>
+<span class="sourceLineNo">802</span> wals.clear();<a name="line.802"></a>
+<span class="sourceLineNo">803</span> }<a name="line.803"></a>
+<span class="sourceLineNo">804</span> if (logPrefix.equals(walsEntry.getKey())) {<a name="line.804"></a>
+<span class="sourceLineNo">805</span> wals.add(logName);<a name="line.805"></a>
+<span class="sourceLineNo">806</span> existingPrefix = true;<a name="line.806"></a>
+<span class="sourceLineNo">807</span> }<a name="line.807"></a>
+<span class="sourceLineNo">808</span> }<a name="line.808"></a>
+<span class="sourceLineNo">809</span> if (!existingPrefix) {<a name="line.809"></a>
+<span class="sourceLineNo">810</span> // The new log belongs to a new group, add it into this peer<a name="line.810"></a>
+<span class="sourceLineNo">811</span> LOG.debug("Start tracking logs for wal group {} for peer {}", logPrefix, peerId);<a name="line.811"></a>
+<span class="sourceLineNo">812</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.812"></a>
+<span class="sourceLineNo">813</span> wals.add(logName);<a name="line.813"></a>
+<span class="sourceLineNo">814</span> walsByPrefix.put(logPrefix, wals);<a name="line.814"></a>
+<span class="sourceLineNo">815</span> }<a name="line.815"></a>
+<span class="sourceLineNo">816</span> }<a name="line.816"></a>
+<span class="sourceLineNo">817</span> }<a name="line.817"></a>
+<span class="sourceLineNo">818</span><a name="line.818"></a>
+<span class="sourceLineNo">819</span> // Add to latestPaths<a name="line.819"></a>
+<span class="sourceLineNo">820</span> latestPaths.put(logPrefix, newLog);<a name="line.820"></a>
+<span class="sourceLineNo">821</span> }<a name="line.821"></a>
+<span class="sourceLineNo">822</span> }<a name="line.822"></a>
+<span class="sourceLineNo">823</span><a name="line.823"></a>
+<span class="sourceLineNo">824</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.824"></a>
+<span class="sourceLineNo">825</span> public void postLogRoll(Path newLog) throws IOException {<a name="line.825"></a>
+<span class="sourceLineNo">826</span> // This only updates the sources we own, not the recovered ones<a name="line.826"></a>
+<span class="sourceLineNo">827</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.827"></a>
+<span class="sourceLineNo">828</span> source.enqueueLog(newLog);<a name="line.828"></a>
+<span class="sourceLineNo">829</span> LOG.trace("Enqueued {} to source {} while performing postLogRoll operation.",<a name="line.829"></a>
+<span class="sourceLineNo">830</span> newLog, source.getQueueId());<a name="line.830"></a>
+<span class="sourceLineNo">831</span> }<a name="line.831"></a>
+<span class="sourceLineNo">832</span> }<a name="line.832"></a>
+<span class="sourceLineNo">833</span><a name="line.833"></a>
+<span class="sourceLineNo">834</span> @Override<a name="line.834"></a>
+<span class="sourceLineNo">835</span> public void regionServerRemoved(String regionserver) {<a name="line.835"></a>
+<span class="sourceLineNo">836</span> transferQueues(ServerName.valueOf(regionserver));<a name="line.836"></a>
+<span class="sourceLineNo">837</span> }<a name="line.837"></a>
+<span class="sourceLineNo">838</span><a name="line.838"></a>
+<span class="sourceLineNo">839</span> /**<a name="line.839"></a>
+<span class="sourceLineNo">840</span> * Transfer all the queues of the specified to this region server. First it tries to grab a lock<a name="line.840"></a>
+<span class="sourceLineNo">841</span> * and if it works it will move the old queues and finally will delete the old queues.<a name="line.841"></a>
+<span class="sourceLineNo">842</span> * <p><a name="line.842"></a>
+<span class="sourceLineNo">843</span> * It creates one old source for any type of source of the old rs.<a name="line.843"></a>
+<span class="sourceLineNo">844</span> */<a name="line.844"></a>
+<span class="sourceLineNo">845</span> private void transferQueues(ServerName deadRS) {<a name="line.845"></a>
+<span class="sourceLineNo">846</span> if (server.getServerName().equals(deadRS)) {<a name="line.846"></a>
+<span class="sourceLineNo">847</span> // it's just us, give up<a name="line.847"></a>
+<span class="sourceLineNo">848</span> return;<a name="line.848"></a>
+<span class="sourceLineNo">849</span> }<a name="line.849"></a>
+<span class="sourceLineNo">850</span> NodeFailoverWorker transfer = new NodeFailoverWorker(deadRS);<a name="line.850"></a>
+<span class="sourceLineNo">851</span> try {<a name="line.851"></a>
+<span class="sourceLineNo">852</span> this.executor.execute(transfer);<a name="line.852"></a>
+<span class="sourceLineNo">853</span> } catch (RejectedExecutionException ex) {<a name="line.853"></a>
+<span class="sourceLineNo">854</span> CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)<a name="line.854"></a>
+<span class="sourceLineNo">855</span> .getGlobalSource().incrFailedRecoveryQueue();<a name="line.855"></a>
+<span class="sourceLineNo">856</span> LOG.info("Cancelling the transfer of " + deadRS + " because of " + ex.getMessage());<a name="line.856"></a>
+<span class="sourceLineNo">857</span> }<a name="line.857"></a>
+<span class="sourceLineNo">858</span> }<a name="line.858"></a>
+<span class="sourceLineNo">859</span><a name="line.859"></a>
+<span class="sourceLineNo">860</span> /**<a name="line.860"></a>
+<span class="sourceLineNo">861</span> * Class responsible to setup new ReplicationSources to take care of the queues from dead region<a name="line.861"></a>
+<span class="sourceLineNo">862</span> * servers.<a name="line.862"></a>
+<span class="sourceLineNo">863</span> */<a name="line.863"></a>
+<span class="sourceLineNo">864</span> class NodeFailoverWorker extends Thread {<a name="line.864"></a>
+<span class="sourceLineNo">865</span><a name="line.865"></a>
+<span class="sourceLineNo">866</span> private final ServerName deadRS;<a name="line.866"></a>
+<span class="sourceLineNo">867</span> // After claim the queues from dead region server, the NodeFailoverWorker will skip to start<a name="line.867"></a>
+<span class="sourceLineNo">868</span> // the RecoveredReplicationSource if the peer has been removed. but there's possible that<a name="line.868"></a>
+<span class="sourceLineNo">869</span> // remove a peer with peerId = 2 and add a peer with peerId = 2 again during the<a name="line.869"></a>
+<span class="sourceLineNo">870</span> // NodeFailoverWorker. So we need a deep copied <peerId, peer> map to decide whether we<a name="line.870"></a>
+<span class="sourceLineNo">871</span> // should start the RecoveredReplicationSource. If the latest peer is not the old peer when<a name="line.871"></a>
+<span class="sourceLineNo">872</span> // NodeFailoverWorker begin, we should skip to start the RecoveredReplicationSource, Otherwise<a name="line.872"></a>
+<span class="sourceLineNo">873</span> // the rs will abort (See HBASE-20475).<a name="line.873"></a>
+<span class="sourceLineNo">874</span> private final Map<String, ReplicationPeerImpl> peersSnapshot;<a name="line.874"></a>
+<span class="sourceLineNo">875</span><a name="line.875"></a>
+<span class="sourceLineNo">876</span> public NodeFailoverWorker(ServerName deadRS) {<a name="line.876"></a>
+<span class="sourceLineNo">877</span> super("Failover-for-" + deadRS);<a name="line.877"></a>
+<span class="sourceLineNo">878</span> this.deadRS = deadRS;<a name="line.878"></a>
+<span class="sourceLineNo">879</span> peersSnapshot = new HashMap<>(replicationPeers.getPeerCache());<a name="line.879"></a>
+<span class="sourceLineNo">880</span> }<a name="line.880"></a>
+<span class="sourceLineNo">881</span><a name="line.881"></a>
+<span class="sourceLineNo">882</span> private boolean isOldPeer(String peerId, ReplicationPeerImpl newPeerRef) {<a name="line.882"></a>
+<span class="sourceLineNo">883</span> ReplicationPeerImpl oldPeerRef = peersSnapshot.get(peerId);<a name="line.883"></a>
+<span class="sourceLineNo">884</span> return oldPeerRef != null && oldPeerRef == newPeerRef;<a name="line.884"></a>
+<span class="sourceLineNo">885</span> }<a name="line.885"></a>
+<span class="sourceLineNo">886</span><a name="line.886"></a>
+<span class="sourceLineNo">887</span> @Override<a name="line.887"></a>
+<span class="sourceLineNo">888</span> public void run() {<a name="line.888"></a>
+<span class="sourceLineNo">889</span> // Wait a bit before transferring the queues, we may be shutting down.<a name="line.889"></a>
+<span class="sourceLineNo">890</span> // This sleep may not be enough in some cases.<a name="line.890"></a>
+<span class="sourceLineNo">891</span> try {<a name="line.891"></a>
+<span class="sourceLineNo">892</span> Thread.sleep(sleepBeforeFailover +<a name="line.892"></a>
+<span class="sourceLineNo">893</span> (long) (ThreadLocalRandom.current().nextFloat() * sleepBeforeFailover));<a name="line.893"></a>
+<span class="sourceLineNo">894</span> } catch (InterruptedException e) {<a name="line.894"></a>
+<span class="sourceLineNo">895</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.895"></a>
+<span class="sourceLineNo">896</span> Thread.currentThread().interrupt();<a name="line.896"></a>
+<span class="sourceLineNo">897</span> }<a name="line.897"></a>
+<span class="sourceLineNo">898</span> // We try to lock that rs' queue directory<a name="line.898"></a>
+<span class="sourceLineNo">899</span> if (server.isStopped()) {<a name="line.899"></a>
+<span class="sourceLineNo">900</span> LOG.info("Not transferring queue since we are shutting down");<a name="line.900"></a>
+<span class="sourceLineNo">901</span> return;<a name="line.901"></a>
+<span class="sourceLineNo">902</span> }<a name="line.902"></a>
+<span class="sourceLineNo">903</span> Map<String, Set<String>> newQueues = new HashMap<>();<a name="line.903"></a>
+<span class="sourceLineNo">904</span> try {<a name="line.904"></a>
+<span class="sourceLineNo">905</span> List<String> queues = queueStorage.getAllQueues(deadRS);<a name="line.905"></a>
+<span class="sourceLineNo">906</span> while (!queues.isEmpty()) {<a name="line.906"></a>
+<span class="sourceLineNo">907</span> Pair<String, SortedSet<String>> peer = queueStorage.claimQueue(deadRS,<a name="line.907"></a>
+<span class="sourceLineNo">908</span> queues.get(ThreadLocalRandom.current().nextInt(queues.size())), server.getServerName());<a name="line.908"></a>
+<span class="sourceLineNo">909</span> long sleep = sleepBeforeFailover / 2;<a name="line.909"></a>
+<span class="sourceLineNo">910</span> if (!peer.getSecond().isEmpty()) {<a name="line.910"></a>
+<span class="sourceLineNo">911</span> newQueues.put(peer.getFirst(), peer.getSecond());<a name="line.911"></a>
+<span class="sourceLineNo">912</span> sleep = sleepBeforeFailover;<a name="line.912"></a>
+<span class="sourceLineNo">913</span> }<a name="line.913"></a>
+<span class="sourceLineNo">914</span> try {<a name="line.914"></a>
+<span class="sourceLineNo">915</span> Thread.sleep(sleep);<a name="line.915"></a>
+<span class="sourceLineNo">916</span> } catch (InterruptedException e) {<a name="line.916"></a>
+<span class="sourceLineNo">917</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.917"></a>
+<span class="sourceLineNo">918</span> Thread.currentThread().interrupt();<a name="line.918"></a>
+<span class="sourceLineNo">919</span> }<a name="line.919"></a>
+<span class="sourceLineNo">920</span> queues = queueStorage.getAllQueues(deadRS);<a name="line.920"></a>
+<span class="sourceLineNo">921</span> }<a name="line.921"></a>
+<span class="sourceLineNo">922</span> if (queues.isEmpty()) {<a name="line.922"></a>
+<span class="sourceLineNo">923</span> queueStorage.removeReplicatorIfQueueIsEmpty(deadRS);<a name="line.923"></a>
+<span class="sourceLineNo">924</span> }<a name="line.924"></a>
+<span class="sourceLineNo">925</span> } catch (ReplicationException e) {<a name="line.925"></a>
+<span class="sourceLineNo">926</span> LOG.error(String.format("ReplicationException: cannot claim dead region (%s)'s " +<a name="line.926"></a>
+<span class="sourceLineNo">927</span> "replication queue. Znode : (%s)" +<a name="line.927"></a>
+<span class="sourceLineNo">928</span> " Possible solution: check if znode size exceeds jute.maxBuffer value. " +<a name="line.928"></a>
+<span class="sourceLineNo">929</span> " If so, increase it for both client and server side." + e), deadRS,<a name="line.929"></a>
+<span class="sourceLineNo">930</span> queueStorage.getRsNode(deadRS));<a name="line.930"></a>
+<span class="sourceLineNo">931</span> server.abort("Failed to claim queue from dead regionserver.", e);<a name="line.931"></a>
+<span class="sourceLineNo">932</span> return;<a name="line.932"></a>
+<span class="sourceLineNo">933</span> }<a name="line.933"></a>
+<span class="sourceLineNo">934</span> // Copying over the failed queue is completed.<a name="line.934"></a>
+<span class="sourceLineNo">935</span> if (newQueues.isEmpty()) {<a name="line.935"></a>
+<span class="sourceLineNo">936</span> // We either didn't get the lock or the failed region server didn't have any outstanding<a name="line.936"></a>
+<span class="sourceLineNo">937</span> // WALs to replicate, so we are done.<a name="line.937"></a>
+<span class="sourceLineNo">938</span> return;<a name="line.938"></a>
+<span class="sourceLineNo">939</span> }<a name="line.939"></a>
+<span class="sourceLineNo">940</span><a name="line.940"></a>
+<span class="sourceLineNo">941</span> for (Map.Entry<String, Set<String>> entry : newQueues.entrySet()) {<a name="line.941"></a>
+<span class="sourceLineNo">942</span> String queueId = entry.getKey();<a name="line.942"></a>
+<span class="sourceLineNo">943</span> Set<String> walsSet = entry.getValue();<a name="line.943"></a>
+<span class="sourceLineNo">944</span> try {<a name="line.944"></a>
+<span class="sourceLineNo">945</span> // there is not an actual peer defined corresponding to peerId for the failover.<a name="line.945"></a>
+<span class="sourceLineNo">946</span> ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);<a name="line.946"></a>
+<span class="sourceLineNo">947</span> String actualPeerId = replicationQueueInfo.getPeerId();<a name="line.947"></a>
+<span class="sourceLineNo">948</span><a name="line.948"></a>
+<span class="sourceLineNo">949</span> ReplicationPeerImpl peer = replicationPeers.getPeer(actualPeerId);<a name="line.949"></a>
+<span class="sourceLineNo">950</span> if (peer == null || !isOldPeer(actualPeerId, peer)) {<a name="line.950"></a>
+<span class="sourceLineNo">951</span> LOG.warn("Skipping failover for peer {} of node {}, peer is null", actualPeerId,<a name="line.951"></a>
+<span class="sourceLineNo">952</span> deadRS);<a name="line.952"></a>
+<span class="sourceLineNo">953</span> abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.953"></a>
+<span class="sourceLineNo">954</span> continue;<a name="line.954"></a>
+<span class="sourceLineNo">955</span> }<a name="line.955"></a>
+<span class="sourceLineNo">956</span> if (server instanceof ReplicationSyncUp.DummyServer<a name="line.956"></a>
+<span class="sourceLineNo">957</span> && peer.getPeerState().equals(PeerState.DISABLED)) {<a name="line.957"></a>
+<span class="sourceLineNo">958</span> LOG.warn("Peer {} is disabled. ReplicationSyncUp tool will skip "<a name="line.958"></a>
+<span class="sourceLineNo">959</span> + "replicating data to this peer.",<a name="line.959"></a>
+<span class="sourceLineNo">960</span> actualPeerId);<a name="line.960"></a>
+<span class="sourceLineNo">961</span> continue;<a name="line.961"></a>
+<span class="sourceLineNo">962</span> }<a name="line.962"></a>
+<span class="sourceLineNo">963</span><a name="line.963"></a>
+<span class="sourceLineNo">964</span> ReplicationSourceInterface src = createSource(queueId, peer);<a name="line.964"></a>
+<span class="sourceLineNo">965</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.965"></a>
+<span class="sourceLineNo">966</span> synchronized (oldsources) {<a name="line.966"></a>
+<span class="sourceLineNo">967</span> peer = replicationPeers.getPeer(src.getPeerId());<a name="line.967"></a>
+<span class="sourceLineNo">968</span> if (peer == null || !isOldPeer(src.getPeerId(), peer)) {<a name="line.968"></a>
+<span class="sourceLineNo">969</span> src.terminate("Recovered queue doesn't belong to any current peer");<a name="line.969"></a>
+<span class="sourceLineNo">970</span> deleteQueue(queueId);<a name="line.970"></a>
+<span class="sourceLineNo">971</span> continue;<a name="line.971"></a>
+<span class="sourceLineNo">972</span> }<a name="line.972"></a>
+<span class="sourceLineNo">973</span> // Do not setup recovered queue if a sync replication peer is in STANDBY state, or is<a name="line.973"></a>
+<span class="sourceLineNo">974</span> // transiting to STANDBY state. The only exception is we are in STANDBY state and<a name="line.974"></a>
+<span class="sourceLineNo">975</span> // transiting to DA, under this state we will replay the remote WAL and they need to be<a name="line.975"></a>
+<span class="sourceLineNo">976</span> // replicated back.<a name="line.976"></a>
+<span class="sourceLineNo">977</span> if (peer.getPeerConfig().isSyncReplication()) {<a name="line.977"></a>
+<span class="sourceLineNo">978</span> Pair<SyncReplicationState, SyncReplicationState> stateAndNewState =<a name="line.978"></a>
+<span class="sourceLineNo">979</span> peer.getSyncReplicationStateAndNewState();<a name="line.979"></a>
+<span class="sourceLineNo">980</span> if ((stateAndNewState.getFirst().equals(SyncReplicationState.STANDBY) &&<a name="line.980"></a>
+<span class="sourceLineNo">981</span> stateAndNewState.getSecond().equals(SyncReplicationState.NONE)) ||<a name="line.981"></a>
+<span class="sourceLineNo">982</span> stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)) {<a name="line.982"></a>
+<span class="sourceLineNo">983</span> src.terminate("Sync replication peer is in STANDBY state");<a name="line.983"></a>
+<span class="sourceLineNo">984</span> deleteQueue(queueId);<a name="line.984"></a>
+<span class="sourceLineNo">985</span> continue;<a name="line.985"></a>
+<span class="sourceLineNo">986</span> }<a name="line.986"></a>
+<span class="sourceLineNo">987</span> }<a name="line.987"></a>
+<span class="sourceLineNo">988</span> // track sources in walsByIdRecoveredQueues<a name="line.988"></a>
+<span class="sourceLineNo">989</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.989"></a>
+<span class="sourceLineNo">990</span> walsByIdRecoveredQueues.put(queueId, walsByGroup);<a name="line.990"></a>
+<span class="sourceLineNo">991</span> for (String wal : walsSet) {<a name="line.991"></a>
+<span class="sourceLineNo">992</span> String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal);<a name="line.992"></a>
+<span class="sourceLineNo">993</span> NavigableSet<String> wals = walsByGroup.get(walPrefix);<a name="line.993"></a>
+<span class="sourceLineNo">994</span> if (wals == null) {<a name="line.994"></a>
+<span class="sourceLineNo">995</span> wals = new TreeSet<>();<a name="line.995"></a>
+<span class="sourceLineNo">996</span> walsByGroup.put(walPrefix, wals);<a name="line.996"></a>
+<span class="sourceLineNo">997</span> }<a name="line.997"></a>
+<span class="sourceLineNo">998</span> wals.add(wal);<a name="line.998"></a>
+<span class="sourceLineNo">999</span> }<a name="line.999"></a>
+<span class="sourceLineNo">1000</span> oldsources.add(src);<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span> LOG.info("Added source for recovered queue {}", src.getQueueId());<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span> for (String wal : walsSet) {<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span> LOG.trace("Enqueueing log from recovered queue for source: " + src.getQueueId());<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span> src.enqueueLog(new Path(oldLogDir, wal));<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span> src.startup();<a name="line.1006"></a>
+<span class="sourceLineNo">1007</span> }<a name="line.1007"></a>
+<span class="sourceLineNo">1008</span> } catch (IOException e) {<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span> // TODO manage it<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span> LOG.error("Failed creating a source", e);<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span> }<a name="line.1011"></a>
+<span class="sourceLineNo">1012</span> }<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span> }<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span><a name="line.1015"></a>
+<span class="sourceLineNo">1016</span> /**<a name="line.1016"></a>
+<span class="sourceLineNo">1017</span> * Terminate the replication on this region server<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span> */<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span> public void join() {<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span> this.executor.shutdown();<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span> source.terminate("Region server is closing");<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
+<span class="sourceLineNo">1024</span> }<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span><a name="line.1025"></a>
+<span class="sourceLineNo">1026</span> /**<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span> * Get a copy of the wals of the normal sources on this rs<a name="line.1027"></a>
+<span class="sourceLineNo">1028</span> * @return a sorted set of wal names<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span> */<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span> public Map<String, Map<String, NavigableSet<String>>> getWALs() {<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span> return Collections.unmodifiableMap(walsById);<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span><a name="line.1033"></a>
+<span class="sourceLineNo">1034</span> /**<a name="line.1034"></a>
+<span class="sourceLineNo">1035</span> * Get a copy of the wals of the recovered sources on this rs<a name="line.1035"></a>
+<span class="sourceLineNo">1036</span> * @return a sorted set of wal names<a name="line.1036"></a>
+<span class="sourceLineNo">1037</span> */<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span> Map<String, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span> return Collections.unmodifiableMap(walsByIdRecoveredQueues);<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span> }<a name="line.1040"></a>
+<span class="sourceLineNo">1041</span><a name="line.1041"></a>
+<span class="sourceLineNo">1042</span> /**<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span> * Get a list of all the normal sources of this rs<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span> * @return list of all normal sources<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span> */<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span> public List<ReplicationSourceInterface> getSources() {<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span> return new ArrayList<>(this.sources.values());<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span> }<a name="line.1048"></a>
+<span class="sourceLineNo">1049</span><a name="line.1049"></a>
+<span class="sourceLineNo">1050</span> /**<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span> * Get a list of all the recovered sources of this rs<a name="line.1051"></a>
+<span class="sourceLineNo">1052</span> * @return list of all recovered sources<a name="line.1052"></a>
+<span class="sourceLineNo">1053</span> */<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span> public List<ReplicationSourceInterface> getOldSources() {<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span> return this.oldsources;<a name="line.1055"></a>
+<span class="sourceLineNo">1056</span> }<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span><a name="line.1057"></a>
+<span class="sourceLineNo">1058</span> /**<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span> * Get the normal source for a given peer<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span> * @return the normal source for the give peer if it exists, otherwise null.<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span> */<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span> public ReplicationSourceInterface getSource(String peerId) {<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span> return this.sources.get(peerId);<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span> }<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span><a name="line.1065"></a>
+<span class="sourceLineNo">1066</span> List<String> getAllQueues() throws IOException {<a name="line.1066"></a>
+<span class="sourceLineNo">1067</span> List<String> allQueues = Collections.emptyList();<a name="line.1067"></a>
+<span class="sourceLineNo">1068</span> try {<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span> allQueues = queueStorage.getAllQueues(server.getServerName());<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span> } catch (ReplicationException e) {<a name="line.1070"></a>
+<span class="sourceLineNo">1071</span> throw new IOException(e);<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span> }<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span> return allQueues;<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span><a name="line.1075"></a>
+<span class="sourceLineNo">1076</span> int getSizeOfLatestPath() {<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span> synchronized (latestPaths) {<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span> return latestPaths.size();<a name="line.1078"></a>
+<span class="sourceLineNo">1079</span> }<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span> }<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span><a name="line.1081"></a>
+<span class="sourceLineNo">1082</span> Set<Path> getLastestPath() {<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span> synchronized (latestPaths) {<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span> return Sets.newHashSet(latestPaths.values());<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span> }<a name="line.1085"></a>
+<span class="sourceLineNo">1086</span> }<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span><a name="line.1087"></a>
+<span class="sourceLineNo">1088</span> public AtomicLong getTotalBufferUsed() {<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span> return totalBufferUsed;<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span> }<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span><a name="line.1091"></a>
+<span class="sourceLineNo">1092</span> /**<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span> * Returns the maximum size in bytes of edits held in memory which are pending replication<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span> * across all sources inside this RegionServer.<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span> */<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span> public long getTotalBufferLimit() {<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span> return totalBufferLimit;<a name="line.1097"></a>
+<span class="sourceLineNo">1098</span> }<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span><a name="line.1099"></a>
+<span class="sourceLineNo">1100</span> /**<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span> * Get the directory where wals are archived<a name="line.1101"></a>
+<span class="sourceLineNo">1102</span> * @return the directory where wals are archived<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span> */<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span> public Path getOldLogDir() {<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span> return this.oldLogDir;<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span> }<a name="line.1106"></a>
+<span class="sourceLineNo">1107</span><a name="line.1107"></a>
+<span class="sourceLineNo">1108</span> /**<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span> * Get the directory where wals are stored by their RSs<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span> * @return the directory where wals are stored by their RSs<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span> */<a name="line.1111"></a>
+<span class="sourceLineNo">1112</span> public Path getLogDir() {<a name="line.1112"></a>
+<span class="sourceLineNo">1113</span> return this.logDir;<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span> }<a name="line.1114"></a>
+<span class="sourceLineNo">1115</span><a name="line.1115"></a>
+<span class="sourceLineNo">1116</span> /**<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span> * Get the handle on the local file system<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span> * @return Handle on the local file system<a name="line.1118"></a>
+<span class="sourceLineNo">1119</span> */<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span> public FileSystem getFs() {<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span> return this.fs;<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span> }<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span><a name="line.1123"></a>
+<span class="sourceLineNo">1124</span> /**<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span> * Get the ReplicationPeers used by this ReplicationSourceManager<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span> * @return the ReplicationPeers used by this ReplicationSourceManager<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span> */<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span> public ReplicationPeers getReplicationPeers() {<a name="line.1128"></a>
+<span class="sourceLineNo">1129</span> return this.replicationPeers;<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span> }<a name="line.1130"></a>
+<span class="sourceLineNo">1131</span><a name="line.1131"></a>
+<span class="sourceLineNo">1132</span> /**<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span> * Get a string representation of all the sources' metrics<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span> */<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span> public String getStats() {<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span> StringBuilder stats = new StringBuilder();<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span> // Print stats that apply across all Replication Sources<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span> stats.append("Global stats: ");<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span> stats.append("WAL Edits Buffer Used=").append(getTotalBufferUsed().get()).append("B, Limit=")<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span> .append(getTotalBufferLimit()).append("B\n");<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span> stats.append("Normal source for cluster " + source.getPeerId() + ": ");<a name="line.1142"></a>
+<span class="sourceLineNo">1143</span> stats.append(source.getStats() + "\n");<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span> }<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span> for (ReplicationSourceInterface oldSource : oldsources) {<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span> stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerId() + ": ");<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span> stats.append(oldSource.getStats() + "\n");<a name="line.1147"></a>
+<span class="sourceLineNo">1148</span> }<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span> return stats.toString();<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span> }<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span><a name="line.1151"></a>
+<span class="sourceLineNo">1152</span> public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span> throws IOException {<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span> throwIOExceptionWhenFail(() -> source.addHFileRefs(tableName, family, pairs));<a name="line.1155"></a>
+<span class="sourceLineNo">1156</span> }<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span> }<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span><a name="line.1158"></a>
+<span class="sourceLineNo">1159</span> public void cleanUpHFileRefs(String peerId, List<String> files) {<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span> interruptOrAbortWhenFail(() -> this.queueStorage.removeHFileRefs(peerId, files));<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span> }<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span><a name="line.1162"></a>
+<span class="sourceLineNo">1163</span> int activeFailoverTaskCount() {<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span> return executor.getActiveCount();<a name="line.1164"></a>
+<span class="sourceLineNo">1165</span> }<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span><a name="line.1166"></a>
+<span class="sourceLineNo">1167</span> MetricsReplicationGlobalSourceSource getGlobalMetrics() {<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span> return this.globalMetrics;<a name="line.1168"></a>
+<span class="sourceLineNo">1169</span> }<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span><a name="line.1170"></a>
+<span class="sourceLineNo">1171</span> /**<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span> * Add an hbase:meta Catalog replication source. Called on open of an hbase:meta Region.<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span> * Create it once only. If exists already, use the existing one.<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span> * @see #removeCatalogReplicationSource(RegionInfo)<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span> * @see #addSource(String) This is specialization on the addSource method.<a name="line.1175"></a>
+<span class="sourceLineNo">1176</span> */<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span> public ReplicationSourceInterface addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span> throws IOException {<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span> // Poor-man's putIfAbsent<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span> synchronized (this.catalogReplicationSource) {<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span> ReplicationSourceInterface rs = this.catalogReplicationSource.get();<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span> return rs != null ? rs :<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span> this.catalogReplicationSource.getAndSet(createCatalogReplicationSource(regionInfo));<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span> }<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span> }<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span><a name="line.1186"></a>
+<span class="sourceLineNo">1187</span> /**<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span> * Remove the hbase:meta Catalog replication source.<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span> * Called when we close hbase:meta.<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span> * @see #addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span> */<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span> public void removeCatalogReplicationSource(RegionInfo regionInfo) {<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span> // Nothing to do. Leave any CatalogReplicationSource in place in case an hbase:meta Region<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span> // comes back to this server.<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span> }<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span><a name="line.1196"></a>
+<span class="sourceLineNo">1197</span> /**<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span> * Create, initialize, and start the Catalog ReplicationSource.<a name="line.1198"></a>
+<span class="sourceLineNo">1199</span> * Presumes called one-time only (caller must ensure one-time only call).<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span> * This ReplicationSource is NOT created via {@link ReplicationSourceFactory}.<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span> * @see #addSource(String) This is a specialization of the addSource call.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span> * @see #catalogReplicationSource for a note on this ReplicationSource's lifecycle (and more on<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span> * why the special handling).<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span> */<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span> private ReplicationSourceInterface createCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span> throws IOException {<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span> // Instantiate meta walProvider. Instantiated here or over in the #warmupRegion call made by the<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span> // Master on a 'move' operation. Need to do extra work if we did NOT instantiate the provider.<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span> WALProvider walProvider = this.walFactory.getMetaWALProvider();<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span> boolean instantiate = walProvider == null;<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span> if (instantiate) {<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span> walProvider = this.walFactory.getMetaProvider();<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span> }<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span> // Here we do a specialization on what {@link ReplicationSourceFactory} does. There is no need<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span> // for persisting offset into WALs up in zookeeper (via ReplicationQueueInfo) as the catalog<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span> // read replicas feature that makes use of the source does a reset on a crash of the WAL<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span> // source process. See "4.1 Skip maintaining zookeeper replication queue (offsets/WALs)" in the<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span> // design doc attached to HBASE-18070 'Enable memstore replication for meta replica' for detail.<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span> CatalogReplicationSourcePeer peer = new CatalogReplicationSourcePeer(this.conf,<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span> this.clusterId.toString(), "meta_" + ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER);<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span> final ReplicationSourceInterface crs = new CatalogReplicationSource();<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span> crs.init(conf, fs, this, new NoopReplicationQueueStorage(), peer, server, peer.getId(),<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span> clusterId, walProvider.getWALFileLengthProvider(), new MetricsSource(peer.getId()));<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span> // Add listener on the provider so we can pick up the WAL to replicate on roll.<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span> WALActionsListener listener = new WALActionsListener() {<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span> @Override public void postLogRoll(Path oldPath, Path newPath) throws IOException {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span> crs.enqueueLog(newPath);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span> }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span> };<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span> walProvider.addWALActionsListener(listener);<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span> if (!instantiate) {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span> // If we did not instantiate provider, need to add our listener on already-created WAL<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span> // instance too (listeners are passed by provider to WAL instance on creation but if provider<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span> // created already, our listener add above is missed). And add the current WAL file to the<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span> // Replication Source so it can start replicating it.<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span> WAL wal = walProvider.getWAL(regionInfo);<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span> wal.registerWALActionsListener(listener);<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span> crs.enqueueLog(((AbstractFSWAL)wal).getCurrentFileName());<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span> }<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span> return crs.startup();<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span> }<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span>}<a name="line.1242"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html
index cf69e73..26166d2 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.ReplicationQueueOperation.html
@@ -291,964 +291,963 @@
<span class="sourceLineNo">283</span> if (currentReplicators == null || currentReplicators.isEmpty()) {<a name="line.283"></a>
<span class="sourceLineNo">284</span> return;<a name="line.284"></a>
<span class="sourceLineNo">285</span> }<a name="line.285"></a>
-<span class="sourceLineNo">286</span> List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers().stream()<a name="line.286"></a>
-<span class="sourceLineNo">287</span> .map(ServerName::valueOf).collect(Collectors.toList());<a name="line.287"></a>
-<span class="sourceLineNo">288</span> LOG.info(<a name="line.288"></a>
-<span class="sourceLineNo">289</span> "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);<a name="line.289"></a>
-<span class="sourceLineNo">290</span><a name="line.290"></a>
-<span class="sourceLineNo">291</span> // Look if there's anything to process after a restart<a name="line.291"></a>
-<span class="sourceLineNo">292</span> for (ServerName rs : currentReplicators) {<a name="line.292"></a>
-<span class="sourceLineNo">293</span> if (!otherRegionServers.contains(rs)) {<a name="line.293"></a>
-<span class="sourceLineNo">294</span> transferQueues(rs);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> }<a name="line.295"></a>
-<span class="sourceLineNo">296</span> }<a name="line.296"></a>
-<span class="sourceLineNo">297</span> }<a name="line.297"></a>
-<span class="sourceLineNo">298</span><a name="line.298"></a>
-<span class="sourceLineNo">299</span> /**<a name="line.299"></a>
-<span class="sourceLineNo">300</span> * <ol><a name="line.300"></a>
-<span class="sourceLineNo">301</span> * <li>Add peer to replicationPeers</li><a name="line.301"></a>
-<span class="sourceLineNo">302</span> * <li>Add the normal source and related replication queue</li><a name="line.302"></a>
-<span class="sourceLineNo">303</span> * <li>Add HFile Refs</li><a name="line.303"></a>
-<span class="sourceLineNo">304</span> * </ol><a name="line.304"></a>
-<span class="sourceLineNo">305</span> * @param peerId the id of replication peer<a name="line.305"></a>
-<span class="sourceLineNo">306</span> */<a name="line.306"></a>
-<span class="sourceLineNo">307</span> public void addPeer(String peerId) throws IOException {<a name="line.307"></a>
-<span class="sourceLineNo">308</span> boolean added = false;<a name="line.308"></a>
-<span class="sourceLineNo">309</span> try {<a name="line.309"></a>
-<span class="sourceLineNo">310</span> added = this.replicationPeers.addPeer(peerId);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> } catch (ReplicationException e) {<a name="line.311"></a>
-<span class="sourceLineNo">312</span> throw new IOException(e);<a name="line.312"></a>
-<span class="sourceLineNo">313</span> }<a name="line.313"></a>
-<span class="sourceLineNo">314</span> if (added) {<a name="line.314"></a>
-<span class="sourceLineNo">315</span> addSource(peerId);<a name="line.315"></a>
-<span class="sourceLineNo">316</span> if (replicationForBulkLoadDataEnabled) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span> throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));<a name="line.317"></a>
-<span class="sourceLineNo">318</span> }<a name="line.318"></a>
-<span class="sourceLineNo">319</span> }<a name="line.319"></a>
-<span class="sourceLineNo">320</span> }<a name="line.320"></a>
-<span class="sourceLineNo">321</span><a name="line.321"></a>
-<span class="sourceLineNo">322</span> /**<a name="line.322"></a>
-<span class="sourceLineNo">323</span> * <ol><a name="line.323"></a>
-<span class="sourceLineNo">324</span> * <li>Remove peer for replicationPeers</li><a name="line.324"></a>
-<span class="sourceLineNo">325</span> * <li>Remove all the recovered sources for the specified id and related replication queues</li><a name="line.325"></a>
-<span class="sourceLineNo">326</span> * <li>Remove the normal source and related replication queue</li><a name="line.326"></a>
-<span class="sourceLineNo">327</span> * <li>Remove HFile Refs</li><a name="line.327"></a>
-<span class="sourceLineNo">328</span> * </ol><a name="line.328"></a>
-<span class="sourceLineNo">329</span> * @param peerId the id of the replication peer<a name="line.329"></a>
-<span class="sourceLineNo">330</span> */<a name="line.330"></a>
-<span class="sourceLineNo">331</span> public void removePeer(String peerId) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> ReplicationPeer peer = replicationPeers.removePeer(peerId);<a name="line.332"></a>
-<span class="sourceLineNo">333</span> String terminateMessage = "Replication stream was removed by a user";<a name="line.333"></a>
-<span class="sourceLineNo">334</span> List<ReplicationSourceInterface> oldSourcesToDelete = new ArrayList<>();<a name="line.334"></a>
-<span class="sourceLineNo">335</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.335"></a>
-<span class="sourceLineNo">336</span> // see NodeFailoverWorker.run<a name="line.336"></a>
-<span class="sourceLineNo">337</span> synchronized (this.oldsources) {<a name="line.337"></a>
-<span class="sourceLineNo">338</span> // First close all the recovered sources for this peer<a name="line.338"></a>
-<span class="sourceLineNo">339</span> for (ReplicationSourceInterface src : oldsources) {<a name="line.339"></a>
-<span class="sourceLineNo">340</span> if (peerId.equals(src.getPeerId())) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span> oldSourcesToDelete.add(src);<a name="line.341"></a>
-<span class="sourceLineNo">342</span> }<a name="line.342"></a>
-<span class="sourceLineNo">343</span> }<a name="line.343"></a>
-<span class="sourceLineNo">344</span> for (ReplicationSourceInterface src : oldSourcesToDelete) {<a name="line.344"></a>
-<span class="sourceLineNo">345</span> src.terminate(terminateMessage);<a name="line.345"></a>
-<span class="sourceLineNo">346</span> removeRecoveredSource(src);<a name="line.346"></a>
-<span class="sourceLineNo">347</span> }<a name="line.347"></a>
-<span class="sourceLineNo">348</span> }<a name="line.348"></a>
-<span class="sourceLineNo">349</span> LOG.info(<a name="line.349"></a>
-<span class="sourceLineNo">350</span> "Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());<a name="line.350"></a>
-<span class="sourceLineNo">351</span> // Now close the normal source for this peer<a name="line.351"></a>
-<span class="sourceLineNo">352</span> ReplicationSourceInterface srcToRemove = this.sources.get(peerId);<a name="line.352"></a>
-<span class="sourceLineNo">353</span> if (srcToRemove != null) {<a name="line.353"></a>
-<span class="sourceLineNo">354</span> srcToRemove.terminate(terminateMessage);<a name="line.354"></a>
-<span class="sourceLineNo">355</span> removeSource(srcToRemove);<a name="line.355"></a>
-<span class="sourceLineNo">356</span> } else {<a name="line.356"></a>
-<span class="sourceLineNo">357</span> // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup<a name="line.357"></a>
-<span class="sourceLineNo">358</span> // Delete queue from storage and memory and queue id is same with peer id for normal<a name="line.358"></a>
-<span class="sourceLineNo">359</span> // source<a name="line.359"></a>
-<span class="sourceLineNo">360</span> deleteQueue(peerId);<a name="line.360"></a>
-<span class="sourceLineNo">361</span> this.walsById.remove(peerId);<a name="line.361"></a>
-<span class="sourceLineNo">362</span> }<a name="line.362"></a>
-<span class="sourceLineNo">363</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> if (peerConfig.isSyncReplication()) {<a name="line.364"></a>
-<span class="sourceLineNo">365</span> syncReplicationPeerMappingManager.remove(peerId, peerConfig);<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> // Remove HFile Refs<a name="line.367"></a>
-<span class="sourceLineNo">368</span> abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span> /**<a name="line.371"></a>
-<span class="sourceLineNo">372</span> * @return a new 'classic' user-space replication source.<a name="line.372"></a>
-<span class="sourceLineNo">373</span> * @param queueId the id of the replication queue to associate the ReplicationSource with.<a name="line.373"></a>
-<span class="sourceLineNo">374</span> * @see #createCatalogReplicationSource(RegionInfo) for creating a ReplicationSource for meta.<a name="line.374"></a>
-<span class="sourceLineNo">375</span> */<a name="line.375"></a>
-<span class="sourceLineNo">376</span> private ReplicationSourceInterface createSource(String queueId, ReplicationPeer replicationPeer)<a name="line.376"></a>
-<span class="sourceLineNo">377</span> throws IOException {<a name="line.377"></a>
-<span class="sourceLineNo">378</span> ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueId);<a name="line.378"></a>
-<span class="sourceLineNo">379</span> // Init the just created replication source. Pass the default walProvider's wal file length<a name="line.379"></a>
-<span class="sourceLineNo">380</span> // provider. Presumption is we replicate user-space Tables only. For hbase:meta region replica<a name="line.380"></a>
-<span class="sourceLineNo">381</span> // replication, see #createCatalogReplicationSource().<a name="line.381"></a>
-<span class="sourceLineNo">382</span> WALFileLengthProvider walFileLengthProvider =<a name="line.382"></a>
-<span class="sourceLineNo">383</span> this.walFactory.getWALProvider() != null?<a name="line.383"></a>
-<span class="sourceLineNo">384</span> this.walFactory.getWALProvider().getWALFileLengthProvider() : p -> OptionalLong.empty();<a name="line.384"></a>
-<span class="sourceLineNo">385</span> src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId,<a name="line.385"></a>
-<span class="sourceLineNo">386</span> walFileLengthProvider, new MetricsSource(queueId));<a name="line.386"></a>
-<span class="sourceLineNo">387</span> return src;<a name="line.387"></a>
-<span class="sourceLineNo">388</span> }<a name="line.388"></a>
-<span class="sourceLineNo">389</span><a name="line.389"></a>
-<span class="sourceLineNo">390</span> /**<a name="line.390"></a>
-<span class="sourceLineNo">391</span> * Add a normal source for the given peer on this region server. Meanwhile, add new replication<a name="line.391"></a>
-<span class="sourceLineNo">392</span> * queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal<a name="line.392"></a>
-<span class="sourceLineNo">393</span> * group and do replication<a name="line.393"></a>
-<span class="sourceLineNo">394</span> * @param peerId the id of the replication peer<a name="line.394"></a>
-<span class="sourceLineNo">395</span> * @return the source that was created<a name="line.395"></a>
-<span class="sourceLineNo">396</span> */<a name="line.396"></a>
-<span class="sourceLineNo">397</span> ReplicationSourceInterface addSource(String peerId) throws IOException {<a name="line.397"></a>
-<span class="sourceLineNo">398</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.398"></a>
-<span class="sourceLineNo">399</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.399"></a>
-<span class="sourceLineNo">400</span> // synchronized on latestPaths to avoid missing the new log<a name="line.400"></a>
-<span class="sourceLineNo">401</span> synchronized (this.latestPaths) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> this.sources.put(peerId, src);<a name="line.402"></a>
-<span class="sourceLineNo">403</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.403"></a>
-<span class="sourceLineNo">404</span> this.walsById.put(peerId, walsByGroup);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> // Add the latest wal to that source's queue<a name="line.405"></a>
-<span class="sourceLineNo">406</span> if (!latestPaths.isEmpty()) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span> for (Map.Entry<String, Path> walPrefixAndPath : latestPaths.entrySet()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> Path walPath = walPrefixAndPath.getValue();<a name="line.408"></a>
-<span class="sourceLineNo">409</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.409"></a>
-<span class="sourceLineNo">410</span> wals.add(walPath.getName());<a name="line.410"></a>
-<span class="sourceLineNo">411</span> walsByGroup.put(walPrefixAndPath.getKey(), wals);<a name="line.411"></a>
-<span class="sourceLineNo">412</span> // Abort RS and throw exception to make add peer failed<a name="line.412"></a>
-<span class="sourceLineNo">413</span> abortAndThrowIOExceptionWhenFail(<a name="line.413"></a>
-<span class="sourceLineNo">414</span> () -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName()));<a name="line.414"></a>
-<span class="sourceLineNo">415</span> src.enqueueLog(walPath);<a name="line.415"></a>
-<span class="sourceLineNo">416</span> LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());<a name="line.416"></a>
-<span class="sourceLineNo">417</span> }<a name="line.417"></a>
-<span class="sourceLineNo">418</span> }<a name="line.418"></a>
-<span class="sourceLineNo">419</span> }<a name="line.419"></a>
-<span class="sourceLineNo">420</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.420"></a>
-<span class="sourceLineNo">421</span> if (peerConfig.isSyncReplication()) {<a name="line.421"></a>
-<span class="sourceLineNo">422</span> syncReplicationPeerMappingManager.add(peer.getId(), peerConfig);<a name="line.422"></a>
-<span class="sourceLineNo">423</span> }<a name="line.423"></a>
-<span class="sourceLineNo">424</span> src.startup();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> return src;<a name="line.425"></a>
-<span class="sourceLineNo">426</span> }<a name="line.426"></a>
-<span class="sourceLineNo">427</span><a name="line.427"></a>
-<span class="sourceLineNo">428</span> /**<a name="line.428"></a>
-<span class="sourceLineNo">429</span> * <p><a name="line.429"></a>
-<span class="sourceLineNo">430</span> * This is used when we transit a sync replication peer to {@link SyncReplicationState#STANDBY}.<a name="line.430"></a>
-<span class="sourceLineNo">431</span> * </p><a name="line.431"></a>
-<span class="sourceLineNo">432</span> * <p><a name="line.432"></a>
-<span class="sourceLineNo">433</span> * When transiting to {@link SyncReplicationState#STANDBY}, we can remove all the pending wal<a name="line.433"></a>
-<span class="sourceLineNo">434</span> * files for a replication peer as we do not need to replicate them any more. And this is<a name="line.434"></a>
-<span class="sourceLineNo">435</span> * necessary, otherwise when we transit back to {@link SyncReplicationState#DOWNGRADE_ACTIVE}<a name="line.435"></a>
-<span class="sourceLineNo">436</span> * later, the stale data will be replicated again and cause inconsistency.<a name="line.436"></a>
-<span class="sourceLineNo">437</span> * </p><a name="line.437"></a>
-<span class="sourceLineNo">438</span> * <p><a name="line.438"></a>
-<span class="sourceLineNo">439</span> * See HBASE-20426 for more details.<a name="line.439"></a>
-<span class="sourceLineNo">440</span> * </p><a name="line.440"></a>
-<span class="sourceLineNo">441</span> * @param peerId the id of the sync replication peer<a name="line.441"></a>
-<span class="sourceLineNo">442</span> */<a name="line.442"></a>
-<span class="sourceLineNo">443</span> public void drainSources(String peerId) throws IOException, ReplicationException {<a name="line.443"></a>
-<span class="sourceLineNo">444</span> String terminateMessage = "Sync replication peer " + peerId +<a name="line.444"></a>
-<span class="sourceLineNo">445</span> " is transiting to STANDBY. Will close the previous replication source and open a new one";<a name="line.445"></a>
-<span class="sourceLineNo">446</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.446"></a>
-<span class="sourceLineNo">447</span> assert peer.getPeerConfig().isSyncReplication();<a name="line.447"></a>
-<span class="sourceLineNo">448</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.448"></a>
-<span class="sourceLineNo">449</span> // synchronized here to avoid race with preLogRoll where we add new log to source and also<a name="line.449"></a>
-<span class="sourceLineNo">450</span> // walsById.<a name="line.450"></a>
-<span class="sourceLineNo">451</span> ReplicationSourceInterface toRemove;<a name="line.451"></a>
-<span class="sourceLineNo">452</span> Map<String, NavigableSet<String>> wals = new HashMap<>();<a name="line.452"></a>
-<span class="sourceLineNo">453</span> synchronized (latestPaths) {<a name="line.453"></a>
-<span class="sourceLineNo">454</span> toRemove = sources.put(peerId, src);<a name="line.454"></a>
-<span class="sourceLineNo">455</span> if (toRemove != null) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.456"></a>
-<span class="sourceLineNo">457</span> toRemove.terminate(terminateMessage);<a name="line.457"></a>
-<span class="sourceLineNo">458</span> toRemove.getSourceMetrics().clear();<a name="line.458"></a>
-<span class="sourceLineNo">459</span> }<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // Here we make a copy of all the remaining wal files and then delete them from the<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // replication queue storage after releasing the lock. It is not safe to just remove the old<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // map from walsById since later we may fail to delete them from the replication queue<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // storage, and when we retry next time, we can not know the wal files that need to be deleted<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // from the replication queue storage.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> walsById.get(peerId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));<a name="line.465"></a>
-<span class="sourceLineNo">466</span> }<a name="line.466"></a>
-<span class="sourceLineNo">467</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.467"></a>
-<span class="sourceLineNo">468</span> src.startup();<a name="line.468"></a>
-<span class="sourceLineNo">469</span> for (NavigableSet<String> walsByGroup : wals.values()) {<a name="line.469"></a>
-<span class="sourceLineNo">470</span> for (String wal : walsByGroup) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> queueStorage.removeWAL(server.getServerName(), peerId, wal);<a name="line.471"></a>
-<span class="sourceLineNo">472</span> }<a name="line.472"></a>
-<span class="sourceLineNo">473</span> }<a name="line.473"></a>
-<span class="sourceLineNo">474</span> synchronized (walsById) {<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Map<String, NavigableSet<String>> oldWals = walsById.get(peerId);<a name="line.475"></a>
-<span class="sourceLineNo">476</span> wals.forEach((k, v) -> {<a name="line.476"></a>
-<span class="sourceLineNo">477</span> NavigableSet<String> walsByGroup = oldWals.get(k);<a name="line.477"></a>
-<span class="sourceLineNo">478</span> if (walsByGroup != null) {<a name="line.478"></a>
-<span class="sourceLineNo">479</span> walsByGroup.removeAll(v);<a name="line.479"></a>
-<span class="sourceLineNo">480</span> }<a name="line.480"></a>
-<span class="sourceLineNo">481</span> });<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // synchronized on oldsources to avoid race with NodeFailoverWorker. Since NodeFailoverWorker is<a name="line.483"></a>
-<span class="sourceLineNo">484</span> // a background task, we will delete the file from replication queue storage under the lock to<a name="line.484"></a>
-<span class="sourceLineNo">485</span> // simplify the logic.<a name="line.485"></a>
-<span class="sourceLineNo">486</span> synchronized (this.oldsources) {<a name="line.486"></a>
-<span class="sourceLineNo">487</span> for (Iterator<ReplicationSourceInterface> iter = oldsources.iterator(); iter.hasNext();) {<a name="line.487"></a>
-<span class="sourceLineNo">488</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.488"></a>
-<span class="sourceLineNo">489</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.489"></a>
-<span class="sourceLineNo">490</span> String queueId = oldSource.getQueueId();<a name="line.490"></a>
-<span class="sourceLineNo">491</span> oldSource.terminate(terminateMessage);<a name="line.491"></a>
-<span class="sourceLineNo">492</span> oldSource.getSourceMetrics().clear();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> queueStorage.removeQueue(server.getServerName(), queueId);<a name="line.493"></a>
-<span class="sourceLineNo">494</span> walsByIdRecoveredQueues.remove(queueId);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> iter.remove();<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span> }<a name="line.497"></a>
-<span class="sourceLineNo">498</span> }<a name="line.498"></a>
-<span class="sourceLineNo">499</span> }<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span> /**<a name="line.501"></a>
-<span class="sourceLineNo">502</span> * Close the previous replication sources of this peer id and open new sources to trigger the new<a name="line.502"></a>
-<span class="sourceLineNo">503</span> * replication state changes or new replication config changes. Here we don't need to change<a name="line.503"></a>
-<span class="sourceLineNo">504</span> * replication queue storage and only to enqueue all logs to the new replication source<a name="line.504"></a>
-<span class="sourceLineNo">505</span> * @param peerId the id of the replication peer<a name="line.505"></a>
-<span class="sourceLineNo">506</span> */<a name="line.506"></a>
-<span class="sourceLineNo">507</span> public void refreshSources(String peerId) throws IOException {<a name="line.507"></a>
-<span class="sourceLineNo">508</span> String terminateMessage = "Peer " + peerId +<a name="line.508"></a>
-<span class="sourceLineNo">509</span> " state or config changed. Will close the previous replication source and open a new one";<a name="line.509"></a>
-<span class="sourceLineNo">510</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.510"></a>
-<span class="sourceLineNo">511</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.511"></a>
-<span class="sourceLineNo">512</span> // synchronized on latestPaths to avoid missing the new log<a name="line.512"></a>
-<span class="sourceLineNo">513</span> synchronized (this.latestPaths) {<a name="line.513"></a>
-<span class="sourceLineNo">514</span> ReplicationSourceInterface toRemove = this.sources.put(peerId, src);<a name="line.514"></a>
-<span class="sourceLineNo">515</span> if (toRemove != null) {<a name="line.515"></a>
-<span class="sourceLineNo">516</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.516"></a>
-<span class="sourceLineNo">517</span> // Do not clear metrics<a name="line.517"></a>
-<span class="sourceLineNo">518</span> toRemove.terminate(terminateMessage, null, false);<a name="line.518"></a>
-<span class="sourceLineNo">519</span> }<a name="line.519"></a>
-<span class="sourceLineNo">520</span> for (NavigableSet<String> walsByGroup : walsById.get(peerId).values()) {<a name="line.520"></a>
-<span class="sourceLineNo">521</span> walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));<a name="line.521"></a>
-<span class="sourceLineNo">522</span> }<a name="line.522"></a>
-<span class="sourceLineNo">523</span> }<a name="line.523"></a>
-<span class="sourceLineNo">524</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.524"></a>
-<span class="sourceLineNo">525</span> src.startup();<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> List<ReplicationSourceInterface> toStartup = new ArrayList<>();<a name="line.527"></a>
-<span class="sourceLineNo">528</span> // synchronized on oldsources to avoid race with NodeFailoverWorker<a name="line.528"></a>
-<span class="sourceLineNo">529</span> synchronized (this.oldsources) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span> List<String> previousQueueIds = new ArrayList<>();<a name="line.530"></a>
-<span class="sourceLineNo">531</span> for (Iterator<ReplicationSourceInterface> iter = this.oldsources.iterator(); iter<a name="line.531"></a>
-<span class="sourceLineNo">532</span> .hasNext();) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.534"></a>
-<span class="sourceLineNo">535</span> previousQueueIds.add(oldSource.getQueueId());<a name="line.535"></a>
-<span class="sourceLineNo">536</span> oldSource.terminate(terminateMessage);<a name="line.536"></a>
-<span class="sourceLineNo">537</span> iter.remove();<a name="line.537"></a>
-<span class="sourceLineNo">538</span> }<a name="line.538"></a>
-<span class="sourceLineNo">539</span> }<a name="line.539"></a>
-<span class="sourceLineNo">540</span> for (String queueId : previousQueueIds) {<a name="line.540"></a>
-<span class="sourceLineNo">541</span> ReplicationSourceInterface recoveredReplicationSource = createSource(queueId, peer);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> this.oldsources.add(recoveredReplicationSource);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {<a name="line.543"></a>
-<span class="sourceLineNo">544</span> walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span> toStartup.add(recoveredReplicationSource);<a name="line.546"></a>
-<span class="sourceLineNo">547</span> }<a name="line.547"></a>
-<span class="sourceLineNo">548</span> }<a name="line.548"></a>
-<span class="sourceLineNo">549</span> for (ReplicationSourceInterface replicationSource : toStartup) {<a name="line.549"></a>
-<span class="sourceLineNo">550</span> replicationSource.startup();<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span> }<a name="line.552"></a>
-<span class="sourceLineNo">553</span><a name="line.553"></a>
-<span class="sourceLineNo">554</span> /**<a name="line.554"></a>
-<span class="sourceLineNo">555</span> * Clear the metrics and related replication queue of the specified old source<a name="line.555"></a>
-<span class="sourceLineNo">556</span> * @param src source to clear<a name="line.556"></a>
-<span class="sourceLineNo">557</span> */<a name="line.557"></a>
-<span class="sourceLineNo">558</span> private boolean removeRecoveredSource(ReplicationSourceInterface src) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> if (!this.oldsources.remove(src)) {<a name="line.559"></a>
-<span class="sourceLineNo">560</span> return false;<a name="line.560"></a>
-<span class="sourceLineNo">561</span> }<a name="line.561"></a>
-<span class="sourceLineNo">562</span> LOG.info("Done with the recovered queue {}", src.getQueueId());<a name="line.562"></a>
-<span class="sourceLineNo">563</span> // Delete queue from storage and memory<a name="line.563"></a>
-<span class="sourceLineNo">564</span> deleteQueue(src.getQueueId());<a name="line.564"></a>
-<span class="sourceLineNo">565</span> this.walsByIdRecoveredQueues.remove(src.getQueueId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> return true;<a name="line.566"></a>
-<span class="sourceLineNo">567</span> }<a name="line.567"></a>
-<span class="sourceLineNo">568</span><a name="line.568"></a>
-<span class="sourceLineNo">569</span> void finishRecoveredSource(ReplicationSourceInterface src) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span> synchronized (oldsources) {<a name="line.570"></a>
-<span class="sourceLineNo">571</span> if (!removeRecoveredSource(src)) {<a name="line.571"></a>
-<span class="sourceLineNo">572</span> return;<a name="line.572"></a>
-<span class="sourceLineNo">573</span> }<a name="line.573"></a>
-<span class="sourceLineNo">574</span> }<a name="line.574"></a>
-<span class="sourceLineNo">575</span> LOG.info("Finished recovering queue {} with the following stats: {}", src.getQueueId(),<a name="line.575"></a>
-<span class="sourceLineNo">576</span> src.getStats());<a name="line.576"></a>
-<span class="sourceLineNo">577</span> }<a name="line.577"></a>
-<span class="sourceLineNo">578</span><a name="line.578"></a>
-<span class="sourceLineNo">579</span> /**<a name="line.579"></a>
-<span class="sourceLineNo">580</span> * Clear the metrics and related replication queue of the specified old source<a name="line.580"></a>
-<span class="sourceLineNo">581</span> * @param src source to clear<a name="line.581"></a>
-<span class="sourceLineNo">582</span> */<a name="line.582"></a>
-<span class="sourceLineNo">583</span> void removeSource(ReplicationSourceInterface src) {<a name="line.583"></a>
-<span class="sourceLineNo">584</span> LOG.info("Done with the queue " + src.getQueueId());<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.sources.remove(src.getPeerId());<a name="line.585"></a>
-<span class="sourceLineNo">586</span> // Delete queue from storage and memory<a name="line.586"></a>
-<span class="sourceLineNo">587</span> deleteQueue(src.getQueueId());<a name="line.587"></a>
-<span class="sourceLineNo">588</span> this.walsById.remove(src.getQueueId());<a name="line.588"></a>
-<span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span> }<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> /**<a name="line.592"></a>
-<span class="sourceLineNo">593</span> * Delete a complete queue of wals associated with a replication source<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * @param queueId the id of replication queue to delete<a name="line.594"></a>
-<span class="sourceLineNo">595</span> */<a name="line.595"></a>
-<span class="sourceLineNo">596</span> private void deleteQueue(String queueId) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span> abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.597"></a>
-<span class="sourceLineNo">598</span> }<a name="line.598"></a>
-<span class="sourceLineNo">599</span><a name="line.599"></a>
-<span class="sourceLineNo">600</span> @FunctionalInterface<a name="line.600"></a>
-<span class="sourceLineNo">601</span> private interface ReplicationQueueOperation {<a name="line.601"></a>
-<span class="sourceLineNo">602</span> void exec() throws ReplicationException;<a name="line.602"></a>
-<span class="sourceLineNo">603</span> }<a name="line.603"></a>
-<span class="sourceLineNo">604</span><a name="line.604"></a>
-<span class="sourceLineNo">605</span> /**<a name="line.605"></a>
-<span class="sourceLineNo">606</span> * Refresh replication source will terminate the old source first, then the source thread will be<a name="line.606"></a>
-<span class="sourceLineNo">607</span> * interrupted. Need to handle it instead of abort the region server.<a name="line.607"></a>
-<span class="sourceLineNo">608</span> */<a name="line.608"></a>
-<span class="sourceLineNo">609</span> private void interruptOrAbortWhenFail(ReplicationQueueOperation op) {<a name="line.609"></a>
-<span class="sourceLineNo">610</span> try {<a name="line.610"></a>
-<span class="sourceLineNo">611</span> op.exec();<a name="line.611"></a>
-<span class="sourceLineNo">612</span> } catch (ReplicationException e) {<a name="line.612"></a>
-<span class="sourceLineNo">613</span> if (e.getCause() != null && e.getCause() instanceof KeeperException.SystemErrorException<a name="line.613"></a>
-<span class="sourceLineNo">614</span> && e.getCause().getCause() != null && e.getCause()<a name="line.614"></a>
-<span class="sourceLineNo">615</span> .getCause() instanceof InterruptedException) {<a name="line.615"></a>
-<span class="sourceLineNo">616</span> // ReplicationRuntimeException(a RuntimeException) is thrown out here. The reason is<a name="line.616"></a>
-<span class="sourceLineNo">617</span> // that thread is interrupted deep down in the stack, it should pass the following<a name="line.617"></a>
-<span class="sourceLineNo">618</span> // processing logic and propagate to the most top layer which can handle this exception<a name="line.618"></a>
-<span class="sourceLineNo">619</span> // properly. In this specific case, the top layer is ReplicationSourceShipper#run().<a name="line.619"></a>
-<span class="sourceLineNo">620</span> throw new ReplicationRuntimeException(<a name="line.620"></a>
-<span class="sourceLineNo">621</span> "Thread is interrupted, the replication source may be terminated",<a name="line.621"></a>
-<span class="sourceLineNo">622</span> e.getCause().getCause());<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span> server.abort("Failed to operate on replication queue", e);<a name="line.624"></a>
-<span class="sourceLineNo">625</span> }<a name="line.625"></a>
-<span class="sourceLineNo">626</span> }<a name="line.626"></a>
-<span class="sourceLineNo">627</span><a name="line.627"></a>
-<span class="sourceLineNo">628</span> private void abortWhenFail(ReplicationQueueOperation op) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> op.exec();<a name="line.630"></a>
-<span class="sourceLineNo">631</span> } catch (ReplicationException e) {<a name="line.631"></a>
-<span class="sourceLineNo">632</span> server.abort("Failed to operate on replication queue", e);<a name="line.632"></a>
-<span class="sourceLineNo">633</span> }<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span><a name="line.635"></a>
-<span class="sourceLineNo">636</span> private void throwIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.636"></a>
-<span class="sourceLineNo">637</span> try {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> op.exec();<a name="line.638"></a>
-<span class="sourceLineNo">639</span> } catch (ReplicationException e) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> throw new IOException(e);<a name="line.640"></a>
-<span class="sourceLineNo">641</span> }<a name="line.641"></a>
-<span class="sourceLineNo">642</span> }<a name="line.642"></a>
-<span class="sourceLineNo">643</span><a name="line.643"></a>
-<span class="sourceLineNo">644</span> private void abortAndThrowIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> try {<a name="line.645"></a>
-<span class="sourceLineNo">646</span> op.exec();<a name="line.646"></a>
-<span class="sourceLineNo">647</span> } catch (ReplicationException e) {<a name="line.647"></a>
-<span class="sourceLineNo">648</span> server.abort("Failed to operate on replication queue", e);<a name="line.648"></a>
-<span class="sourceLineNo">649</span> throw new IOException(e);<a name="line.649"></a>
-<span class="sourceLineNo">650</span> }<a name="line.650"></a>
-<span class="sourceLineNo">651</span> }<a name="line.651"></a>
-<span class="sourceLineNo">652</span><a name="line.652"></a>
-<span class="sourceLineNo">653</span> /**<a name="line.653"></a>
-<span class="sourceLineNo">654</span> * This method will log the current position to storage. And also clean old logs from the<a name="line.654"></a>
-<span class="sourceLineNo">655</span> * replication queue.<a name="line.655"></a>
-<span class="sourceLineNo">656</span> * @param source the replication source<a name="line.656"></a>
-<span class="sourceLineNo">657</span> * @param entryBatch the wal entry batch we just shipped<a name="line.657"></a>
-<span class="sourceLineNo">658</span> */<a name="line.658"></a>
-<span class="sourceLineNo">659</span> public void logPositionAndCleanOldLogs(ReplicationSourceInterface source,<a name="line.659"></a>
-<span class="sourceLineNo">660</span> WALEntryBatch entryBatch) {<a name="line.660"></a>
-<span class="sourceLineNo">661</span> String fileName = entryBatch.getLastWalPath().getName();<a name="line.661"></a>
-<span class="sourceLineNo">662</span> interruptOrAbortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(),<a name="line.662"></a>
-<span class="sourceLineNo">663</span> source.getQueueId(), fileName, entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds()));<a name="line.663"></a>
-<span class="sourceLineNo">664</span> cleanOldLogs(fileName, entryBatch.isEndOfFile(), source);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> }<a name="line.665"></a>
-<span class="sourceLineNo">666</span><a name="line.666"></a>
-<span class="sourceLineNo">667</span> /**<a name="line.667"></a>
-<span class="sourceLineNo">668</span> * Cleans a log file and all older logs from replication queue. Called when we are sure that a log<a name="line.668"></a>
-<span class="sourceLineNo">669</span> * file is closed and has no more entries.<a name="line.669"></a>
-<span class="sourceLineNo">670</span> * @param log Path to the log<a name="line.670"></a>
-<span class="sourceLineNo">671</span> * @param inclusive whether we should also remove the given log file<a name="line.671"></a>
-<span class="sourceLineNo">672</span> * @param source the replication source<a name="line.672"></a>
-<span class="sourceLineNo">673</span> */<a name="line.673"></a>
-<span class="sourceLineNo">674</span> void cleanOldLogs(String log, boolean inclusive, ReplicationSourceInterface source) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(log);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> if (source.isRecovered()) {<a name="line.676"></a>
-<span class="sourceLineNo">677</span> NavigableSet<String> wals = walsByIdRecoveredQueues.get(source.getQueueId()).get(logPrefix);<a name="line.677"></a>
-<span class="sourceLineNo">678</span> if (wals != null) {<a name="line.678"></a>
-<span class="sourceLineNo">679</span> NavigableSet<String> walsToRemove = wals.headSet(log, inclusive);<a name="line.679"></a>
-<span class="sourceLineNo">680</span> if (walsToRemove.isEmpty()) {<a name="line.680"></a>
-<span class="sourceLineNo">681</span> return;<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> cleanOldLogs(walsToRemove, source);<a name="line.683"></a>
-<span class="sourceLineNo">684</span> walsToRemove.clear();<a name="line.684"></a>
-<span class="sourceLineNo">685</span> }<a name="line.685"></a>
-<span class="sourceLineNo">686</span> } else {<a name="line.686"></a>
-<span class="sourceLineNo">687</span> NavigableSet<String> wals;<a name="line.687"></a>
-<span class="sourceLineNo">688</span> NavigableSet<String> walsToRemove;<a name="line.688"></a>
-<span class="sourceLineNo">689</span> // synchronized on walsById to avoid race with preLogRoll<a name="line.689"></a>
-<span class="sourceLineNo">690</span> synchronized (this.walsById) {<a name="line.690"></a>
-<span class="sourceLineNo">691</span> wals = walsById.get(source.getQueueId()).get(logPrefix);<a name="line.691"></a>
-<span class="sourceLineNo">692</span> if (wals == null) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span> return;<a name="line.693"></a>
-<span class="sourceLineNo">694</span> }<a name="line.694"></a>
-<span class="sourceLineNo">695</span> walsToRemove = wals.headSet(log, inclusive);<a name="line.695"></a>
-<span class="sourceLineNo">696</span> if (walsToRemove.isEmpty()) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span> return;<a name="line.697"></a>
-<span class="sourceLineNo">698</span> }<a name="line.698"></a>
-<span class="sourceLineNo">699</span> walsToRemove = new TreeSet<>(walsToRemove);<a name="line.699"></a>
-<span class="sourceLineNo">700</span> }<a name="line.700"></a>
-<span class="sourceLineNo">701</span> // cleanOldLogs may spend some time, especially for sync replication where we may want to<a name="line.701"></a>
-<span class="sourceLineNo">702</span> // remove remote wals as the remote cluster may have already been down, so we do it outside<a name="line.702"></a>
-<span class="sourceLineNo">703</span> // the lock to avoid block preLogRoll<a name="line.703"></a>
-<span class="sourceLineNo">704</span> cleanOldLogs(walsToRemove, source);<a name="line.704"></a>
-<span class="sourceLineNo">705</span> // now let's remove the files in the set<a name="line.705"></a>
-<span class="sourceLineNo">706</span> synchronized (this.walsById) {<a name="line.706"></a>
-<span class="sourceLineNo">707</span> wals.removeAll(walsToRemove);<a name="line.707"></a>
-<span class="sourceLineNo">708</span> }<a name="line.708"></a>
-<span class="sourceLineNo">709</span> }<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> private void removeRemoteWALs(String peerId, String remoteWALDir, Collection<String> wals)<a name="line.712"></a>
-<span class="sourceLineNo">713</span> throws IOException {<a name="line.713"></a>
-<span class="sourceLineNo">714</span> Path remoteWALDirForPeer = ReplicationUtils.getPeerRemoteWALDir(remoteWALDir, peerId);<a name="line.714"></a>
-<span class="sourceLineNo">715</span> FileSystem fs = ReplicationUtils.getRemoteWALFileSystem(conf, remoteWALDir);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> for (String wal : wals) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> Path walFile = new Path(remoteWALDirForPeer, wal);<a name="line.717"></a>
-<span class="sourceLineNo">718</span> try {<a name="line.718"></a>
-<span class="sourceLineNo">719</span> if (!fs.delete(walFile, false) && fs.exists(walFile)) {<a name="line.719"></a>
-<span class="sourceLineNo">720</span> throw new IOException("Can not delete " + walFile);<a name="line.720"></a>
-<span class="sourceLineNo">721</span> }<a name="line.721"></a>
-<span class="sourceLineNo">722</span> } catch (FileNotFoundException e) {<a name="line.722"></a>
-<span class="sourceLineNo">723</span> // Just ignore since this means the file has already been deleted.<a name="line.723"></a>
-<span class="sourceLineNo">724</span> // The javadoc of the FileSystem.delete methods does not specify the behavior of deleting an<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // inexistent file, so here we deal with both, i.e, check the return value of the<a name="line.725"></a>
-<span class="sourceLineNo">726</span> // FileSystem.delete, and also catch FNFE.<a name="line.726"></a>
-<span class="sourceLineNo">727</span> LOG.debug("The remote wal {} has already been deleted?", walFile, e);<a name="line.727"></a>
-<span class="sourceLineNo">728</span> }<a name="line.728"></a>
-<span class="sourceLineNo">729</span> }<a name="line.729"></a>
-<span class="sourceLineNo">730</span> }<a name="line.730"></a>
-<span class="sourceLineNo">731</span><a name="line.731"></a>
-<span class="sourceLineNo">732</span> private void cleanOldLogs(NavigableSet<String> wals, ReplicationSourceInterface source) {<a name="line.732"></a>
-<span class="sourceLineNo">733</span> LOG.debug("Removing {} logs in the list: {}", wals.size(), wals);<a name="line.733"></a>
-<span class="sourceLineNo">734</span> // The intention here is that, we want to delete the remote wal files ASAP as it may effect the<a name="line.734"></a>
-<span class="sourceLineNo">735</span> // failover time if you want to transit the remote cluster from S to A. And the infinite retry<a name="line.735"></a>
-<span class="sourceLineNo">736</span> // is not a problem, as if we can not contact with the remote HDFS cluster, then usually we can<a name="line.736"></a>
-<span class="sourceLineNo">737</span> // not contact with the HBase cluster either, so the replication will be blocked either.<a name="line.737"></a>
-<span class="sourceLineNo">738</span> if (source.isSyncReplication()) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span> String peerId = source.getPeerId();<a name="line.739"></a>
-<span class="sourceLineNo">740</span> String remoteWALDir = source.getPeer().getPeerConfig().getRemoteWALDir();<a name="line.740"></a>
-<span class="sourceLineNo">741</span> // Filter out the wals need to be removed from the remote directory. Its name should be the<a name="line.741"></a>
-<span class="sourceLineNo">742</span> // special format, and also, the peer id in its name should match the peer id for the<a name="line.742"></a>
-<span class="sourceLineNo">743</span> // replication source.<a name="line.743"></a>
-<span class="sourceLineNo">744</span> List<String> remoteWals = wals.stream().filter(w -> SyncReplicationWALProvider<a name="line.744"></a>
-<span class="sourceLineNo">745</span> .getSyncReplicationPeerIdFromWALName(w).map(peerId::equals).orElse(false))<a name="line.745"></a>
-<span class="sourceLineNo">746</span> .collect(Collectors.toList());<a name="line.746"></a>
-<span class="sourceLineNo">747</span> LOG.debug("Removing {} logs from remote dir {} in the list: {}", remoteWals.size(),<a name="line.747"></a>
-<span class="sourceLineNo">748</span> remoteWALDir, remoteWals);<a name="line.748"></a>
-<span class="sourceLineNo">749</span> if (!remoteWals.isEmpty()) {<a name="line.749"></a>
-<span class="sourceLineNo">750</span> for (int sleepMultiplier = 0;;) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> try {<a name="line.751"></a>
-<span class="sourceLineNo">752</span> removeRemoteWALs(peerId, remoteWALDir, remoteWals);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> break;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> LOG.warn("Failed to delete remote wals from remote dir {} for peer {}", remoteWALDir,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> peerId);<a name="line.756"></a>
-<span class="sourceLineNo">757</span> }<a name="line.757"></a>
-<span class="sourceLineNo">758</span> if (!source.isSourceActive()) {<a name="line.758"></a>
-<span class="sourceLineNo">759</span> // skip the following operations<a name="line.759"></a>
-<span class="sourceLineNo">760</span> return;<a name="line.760"></a>
-<span class="sourceLineNo">761</span> }<a name="line.761"></a>
-<span class="sourceLineNo">762</span> if (ReplicationUtils.sleepForRetries("Failed to delete remote wals", sleepForRetries,<a name="line.762"></a>
-<span class="sourceLineNo">763</span> sleepMultiplier, maxRetriesMultiplier)) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> sleepMultiplier++;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> }<a name="line.766"></a>
-<span class="sourceLineNo">767</span> }<a name="line.767"></a>
-<span class="sourceLineNo">768</span> }<a name="line.768"></a>
-<span class="sourceLineNo">769</span> String queueId = source.getQueueId();<a name="line.769"></a>
-<span class="sourceLineNo">770</span> for (String wal : wals) {<a name="line.770"></a>
-<span class="sourceLineNo">771</span> interruptOrAbortWhenFail(<a name="line.771"></a>
-<span class="sourceLineNo">772</span> () -> this.queueStorage.removeWAL(server.getServerName(), queueId, wal));<a name="line.772"></a>
-<span class="sourceLineNo">773</span> }<a name="line.773"></a>
-<span class="sourceLineNo">774</span> }<a name="line.774"></a>
-<span class="sourceLineNo">775</span><a name="line.775"></a>
-<span class="sourceLineNo">776</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.776"></a>
-<span class="sourceLineNo">777</span> public void preLogRoll(Path newLog) throws IOException {<a name="line.777"></a>
-<span class="sourceLineNo">778</span> String logName = newLog.getName();<a name="line.778"></a>
-<span class="sourceLineNo">779</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);<a name="line.779"></a>
-<span class="sourceLineNo">780</span> // synchronized on latestPaths to avoid the new open source miss the new log<a name="line.780"></a>
-<span class="sourceLineNo">781</span> synchronized (this.latestPaths) {<a name="line.781"></a>
-<span class="sourceLineNo">782</span> // Add log to queue storage<a name="line.782"></a>
-<span class="sourceLineNo">783</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span> // If record log to queue storage failed, abort RS and throw exception to make log roll<a name="line.784"></a>
-<span class="sourceLineNo">785</span> // failed<a name="line.785"></a>
-<span class="sourceLineNo">786</span> abortAndThrowIOExceptionWhenFail(<a name="line.786"></a>
-<span class="sourceLineNo">787</span> () -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));<a name="line.787"></a>
-<span class="sourceLineNo">788</span> }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span> // synchronized on walsById to avoid race with cleanOldLogs<a name="line.790"></a>
-<span class="sourceLineNo">791</span> synchronized (this.walsById) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span> // Update walsById map<a name="line.792"></a>
-<span class="sourceLineNo">793</span> for (Map.Entry<String, Map<String, NavigableSet<String>>> entry : this.walsById<a name="line.793"></a>
-<span class="sourceLineNo">794</span> .entrySet()) {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> String peerId = entry.getKey();<a name="line.795"></a>
-<span class="sourceLineNo">796</span> Map<String, NavigableSet<String>> walsByPrefix = entry.getValue();<a name="line.796"></a>
-<span class="sourceLineNo">797</span> boolean existingPrefix = false;<a name="line.797"></a>
-<span class="sourceLineNo">798</span> for (Map.Entry<String, NavigableSet<String>> walsEntry : walsByPrefix.entrySet()) {<a name="line.798"></a>
-<span class="sourceLineNo">799</span> SortedSet<String> wals = walsEntry.getValue();<a name="line.799"></a>
-<span class="sourceLineNo">800</span> if (this.sources.isEmpty()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span> // If there's no slaves, don't need to keep the old wals since<a name="line.801"></a>
-<span class="sourceLineNo">802</span> // we only consider the last one when a new slave comes in<a name="line.802"></a>
-<span class="sourceLineNo">803</span> wals.clear();<a name="line.803"></a>
-<span class="sourceLineNo">804</span> }<a name="line.804"></a>
-<span class="sourceLineNo">805</span> if (logPrefix.equals(walsEntry.getKey())) {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> wals.add(logName);<a name="line.806"></a>
-<span class="sourceLineNo">807</span> existingPrefix = true;<a name="line.807"></a>
-<span class="sourceLineNo">808</span> }<a name="line.808"></a>
-<span class="sourceLineNo">809</span> }<a name="line.809"></a>
-<span class="sourceLineNo">810</span> if (!existingPrefix) {<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // The new log belongs to a new group, add it into this peer<a name="line.811"></a>
-<span class="sourceLineNo">812</span> LOG.debug("Start tracking logs for wal group {} for peer {}", logPrefix, peerId);<a name="line.812"></a>
-<span class="sourceLineNo">813</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.813"></a>
-<span class="sourceLineNo">814</span> wals.add(logName);<a name="line.814"></a>
-<span class="sourceLineNo">815</span> walsByPrefix.put(logPrefix, wals);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span> }<a name="line.817"></a>
-<span class="sourceLineNo">818</span> }<a name="line.818"></a>
-<span class="sourceLineNo">819</span><a name="line.819"></a>
-<span class="sourceLineNo">820</span> // Add to latestPaths<a name="line.820"></a>
-<span class="sourceLineNo">821</span> latestPaths.put(logPrefix, newLog);<a name="line.821"></a>
-<span class="sourceLineNo">822</span> }<a name="line.822"></a>
-<span class="sourceLineNo">823</span> }<a name="line.823"></a>
-<span class="sourceLineNo">824</span><a name="line.824"></a>
-<span class="sourceLineNo">825</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.825"></a>
-<span class="sourceLineNo">826</span> public void postLogRoll(Path newLog) throws IOException {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> // This only updates the sources we own, not the recovered ones<a name="line.827"></a>
-<span class="sourceLineNo">828</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.828"></a>
-<span class="sourceLineNo">829</span> source.enqueueLog(newLog);<a name="line.829"></a>
-<span class="sourceLineNo">830</span> LOG.trace("Enqueued {} to source {} while performing postLogRoll operation.",<a name="line.830"></a>
-<span class="sourceLineNo">831</span> newLog, source.getQueueId());<a name="line.831"></a>
-<span class="sourceLineNo">832</span> }<a name="line.832"></a>
-<span class="sourceLineNo">833</span> }<a name="line.833"></a>
-<span class="sourceLineNo">834</span><a name="line.834"></a>
-<span class="sourceLineNo">835</span> @Override<a name="line.835"></a>
-<span class="sourceLineNo">836</span> public void regionServerRemoved(String regionserver) {<a name="line.836"></a>
-<span class="sourceLineNo">837</span> transferQueues(ServerName.valueOf(regionserver));<a name="line.837"></a>
-<span class="sourceLineNo">838</span> }<a name="line.838"></a>
-<span class="sourceLineNo">839</span><a name="line.839"></a>
-<span class="sourceLineNo">840</span> /**<a name="line.840"></a>
-<span class="sourceLineNo">841</span> * Transfer all the queues of the specified to this region server. First it tries to grab a lock<a name="line.841"></a>
-<span class="sourceLineNo">842</span> * and if it works it will move the old queues and finally will delete the old queues.<a name="line.842"></a>
-<span class="sourceLineNo">843</span> * <p><a name="line.843"></a>
-<span class="sourceLineNo">844</span> * It creates one old source for any type of source of the old rs.<a name="line.844"></a>
-<span class="sourceLineNo">845</span> */<a name="line.845"></a>
-<span class="sourceLineNo">846</span> private void transferQueues(ServerName deadRS) {<a name="line.846"></a>
-<span class="sourceLineNo">847</span> if (server.getServerName().equals(deadRS)) {<a name="line.847"></a>
-<span class="sourceLineNo">848</span> // it's just us, give up<a name="line.848"></a>
-<span class="sourceLineNo">849</span> return;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> }<a name="line.850"></a>
-<span class="sourceLineNo">851</span> NodeFailoverWorker transfer = new NodeFailoverWorker(deadRS);<a name="line.851"></a>
-<span class="sourceLineNo">852</span> try {<a name="line.852"></a>
-<span class="sourceLineNo">853</span> this.executor.execute(transfer);<a name="line.853"></a>
-<span class="sourceLineNo">854</span> } catch (RejectedExecutionException ex) {<a name="line.854"></a>
-<span class="sourceLineNo">855</span> CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)<a name="line.855"></a>
-<span class="sourceLineNo">856</span> .getGlobalSource().incrFailedRecoveryQueue();<a name="line.856"></a>
-<span class="sourceLineNo">857</span> LOG.info("Cancelling the transfer of " + deadRS + " because of " + ex.getMessage());<a name="line.857"></a>
-<span class="sourceLineNo">858</span> }<a name="line.858"></a>
-<span class="sourceLineNo">859</span> }<a name="line.859"></a>
-<span class="sourceLineNo">860</span><a name="line.860"></a>
-<span class="sourceLineNo">861</span> /**<a name="line.861"></a>
-<span class="sourceLineNo">862</span> * Class responsible to setup new ReplicationSources to take care of the queues from dead region<a name="line.862"></a>
-<span class="sourceLineNo">863</span> * servers.<a name="line.863"></a>
-<span class="sourceLineNo">864</span> */<a name="line.864"></a>
-<span class="sourceLineNo">865</span> class NodeFailoverWorker extends Thread {<a name="line.865"></a>
-<span class="sourceLineNo">866</span><a name="line.866"></a>
-<span class="sourceLineNo">867</span> private final ServerName deadRS;<a name="line.867"></a>
-<span class="sourceLineNo">868</span> // After claim the queues from dead region server, the NodeFailoverWorker will skip to start<a name="line.868"></a>
-<span class="sourceLineNo">869</span> // the RecoveredReplicationSource if the peer has been removed. but there's possible that<a name="line.869"></a>
-<span class="sourceLineNo">870</span> // remove a peer with peerId = 2 and add a peer with peerId = 2 again during the<a name="line.870"></a>
-<span class="sourceLineNo">871</span> // NodeFailoverWorker. So we need a deep copied <peerId, peer> map to decide whether we<a name="line.871"></a>
-<span class="sourceLineNo">872</span> // should start the RecoveredReplicationSource. If the latest peer is not the old peer when<a name="line.872"></a>
-<span class="sourceLineNo">873</span> // NodeFailoverWorker begin, we should skip to start the RecoveredReplicationSource, Otherwise<a name="line.873"></a>
-<span class="sourceLineNo">874</span> // the rs will abort (See HBASE-20475).<a name="line.874"></a>
-<span class="sourceLineNo">875</span> private final Map<String, ReplicationPeerImpl> peersSnapshot;<a name="line.875"></a>
-<span class="sourceLineNo">876</span><a name="line.876"></a>
-<span class="sourceLineNo">877</span> public NodeFailoverWorker(ServerName deadRS) {<a name="line.877"></a>
-<span class="sourceLineNo">878</span> super("Failover-for-" + deadRS);<a name="line.878"></a>
-<span class="sourceLineNo">879</span> this.deadRS = deadRS;<a name="line.879"></a>
-<span class="sourceLineNo">880</span> peersSnapshot = new HashMap<>(replicationPeers.getPeerCache());<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span><a name="line.882"></a>
-<span class="sourceLineNo">883</span> private boolean isOldPeer(String peerId, ReplicationPeerImpl newPeerRef) {<a name="line.883"></a>
-<span class="sourceLineNo">884</span> ReplicationPeerImpl oldPeerRef = peersSnapshot.get(peerId);<a name="line.884"></a>
-<span class="sourceLineNo">885</span> return oldPeerRef != null && oldPeerRef == newPeerRef;<a name="line.885"></a>
-<span class="sourceLineNo">886</span> }<a name="line.886"></a>
-<span class="sourceLineNo">887</span><a name="line.887"></a>
-<span class="sourceLineNo">888</span> @Override<a name="line.888"></a>
-<span class="sourceLineNo">889</span> public void run() {<a name="line.889"></a>
-<span class="sourceLineNo">890</span> // Wait a bit before transferring the queues, we may be shutting down.<a name="line.890"></a>
-<span class="sourceLineNo">891</span> // This sleep may not be enough in some cases.<a name="line.891"></a>
-<span class="sourceLineNo">892</span> try {<a name="line.892"></a>
-<span class="sourceLineNo">893</span> Thread.sleep(sleepBeforeFailover +<a name="line.893"></a>
-<span class="sourceLineNo">894</span> (long) (ThreadLocalRandom.current().nextFloat() * sleepBeforeFailover));<a name="line.894"></a>
-<span class="sourceLineNo">895</span> } catch (InterruptedException e) {<a name="line.895"></a>
-<span class="sourceLineNo">896</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.896"></a>
-<span class="sourceLineNo">897</span> Thread.currentThread().interrupt();<a name="line.897"></a>
-<span class="sourceLineNo">898</span> }<a name="line.898"></a>
-<span class="sourceLineNo">899</span> // We try to lock that rs' queue directory<a name="line.899"></a>
-<span class="sourceLineNo">900</span> if (server.isStopped()) {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> LOG.info("Not transferring queue since we are shutting down");<a name="line.901"></a>
-<span class="sourceLineNo">902</span> return;<a name="line.902"></a>
-<span class="sourceLineNo">903</span> }<a name="line.903"></a>
-<span class="sourceLineNo">904</span> Map<String, Set<String>> newQueues = new HashMap<>();<a name="line.904"></a>
-<span class="sourceLineNo">905</span> try {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> List<String> queues = queueStorage.getAllQueues(deadRS);<a name="line.906"></a>
-<span class="sourceLineNo">907</span> while (!queues.isEmpty()) {<a name="line.907"></a>
-<span class="sourceLineNo">908</span> Pair<String, SortedSet<String>> peer = queueStorage.claimQueue(deadRS,<a name="line.908"></a>
-<span class="sourceLineNo">909</span> queues.get(ThreadLocalRandom.current().nextInt(queues.size())), server.getServerName());<a name="line.909"></a>
-<span class="sourceLineNo">910</span> long sleep = sleepBeforeFailover / 2;<a name="line.910"></a>
-<span class="sourceLineNo">911</span> if (!peer.getSecond().isEmpty()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> newQueues.put(peer.getFirst(), peer.getSecond());<a name="line.912"></a>
-<span class="sourceLineNo">913</span> sleep = sleepBeforeFailover;<a name="line.913"></a>
-<span class="sourceLineNo">914</span> }<a name="line.914"></a>
-<span class="sourceLineNo">915</span> try {<a name="line.915"></a>
-<span class="sourceLineNo">916</span> Thread.sleep(sleep);<a name="line.916"></a>
-<span class="sourceLineNo">917</span> } catch (InterruptedException e) {<a name="line.917"></a>
-<span class="sourceLineNo">918</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.918"></a>
-<span class="sourceLineNo">919</span> Thread.currentThread().interrupt();<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span> queues = queueStorage.getAllQueues(deadRS);<a name="line.921"></a>
-<span class="sourceLineNo">922</span> }<a name="line.922"></a>
-<span class="sourceLineNo">923</span> if (queues.isEmpty()) {<a name="line.923"></a>
-<span class="sourceLineNo">924</span> queueStorage.removeReplicatorIfQueueIsEmpty(deadRS);<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span> } catch (ReplicationException e) {<a name="line.926"></a>
-<span class="sourceLineNo">927</span> LOG.error(String.format("ReplicationException: cannot claim dead region (%s)'s " +<a name="line.927"></a>
-<span class="sourceLineNo">928</span> "replication queue. Znode : (%s)" +<a name="line.928"></a>
-<span class="sourceLineNo">929</span> " Possible solution: check if znode size exceeds jute.maxBuffer value. " +<a name="line.929"></a>
-<span class="sourceLineNo">930</span> " If so, increase it for both client and server side." + e), deadRS,<a name="line.930"></a>
-<span class="sourceLineNo">931</span> queueStorage.getRsNode(deadRS));<a name="line.931"></a>
-<span class="sourceLineNo">932</span> server.abort("Failed to claim queue from dead regionserver.", e);<a name="line.932"></a>
-<span class="sourceLineNo">933</span> return;<a name="line.933"></a>
-<span class="sourceLineNo">934</span> }<a name="line.934"></a>
-<span class="sourceLineNo">935</span> // Copying over the failed queue is completed.<a name="line.935"></a>
-<span class="sourceLineNo">936</span> if (newQueues.isEmpty()) {<a name="line.936"></a>
-<span class="sourceLineNo">937</span> // We either didn't get the lock or the failed region server didn't have any outstanding<a name="line.937"></a>
-<span class="sourceLineNo">938</span> // WALs to replicate, so we are done.<a name="line.938"></a>
-<span class="sourceLineNo">939</span> return;<a name="line.939"></a>
-<span class="sourceLineNo">940</span> }<a name="line.940"></a>
-<span class="sourceLineNo">941</span><a name="line.941"></a>
-<span class="sourceLineNo">942</span> for (Map.Entry<String, Set<String>> entry : newQueues.entrySet()) {<a name="line.942"></a>
-<span class="sourceLineNo">943</span> String queueId = entry.getKey();<a name="line.943"></a>
-<span class="sourceLineNo">944</span> Set<String> walsSet = entry.getValue();<a name="line.944"></a>
-<span class="sourceLineNo">945</span> try {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> // there is not an actual peer defined corresponding to peerId for the failover.<a name="line.946"></a>
-<span class="sourceLineNo">947</span> ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);<a name="line.947"></a>
-<span class="sourceLineNo">948</span> String actualPeerId = replicationQueueInfo.getPeerId();<a name="line.948"></a>
-<span class="sourceLineNo">949</span><a name="line.949"></a>
-<span class="sourceLineNo">950</span> ReplicationPeerImpl peer = replicationPeers.getPeer(actualPeerId);<a name="line.950"></a>
-<span class="sourceLineNo">951</span> if (peer == null || !isOldPeer(actualPeerId, peer)) {<a name="line.951"></a>
-<span class="sourceLineNo">952</span> LOG.warn("Skipping failover for peer {} of node {}, peer is null", actualPeerId,<a name="line.952"></a>
-<span class="sourceLineNo">953</span> deadRS);<a name="line.953"></a>
-<span class="sourceLineNo">954</span> abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.954"></a>
-<span class="sourceLineNo">955</span> continue;<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> if (server instanceof ReplicationSyncUp.DummyServer<a name="line.957"></a>
-<span class="sourceLineNo">958</span> && peer.getPeerState().equals(PeerState.DISABLED)) {<a name="line.958"></a>
-<span class="sourceLineNo">959</span> LOG.warn("Peer {} is disabled. ReplicationSyncUp tool will skip "<a name="line.959"></a>
-<span class="sourceLineNo">960</span> + "replicating data to this peer.",<a name="line.960"></a>
-<span class="sourceLineNo">961</span> actualPeerId);<a name="line.961"></a>
-<span class="sourceLineNo">962</span> continue;<a name="line.962"></a>
-<span class="sourceLineNo">963</span> }<a name="line.963"></a>
-<span class="sourceLineNo">964</span><a name="line.964"></a>
-<span class="sourceLineNo">965</span> ReplicationSourceInterface src = createSource(queueId, peer);<a name="line.965"></a>
-<span class="sourceLineNo">966</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.966"></a>
-<span class="sourceLineNo">967</span> synchronized (oldsources) {<a name="line.967"></a>
-<span class="sourceLineNo">968</span> peer = replicationPeers.getPeer(src.getPeerId());<a name="line.968"></a>
-<span class="sourceLineNo">969</span> if (peer == null || !isOldPeer(src.getPeerId(), peer)) {<a name="line.969"></a>
-<span class="sourceLineNo">970</span> src.terminate("Recovered queue doesn't belong to any current peer");<a name="line.970"></a>
-<span class="sourceLineNo">971</span> deleteQueue(queueId);<a name="line.971"></a>
-<span class="sourceLineNo">972</span> continue;<a name="line.972"></a>
-<span class="sourceLineNo">973</span> }<a name="line.973"></a>
-<span class="sourceLineNo">974</span> // Do not setup recovered queue if a sync replication peer is in STANDBY state, or is<a name="line.974"></a>
-<span class="sourceLineNo">975</span> // transiting to STANDBY state. The only exception is we are in STANDBY state and<a name="line.975"></a>
-<span class="sourceLineNo">976</span> // transiting to DA, under this state we will replay the remote WAL and they need to be<a name="line.976"></a>
-<span class="sourceLineNo">977</span> // replicated back.<a name="line.977"></a>
-<span class="sourceLineNo">978</span> if (peer.getPeerConfig().isSyncReplication()) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> Pair<SyncReplicationState, SyncReplicationState> stateAndNewState =<a name="line.979"></a>
-<span class="sourceLineNo">980</span> peer.getSyncReplicationStateAndNewState();<a name="line.980"></a>
-<span class="sourceLineNo">981</span> if ((stateAndNewState.getFirst().equals(SyncReplicationState.STANDBY) &&<a name="line.981"></a>
-<span class="sourceLineNo">982</span> stateAndNewState.getSecond().equals(SyncReplicationState.NONE)) ||<a name="line.982"></a>
-<span class="sourceLineNo">983</span> stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)) {<a name="line.983"></a>
-<span class="sourceLineNo">984</span> src.terminate("Sync replication peer is in STANDBY state");<a name="line.984"></a>
-<span class="sourceLineNo">985</span> deleteQueue(queueId);<a name="line.985"></a>
-<span class="sourceLineNo">986</span> continue;<a name="line.986"></a>
-<span class="sourceLineNo">987</span> }<a name="line.987"></a>
-<span class="sourceLineNo">988</span> }<a name="line.988"></a>
-<span class="sourceLineNo">989</span> // track sources in walsByIdRecoveredQueues<a name="line.989"></a>
-<span class="sourceLineNo">990</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.990"></a>
-<span class="sourceLineNo">991</span> walsByIdRecoveredQueues.put(queueId, walsByGroup);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> for (String wal : walsSet) {<a name="line.992"></a>
-<span class="sourceLineNo">993</span> String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal);<a name="line.993"></a>
-<span class="sourceLineNo">994</span> NavigableSet<String> wals = walsByGroup.get(walPrefix);<a name="line.994"></a>
-<span class="sourceLineNo">995</span> if (wals == null) {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> wals = new TreeSet<>();<a name="line.996"></a>
-<span class="sourceLineNo">997</span> walsByGroup.put(walPrefix, wals);<a name="line.997"></a>
-<span class="sourceLineNo">998</span> }<a name="line.998"></a>
-<span class="sourceLineNo">999</span> wals.add(wal);<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> }<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> oldsources.add(src);<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span> LOG.info("Added source for recovered queue {}", src.getQueueId());<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> for (String wal : walsSet) {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> LOG.trace("Enqueueing log from recovered queue for source: " + src.getQueueId());<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> src.enqueueLog(new Path(oldLogDir, wal));<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span> }<a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> src.startup();<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> }<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> } catch (IOException e) {<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> // TODO manage it<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span> LOG.error("Failed creating a source", e);<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> }<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> }<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> }<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span><a name="line.1016"></a>
-<span class="sourceLineNo">1017</span> /**<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> * Terminate the replication on this region server<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> */<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> public void join() {<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.executor.shutdown();<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> source.terminate("Region server is closing");<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span> }<a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> }<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span><a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> /**<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> * Get a copy of the wals of the normal sources on this rs<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> * @return a sorted set of wal names<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> */<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> public Map<String, Map<String, NavigableSet<String>>> getWALs() {<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> return Collections.unmodifiableMap(walsById);<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> /**<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> * Get a copy of the wals of the recovered sources on this rs<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> * @return a sorted set of wal names<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> */<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> Map<String, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> return Collections.unmodifiableMap(walsByIdRecoveredQueues);<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> }<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span><a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> /**<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> * Get a list of all the normal sources of this rs<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> * @return list of all normal sources<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> */<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> public List<ReplicationSourceInterface> getSources() {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> return new ArrayList<>(this.sources.values());<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> }<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span><a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> /**<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> * Get a list of all the recovered sources of this rs<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> * @return list of all recovered sources<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> */<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span> public List<ReplicationSourceInterface> getOldSources() {<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> return this.oldsources;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> }<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span><a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> /**<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> * Get the normal source for a given peer<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> * @return the normal source for the give peer if it exists, otherwise null.<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> */<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> public ReplicationSourceInterface getSource(String peerId) {<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span> return this.sources.get(peerId);<a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> }<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span><a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> List<String> getAllQueues() throws IOException {<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span> List<String> allQueues = Collections.emptyList();<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> try {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> allQueues = queueStorage.getAllQueues(server.getServerName());<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> } catch (ReplicationException e) {<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> throw new IOException(e);<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> return allQueues;<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> int getSizeOfLatestPath() {<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> synchronized (latestPaths) {<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> return latestPaths.size();<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> }<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> }<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> Set<Path> getLastestPath() {<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> synchronized (latestPaths) {<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span> return Sets.newHashSet(latestPaths.values());<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> }<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> }<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span><a name="line.1088"></a>
-<span class="sourceLineNo">1089</span> public AtomicLong getTotalBufferUsed() {<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span> return totalBufferUsed;<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span> }<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span><a name="line.1092"></a>
-<span class="sourceLineNo">1093</span> /**<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span> * Returns the maximum size in bytes of edits held in memory which are pending replication<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span> * across all sources inside this RegionServer.<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span> */<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span> public long getTotalBufferLimit() {<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span> return totalBufferLimit;<a name="line.1098"></a>
-<span class="sourceLineNo">1099</span> }<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span><a name="line.1100"></a>
-<span class="sourceLineNo">1101</span> /**<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span> * Get the directory where wals are archived<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span> * @return the directory where wals are archived<a name="line.1103"></a>
-<span class="sourceLineNo">1104</span> */<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span> public Path getOldLogDir() {<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span> return this.oldLogDir;<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span> }<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span><a name="line.1108"></a>
-<span class="sourceLineNo">1109</span> /**<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span> * Get the directory where wals are stored by their RSs<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span> * @return the directory where wals are stored by their RSs<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span> */<a name="line.1112"></a>
-<span class="sourceLineNo">1113</span> public Path getLogDir() {<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span> return this.logDir;<a name="line.1114"></a>
-<span class="sourceLineNo">1115</span> }<a name="line.1115"></a>
-<span class="sourceLineNo">1116</span><a name="line.1116"></a>
-<span class="sourceLineNo">1117</span> /**<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span> * Get the handle on the local file system<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span> * @return Handle on the local file system<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span> */<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span> public FileSystem getFs() {<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span> return this.fs;<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span> }<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span><a name="line.1124"></a>
-<span class="sourceLineNo">1125</span> /**<a name="line.1125"></a>
-<span class="sourceLineNo">1126</span> * Get the ReplicationPeers used by this ReplicationSourceManager<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span> * @return the ReplicationPeers used by this ReplicationSourceManager<a name="line.1127"></a>
-<span class="sourceLineNo">1128</span> */<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span> public ReplicationPeers getReplicationPeers() {<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span> return this.replicationPeers;<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span> }<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span><a name="line.1132"></a>
-<span class="sourceLineNo">1133</span> /**<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span> * Get a string representation of all the sources' metrics<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span> */<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span> public String getStats() {<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span> StringBuilder stats = new StringBuilder();<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span> // Print stats that apply across all Replication Sources<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span> stats.append("Global stats: ");<a name="line.1139"></a>
-<span class="sourceLineNo">1140</span> stats.append("WAL Edits Buffer Used=").append(getTotalBufferUsed().get()).append("B, Limit=")<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span> .append(getTotalBufferLimit()).append("B\n");<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span> stats.append("Normal source for cluster " + source.getPeerId() + ": ");<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span> stats.append(source.getStats() + "\n");<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span> }<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span> for (ReplicationSourceInterface oldSource : oldsources) {<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span> stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerId() + ": ");<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span> stats.append(oldSource.getStats() + "\n");<a name="line.1148"></a>
-<span class="sourceLineNo">1149</span> }<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span> return stats.toString();<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span> }<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span><a name="line.1152"></a>
-<span class="sourceLineNo">1153</span> public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span> throws IOException {<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span> throwIOExceptionWhenFail(() -> source.addHFileRefs(tableName, family, pairs));<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span> }<a name="line.1157"></a>
-<span class="sourceLineNo">1158</span> }<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span><a name="line.1159"></a>
-<span class="sourceLineNo">1160</span> public void cleanUpHFileRefs(String peerId, List<String> files) {<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span> interruptOrAbortWhenFail(() -> this.queueStorage.removeHFileRefs(peerId, files));<a name="line.1161"></a>
-<span class="sourceLineNo">1162</span> }<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span><a name="line.1163"></a>
-<span class="sourceLineNo">1164</span> int activeFailoverTaskCount() {<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span> return executor.getActiveCount();<a name="line.1165"></a>
-<span class="sourceLineNo">1166</span> }<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span><a name="line.1167"></a>
-<span class="sourceLineNo">1168</span> MetricsReplicationGlobalSourceSource getGlobalMetrics() {<a name="line.1168"></a>
-<span class="sourceLineNo">1169</span> return this.globalMetrics;<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span> }<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span><a name="line.1171"></a>
-<span class="sourceLineNo">1172</span> /**<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span> * Add an hbase:meta Catalog replication source. Called on open of an hbase:meta Region.<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span> * Create it once only. If exists already, use the existing one.<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span> * @see #removeCatalogReplicationSource(RegionInfo)<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span> * @see #addSource(String) This is specialization on the addSource method.<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span> */<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span> public ReplicationSourceInterface addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span> throws IOException {<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span> // Poor-man's putIfAbsent<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span> synchronized (this.catalogReplicationSource) {<a name="line.1181"></a>
-<span class="sourceLineNo">1182</span> ReplicationSourceInterface rs = this.catalogReplicationSource.get();<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span> return rs != null ? rs :<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span> this.catalogReplicationSource.getAndSet(createCatalogReplicationSource(regionInfo));<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span> }<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span> }<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span><a name="line.1187"></a>
-<span class="sourceLineNo">1188</span> /**<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span> * Remove the hbase:meta Catalog replication source.<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span> * Called when we close hbase:meta.<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span> * @see #addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span> */<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span> public void removeCatalogReplicationSource(RegionInfo regionInfo) {<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span> // Nothing to do. Leave any CatalogReplicationSource in place in case an hbase:meta Region<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span> // comes back to this server.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span> }<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span><a name="line.1197"></a>
-<span class="sourceLineNo">1198</span> /**<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span> * Create, initialize, and start the Catalog ReplicationSource.<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span> * Presumes called one-time only (caller must ensure one-time only call).<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span> * This ReplicationSource is NOT created via {@link ReplicationSourceFactory}.<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span> * @see #addSource(String) This is a specialization of the addSource call.<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span> * @see #catalogReplicationSource for a note on this ReplicationSource's lifecycle (and more on<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span> * why the special handling).<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span> */<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span> private ReplicationSourceInterface createCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span> throws IOException {<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span> // Instantiate meta walProvider. Instantiated here or over in the #warmupRegion call made by the<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span> // Master on a 'move' operation. Need to do extra work if we did NOT instantiate the provider.<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span> WALProvider walProvider = this.walFactory.getMetaWALProvider();<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span> boolean instantiate = walProvider == null;<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span> if (instantiate) {<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span> walProvider = this.walFactory.getMetaProvider();<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span> }<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span> // Here we do a specialization on what {@link ReplicationSourceFactory} does. There is no need<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span> // for persisting offset into WALs up in zookeeper (via ReplicationQueueInfo) as the catalog<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span> // read replicas feature that makes use of the source does a reset on a crash of the WAL<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span> // source process. See "4.1 Skip maintaining zookeeper replication queue (offsets/WALs)" in the<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span> // design doc attached to HBASE-18070 'Enable memstore replication for meta replica' for detail.<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span> CatalogReplicationSourcePeer peer = new CatalogReplicationSourcePeer(this.conf,<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span> this.clusterId.toString(), "meta_" + ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER);<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span> final ReplicationSourceInterface crs = new CatalogReplicationSource();<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span> crs.init(conf, fs, this, new NoopReplicationQueueStorage(), peer, server, peer.getId(),<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span> clusterId, walProvider.getWALFileLengthProvider(), new MetricsSource(peer.getId()));<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span> // Add listener on the provider so we can pick up the WAL to replicate on roll.<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span> WALActionsListener listener = new WALActionsListener() {<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span> @Override public void postLogRoll(Path oldPath, Path newPath) throws IOException {<a name="line.1227"></a>
-<span class="sourceLineNo">1228</span> crs.enqueueLog(newPath);<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span> }<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span> };<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span> walProvider.addWALActionsListener(listener);<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span> if (!instantiate) {<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span> // If we did not instantiate provider, need to add our listener on already-created WAL<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span> // instance too (listeners are passed by provider to WAL instance on creation but if provider<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span> // created already, our listener add above is missed). And add the current WAL file to the<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span> // Replication Source so it can start replicating it.<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span> WAL wal = walProvider.getWAL(regionInfo);<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span> wal.registerWALActionsListener(listener);<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span> crs.enqueueLog(((AbstractFSWAL)wal).getCurrentFileName());<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span> }<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span> return crs.startup();<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span> }<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>}<a name="line.1243"></a>
+<span class="sourceLineNo">286</span> List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers();<a name="line.286"></a>
+<span class="sourceLineNo">287</span> LOG.info(<a name="line.287"></a>
+<span class="sourceLineNo">288</span> "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);<a name="line.288"></a>
+<span class="sourceLineNo">289</span><a name="line.289"></a>
+<span class="sourceLineNo">290</span> // Look if there's anything to process after a restart<a name="line.290"></a>
+<span class="sourceLineNo">291</span> for (ServerName rs : currentReplicators) {<a name="line.291"></a>
+<span class="sourceLineNo">292</span> if (!otherRegionServers.contains(rs)) {<a name="line.292"></a>
+<span class="sourceLineNo">293</span> transferQueues(rs);<a name="line.293"></a>
+<span class="sourceLineNo">294</span> }<a name="line.294"></a>
+<span class="sourceLineNo">295</span> }<a name="line.295"></a>
+<span class="sourceLineNo">296</span> }<a name="line.296"></a>
+<span class="sourceLineNo">297</span><a name="line.297"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * <ol><a name="line.299"></a>
+<span class="sourceLineNo">300</span> * <li>Add peer to replicationPeers</li><a name="line.300"></a>
+<span class="sourceLineNo">301</span> * <li>Add the normal source and related replication queue</li><a name="line.301"></a>
+<span class="sourceLineNo">302</span> * <li>Add HFile Refs</li><a name="line.302"></a>
+<span class="sourceLineNo">303</span> * </ol><a name="line.303"></a>
+<span class="sourceLineNo">304</span> * @param peerId the id of replication peer<a name="line.304"></a>
+<span class="sourceLineNo">305</span> */<a name="line.305"></a>
+<span class="sourceLineNo">306</span> public void addPeer(String peerId) throws IOException {<a name="line.306"></a>
+<span class="sourceLineNo">307</span> boolean added = false;<a name="line.307"></a>
+<span class="sourceLineNo">308</span> try {<a name="line.308"></a>
+<span class="sourceLineNo">309</span> added = this.replicationPeers.addPeer(peerId);<a name="line.309"></a>
+<span class="sourceLineNo">310</span> } catch (ReplicationException e) {<a name="line.310"></a>
+<span class="sourceLineNo">311</span> throw new IOException(e);<a name="line.311"></a>
+<span class="sourceLineNo">312</span> }<a name="line.312"></a>
+<span class="sourceLineNo">313</span> if (added) {<a name="line.313"></a>
+<span class="sourceLineNo">314</span> addSource(peerId);<a name="line.314"></a>
+<span class="sourceLineNo">315</span> if (replicationForBulkLoadDataEnabled) {<a name="line.315"></a>
+<span class="sourceLineNo">316</span> throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));<a name="line.316"></a>
+<span class="sourceLineNo">317</span> }<a name="line.317"></a>
+<span class="sourceLineNo">318</span> }<a name="line.318"></a>
+<span class="sourceLineNo">319</span> }<a name="line.319"></a>
+<span class="sourceLineNo">320</span><a name="line.320"></a>
+<span class="sourceLineNo">321</span> /**<a name="line.321"></a>
+<span class="sourceLineNo">322</span> * <ol><a name="line.322"></a>
+<span class="sourceLineNo">323</span> * <li>Remove peer for replicationPeers</li><a name="line.323"></a>
+<span class="sourceLineNo">324</span> * <li>Remove all the recovered sources for the specified id and related replication queues</li><a name="line.324"></a>
+<span class="sourceLineNo">325</span> * <li>Remove the normal source and related replication queue</li><a name="line.325"></a>
+<span class="sourceLineNo">326</span> * <li>Remove HFile Refs</li><a name="line.326"></a>
+<span class="sourceLineNo">327</span> * </ol><a name="line.327"></a>
+<span class="sourceLineNo">328</span> * @param peerId the id of the replication peer<a name="line.328"></a>
+<span class="sourceLineNo">329</span> */<a name="line.329"></a>
+<span class="sourceLineNo">330</span> public void removePeer(String peerId) {<a name="line.330"></a>
+<span class="sourceLineNo">331</span> ReplicationPeer peer = replicationPeers.removePeer(peerId);<a name="line.331"></a>
+<span class="sourceLineNo">332</span> String terminateMessage = "Replication stream was removed by a user";<a name="line.332"></a>
+<span class="sourceLineNo">333</span> List<ReplicationSourceInterface> oldSourcesToDelete = new ArrayList<>();<a name="line.333"></a>
+<span class="sourceLineNo">334</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.334"></a>
+<span class="sourceLineNo">335</span> // see NodeFailoverWorker.run<a name="line.335"></a>
+<span class="sourceLineNo">336</span> synchronized (this.oldsources) {<a name="line.336"></a>
+<span class="sourceLineNo">337</span> // First close all the recovered sources for this peer<a name="line.337"></a>
+<span class="sourceLineNo">338</span> for (ReplicationSourceInterface src : oldsources) {<a name="line.338"></a>
+<span class="sourceLineNo">339</span> if (peerId.equals(src.getPeerId())) {<a name="line.339"></a>
+<span class="sourceLineNo">340</span> oldSourcesToDelete.add(src);<a name="line.340"></a>
+<span class="sourceLineNo">341</span> }<a name="line.341"></a>
+<span class="sourceLineNo">342</span> }<a name="line.342"></a>
+<span class="sourceLineNo">343</span> for (ReplicationSourceInterface src : oldSourcesToDelete) {<a name="line.343"></a>
+<span class="sourceLineNo">344</span> src.terminate(terminateMessage);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> removeRecoveredSource(src);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> }<a name="line.346"></a>
+<span class="sourceLineNo">347</span> }<a name="line.347"></a>
+<span class="sourceLineNo">348</span> LOG.info(<a name="line.348"></a>
+<span class="sourceLineNo">349</span> "Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());<a name="line.349"></a>
+<span class="sourceLineNo">350</span> // Now close the normal source for this peer<a name="line.350"></a>
+<span class="sourceLineNo">351</span> ReplicationSourceInterface srcToRemove = this.sources.get(peerId);<a name="line.351"></a>
+<span class="sourceLineNo">352</span> if (srcToRemove != null) {<a name="line.352"></a>
+<span class="sourceLineNo">353</span> srcToRemove.terminate(terminateMessage);<a name="line.353"></a>
+<span class="sourceLineNo">354</span> removeSource(srcToRemove);<a name="line.354"></a>
+<span class="sourceLineNo">355</span> } else {<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // Delete queue from storage and memory and queue id is same with peer id for normal<a name="line.357"></a>
+<span class="sourceLineNo">358</span> // source<a name="line.358"></a>
+<span class="sourceLineNo">359</span> deleteQueue(peerId);<a name="line.359"></a>
+<span class="sourceLineNo">360</span> this.walsById.remove(peerId);<a name="line.360"></a>
+<span class="sourceLineNo">361</span> }<a name="line.361"></a>
+<span class="sourceLineNo">362</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.362"></a>
+<span class="sourceLineNo">363</span> if (peerConfig.isSyncReplication()) {<a name="line.363"></a>
+<span class="sourceLineNo">364</span> syncReplicationPeerMappingManager.remove(peerId, peerConfig);<a name="line.364"></a>
+<span class="sourceLineNo">365</span> }<a name="line.365"></a>
+<span class="sourceLineNo">366</span> // Remove HFile Refs<a name="line.366"></a>
+<span class="sourceLineNo">367</span> abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));<a name="line.367"></a>
+<span class="sourceLineNo">368</span> }<a name="line.368"></a>
+<span class="sourceLineNo">369</span><a name="line.369"></a>
+<span class="sourceLineNo">370</span> /**<a name="line.370"></a>
+<span class="sourceLineNo">371</span> * @return a new 'classic' user-space replication source.<a name="line.371"></a>
+<span class="sourceLineNo">372</span> * @param queueId the id of the replication queue to associate the ReplicationSource with.<a name="line.372"></a>
+<span class="sourceLineNo">373</span> * @see #createCatalogReplicationSource(RegionInfo) for creating a ReplicationSource for meta.<a name="line.373"></a>
+<span class="sourceLineNo">374</span> */<a name="line.374"></a>
+<span class="sourceLineNo">375</span> private ReplicationSourceInterface createSource(String queueId, ReplicationPeer replicationPeer)<a name="line.375"></a>
+<span class="sourceLineNo">376</span> throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueId);<a name="line.377"></a>
+<span class="sourceLineNo">378</span> // Init the just created replication source. Pass the default walProvider's wal file length<a name="line.378"></a>
+<span class="sourceLineNo">379</span> // provider. Presumption is we replicate user-space Tables only. For hbase:meta region replica<a name="line.379"></a>
+<span class="sourceLineNo">380</span> // replication, see #createCatalogReplicationSource().<a name="line.380"></a>
+<span class="sourceLineNo">381</span> WALFileLengthProvider walFileLengthProvider =<a name="line.381"></a>
+<span class="sourceLineNo">382</span> this.walFactory.getWALProvider() != null?<a name="line.382"></a>
+<span class="sourceLineNo">383</span> this.walFactory.getWALProvider().getWALFileLengthProvider() : p -> OptionalLong.empty();<a name="line.383"></a>
+<span class="sourceLineNo">384</span> src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId,<a name="line.384"></a>
+<span class="sourceLineNo">385</span> walFileLengthProvider, new MetricsSource(queueId));<a name="line.385"></a>
+<span class="sourceLineNo">386</span> return src;<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> /**<a name="line.389"></a>
+<span class="sourceLineNo">390</span> * Add a normal source for the given peer on this region server. Meanwhile, add new replication<a name="line.390"></a>
+<span class="sourceLineNo">391</span> * queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal<a name="line.391"></a>
+<span class="sourceLineNo">392</span> * group and do replication<a name="line.392"></a>
+<span class="sourceLineNo">393</span> * @param peerId the id of the replication peer<a name="line.393"></a>
+<span class="sourceLineNo">394</span> * @return the source that was created<a name="line.394"></a>
+<span class="sourceLineNo">395</span> */<a name="line.395"></a>
+<span class="sourceLineNo">396</span> ReplicationSourceInterface addSource(String peerId) throws IOException {<a name="line.396"></a>
+<span class="sourceLineNo">397</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.397"></a>
+<span class="sourceLineNo">398</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.398"></a>
+<span class="sourceLineNo">399</span> // synchronized on latestPaths to avoid missing the new log<a name="line.399"></a>
+<span class="sourceLineNo">400</span> synchronized (this.latestPaths) {<a name="line.400"></a>
+<span class="sourceLineNo">401</span> this.sources.put(peerId, src);<a name="line.401"></a>
+<span class="sourceLineNo">402</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.402"></a>
+<span class="sourceLineNo">403</span> this.walsById.put(peerId, walsByGroup);<a name="line.403"></a>
+<span class="sourceLineNo">404</span> // Add the latest wal to that source's queue<a name="line.404"></a>
+<span class="sourceLineNo">405</span> if (!latestPaths.isEmpty()) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span> for (Map.Entry<String, Path> walPrefixAndPath : latestPaths.entrySet()) {<a name="line.406"></a>
+<span class="sourceLineNo">407</span> Path walPath = walPrefixAndPath.getValue();<a name="line.407"></a>
+<span class="sourceLineNo">408</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.408"></a>
+<span class="sourceLineNo">409</span> wals.add(walPath.getName());<a name="line.409"></a>
+<span class="sourceLineNo">410</span> walsByGroup.put(walPrefixAndPath.getKey(), wals);<a name="line.410"></a>
+<span class="sourceLineNo">411</span> // Abort RS and throw exception to make add peer failed<a name="line.411"></a>
+<span class="sourceLineNo">412</span> abortAndThrowIOExceptionWhenFail(<a name="line.412"></a>
+<span class="sourceLineNo">413</span> () -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName()));<a name="line.413"></a>
+<span class="sourceLineNo">414</span> src.enqueueLog(walPath);<a name="line.414"></a>
+<span class="sourceLineNo">415</span> LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());<a name="line.415"></a>
+<span class="sourceLineNo">416</span> }<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.419"></a>
+<span class="sourceLineNo">420</span> if (peerConfig.isSyncReplication()) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span> syncReplicationPeerMappingManager.add(peer.getId(), peerConfig);<a name="line.421"></a>
+<span class="sourceLineNo">422</span> }<a name="line.422"></a>
+<span class="sourceLineNo">423</span> src.startup();<a name="line.423"></a>
+<span class="sourceLineNo">424</span> return src;<a name="line.424"></a>
+<span class="sourceLineNo">425</span> }<a name="line.425"></a>
+<span class="sourceLineNo">426</span><a name="line.426"></a>
+<span class="sourceLineNo">427</span> /**<a name="line.427"></a>
+<span class="sourceLineNo">428</span> * <p><a name="line.428"></a>
+<span class="sourceLineNo">429</span> * This is used when we transit a sync replication peer to {@link SyncReplicationState#STANDBY}.<a name="line.429"></a>
+<span class="sourceLineNo">430</span> * </p><a name="line.430"></a>
+<span class="sourceLineNo">431</span> * <p><a name="line.431"></a>
+<span class="sourceLineNo">432</span> * When transiting to {@link SyncReplicationState#STANDBY}, we can remove all the pending wal<a name="line.432"></a>
+<span class="sourceLineNo">433</span> * files for a replication peer as we do not need to replicate them any more. And this is<a name="line.433"></a>
+<span class="sourceLineNo">434</span> * necessary, otherwise when we transit back to {@link SyncReplicationState#DOWNGRADE_ACTIVE}<a name="line.434"></a>
+<span class="sourceLineNo">435</span> * later, the stale data will be replicated again and cause inconsistency.<a name="line.435"></a>
+<span class="sourceLineNo">436</span> * </p><a name="line.436"></a>
+<span class="sourceLineNo">437</span> * <p><a name="line.437"></a>
+<span class="sourceLineNo">438</span> * See HBASE-20426 for more details.<a name="line.438"></a>
+<span class="sourceLineNo">439</span> * </p><a name="line.439"></a>
+<span class="sourceLineNo">440</span> * @param peerId the id of the sync replication peer<a name="line.440"></a>
+<span class="sourceLineNo">441</span> */<a name="line.441"></a>
+<span class="sourceLineNo">442</span> public void drainSources(String peerId) throws IOException, ReplicationException {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> String terminateMessage = "Sync replication peer " + peerId +<a name="line.443"></a>
+<span class="sourceLineNo">444</span> " is transiting to STANDBY. Will close the previous replication source and open a new one";<a name="line.444"></a>
+<span class="sourceLineNo">445</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.445"></a>
+<span class="sourceLineNo">446</span> assert peer.getPeerConfig().isSyncReplication();<a name="line.446"></a>
+<span class="sourceLineNo">447</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.447"></a>
+<span class="sourceLineNo">448</span> // synchronized here to avoid race with preLogRoll where we add new log to source and also<a name="line.448"></a>
+<span class="sourceLineNo">449</span> // walsById.<a name="line.449"></a>
+<span class="sourceLineNo">450</span> ReplicationSourceInterface toRemove;<a name="line.450"></a>
+<span class="sourceLineNo">451</span> Map<String, NavigableSet<String>> wals = new HashMap<>();<a name="line.451"></a>
+<span class="sourceLineNo">452</span> synchronized (latestPaths) {<a name="line.452"></a>
+<span class="sourceLineNo">453</span> toRemove = sources.put(peerId, src);<a name="line.453"></a>
+<span class="sourceLineNo">454</span> if (toRemove != null) {<a name="line.454"></a>
+<span class="sourceLineNo">455</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.455"></a>
+<span class="sourceLineNo">456</span> toRemove.terminate(terminateMessage);<a name="line.456"></a>
+<span class="sourceLineNo">457</span> toRemove.getSourceMetrics().clear();<a name="line.457"></a>
+<span class="sourceLineNo">458</span> }<a name="line.458"></a>
+<span class="sourceLineNo">459</span> // Here we make a copy of all the remaining wal files and then delete them from the<a name="line.459"></a>
+<span class="sourceLineNo">460</span> // replication queue storage after releasing the lock. It is not safe to just remove the old<a name="line.460"></a>
+<span class="sourceLineNo">461</span> // map from walsById since later we may fail to delete them from the replication queue<a name="line.461"></a>
+<span class="sourceLineNo">462</span> // storage, and when we retry next time, we can not know the wal files that need to be deleted<a name="line.462"></a>
+<span class="sourceLineNo">463</span> // from the replication queue storage.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> walsById.get(peerId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));<a name="line.464"></a>
+<span class="sourceLineNo">465</span> }<a name="line.465"></a>
+<span class="sourceLineNo">466</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.466"></a>
+<span class="sourceLineNo">467</span> src.startup();<a name="line.467"></a>
+<span class="sourceLineNo">468</span> for (NavigableSet<String> walsByGroup : wals.values()) {<a name="line.468"></a>
+<span class="sourceLineNo">469</span> for (String wal : walsByGroup) {<a name="line.469"></a>
+<span class="sourceLineNo">470</span> queueStorage.removeWAL(server.getServerName(), peerId, wal);<a name="line.470"></a>
+<span class="sourceLineNo">471</span> }<a name="line.471"></a>
+<span class="sourceLineNo">472</span> }<a name="line.472"></a>
+<span class="sourceLineNo">473</span> synchronized (walsById) {<a name="line.473"></a>
+<span class="sourceLineNo">474</span> Map<String, NavigableSet<String>> oldWals = walsById.get(peerId);<a name="line.474"></a>
+<span class="sourceLineNo">475</span> wals.forEach((k, v) -> {<a name="line.475"></a>
+<span class="sourceLineNo">476</span> NavigableSet<String> walsByGroup = oldWals.get(k);<a name="line.476"></a>
+<span class="sourceLineNo">477</span> if (walsByGroup != null) {<a name="line.477"></a>
+<span class="sourceLineNo">478</span> walsByGroup.removeAll(v);<a name="line.478"></a>
+<span class="sourceLineNo">479</span> }<a name="line.479"></a>
+<span class="sourceLineNo">480</span> });<a name="line.480"></a>
+<span class="sourceLineNo">481</span> }<a name="line.481"></a>
+<span class="sourceLineNo">482</span> // synchronized on oldsources to avoid race with NodeFailoverWorker. Since NodeFailoverWorker is<a name="line.482"></a>
+<span class="sourceLineNo">483</span> // a background task, we will delete the file from replication queue storage under the lock to<a name="line.483"></a>
+<span class="sourceLineNo">484</span> // simplify the logic.<a name="line.484"></a>
+<span class="sourceLineNo">485</span> synchronized (this.oldsources) {<a name="line.485"></a>
+<span class="sourceLineNo">486</span> for (Iterator<ReplicationSourceInterface> iter = oldsources.iterator(); iter.hasNext();) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.487"></a>
+<span class="sourceLineNo">488</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.488"></a>
+<span class="sourceLineNo">489</span> String queueId = oldSource.getQueueId();<a name="line.489"></a>
+<span class="sourceLineNo">490</span> oldSource.terminate(terminateMessage);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> oldSource.getSourceMetrics().clear();<a name="line.491"></a>
+<span class="sourceLineNo">492</span> queueStorage.removeQueue(server.getServerName(), queueId);<a name="line.492"></a>
+<span class="sourceLineNo">493</span> walsByIdRecoveredQueues.remove(queueId);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> iter.remove();<a name="line.494"></a>
+<span class="sourceLineNo">495</span> }<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span> }<a name="line.498"></a>
+<span class="sourceLineNo">499</span><a name="line.499"></a>
+<span class="sourceLineNo">500</span> /**<a name="line.500"></a>
+<span class="sourceLineNo">501</span> * Close the previous replication sources of this peer id and open new sources to trigger the new<a name="line.501"></a>
+<span class="sourceLineNo">502</span> * replication state changes or new replication config changes. Here we don't need to change<a name="line.502"></a>
+<span class="sourceLineNo">503</span> * replication queue storage and only to enqueue all logs to the new replication source<a name="line.503"></a>
+<span class="sourceLineNo">504</span> * @param peerId the id of the replication peer<a name="line.504"></a>
+<span class="sourceLineNo">505</span> */<a name="line.505"></a>
+<span class="sourceLineNo">506</span> public void refreshSources(String peerId) throws IOException {<a name="line.506"></a>
+<span class="sourceLineNo">507</span> String terminateMessage = "Peer " + peerId +<a name="line.507"></a>
+<span class="sourceLineNo">508</span> " state or config changed. Will close the previous replication source and open a new one";<a name="line.508"></a>
+<span class="sourceLineNo">509</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.509"></a>
+<span class="sourceLineNo">510</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.510"></a>
+<span class="sourceLineNo">511</span> // synchronized on latestPaths to avoid missing the new log<a name="line.511"></a>
+<span class="sourceLineNo">512</span> synchronized (this.latestPaths) {<a name="line.512"></a>
+<span class="sourceLineNo">513</span> ReplicationSourceInterface toRemove = this.sources.put(peerId, src);<a name="line.513"></a>
+<span class="sourceLineNo">514</span> if (toRemove != null) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.515"></a>
+<span class="sourceLineNo">516</span> // Do not clear metrics<a name="line.516"></a>
+<span class="sourceLineNo">517</span> toRemove.terminate(terminateMessage, null, false);<a name="line.517"></a>
+<span class="sourceLineNo">518</span> }<a name="line.518"></a>
+<span class="sourceLineNo">519</span> for (NavigableSet<String> walsByGroup : walsById.get(peerId).values()) {<a name="line.519"></a>
+<span class="sourceLineNo">520</span> walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));<a name="line.520"></a>
+<span class="sourceLineNo">521</span> }<a name="line.521"></a>
+<span class="sourceLineNo">522</span> }<a name="line.522"></a>
+<span class="sourceLineNo">523</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.523"></a>
+<span class="sourceLineNo">524</span> src.startup();<a name="line.524"></a>
+<span class="sourceLineNo">525</span><a name="line.525"></a>
+<span class="sourceLineNo">526</span> List<ReplicationSourceInterface> toStartup = new ArrayList<>();<a name="line.526"></a>
+<span class="sourceLineNo">527</span> // synchronized on oldsources to avoid race with NodeFailoverWorker<a name="line.527"></a>
+<span class="sourceLineNo">528</span> synchronized (this.oldsources) {<a name="line.528"></a>
+<span class="sourceLineNo">529</span> List<String> previousQueueIds = new ArrayList<>();<a name="line.529"></a>
+<span class="sourceLineNo">530</span> for (Iterator<ReplicationSourceInterface> iter = this.oldsources.iterator(); iter<a name="line.530"></a>
+<span class="sourceLineNo">531</span> .hasNext();) {<a name="line.531"></a>
+<span class="sourceLineNo">532</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.532"></a>
+<span class="sourceLineNo">533</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.533"></a>
+<span class="sourceLineNo">534</span> previousQueueIds.add(oldSource.getQueueId());<a name="line.534"></a>
+<span class="sourceLineNo">535</span> oldSource.terminate(terminateMessage);<a name="line.535"></a>
+<span class="sourceLineNo">536</span> iter.remove();<a name="line.536"></a>
+<span class="sourceLineNo">537</span> }<a name="line.537"></a>
+<span class="sourceLineNo">538</span> }<a name="line.538"></a>
+<span class="sourceLineNo">539</span> for (String queueId : previousQueueIds) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span> ReplicationSourceInterface recoveredReplicationSource = createSource(queueId, peer);<a name="line.540"></a>
+<span class="sourceLineNo">541</span> this.oldsources.add(recoveredReplicationSource);<a name="line.541"></a>
+<span class="sourceLineNo">542</span> for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span> toStartup.add(recoveredReplicationSource);<a name="line.545"></a>
+<span class="sourceLineNo">546</span> }<a name="line.546"></a>
+<span class="sourceLineNo">547</span> }<a name="line.547"></a>
+<span class="sourceLineNo">548</span> for (ReplicationSourceInterface replicationSource : toStartup) {<a name="line.548"></a>
+<span class="sourceLineNo">549</span> replicationSource.startup();<a name="line.549"></a>
+<span class="sourceLineNo">550</span> }<a name="line.550"></a>
+<span class="sourceLineNo">551</span> }<a name="line.551"></a>
+<span class="sourceLineNo">552</span><a name="line.552"></a>
+<span class="sourceLineNo">553</span> /**<a name="line.553"></a>
+<span class="sourceLineNo">554</span> * Clear the metrics and related replication queue of the specified old source<a name="line.554"></a>
+<span class="sourceLineNo">555</span> * @param src source to clear<a name="line.555"></a>
+<span class="sourceLineNo">556</span> */<a name="line.556"></a>
+<span class="sourceLineNo">557</span> private boolean removeRecoveredSource(ReplicationSourceInterface src) {<a name="line.557"></a>
+<span class="sourceLineNo">558</span> if (!this.oldsources.remove(src)) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span> return false;<a name="line.559"></a>
+<span class="sourceLineNo">560</span> }<a name="line.560"></a>
+<span class="sourceLineNo">561</span> LOG.info("Done with the recovered queue {}", src.getQueueId());<a name="line.561"></a>
+<span class="sourceLineNo">562</span> // Delete queue from storage and memory<a name="line.562"></a>
+<span class="sourceLineNo">563</span> deleteQueue(src.getQueueId());<a name="line.563"></a>
+<span class="sourceLineNo">564</span> this.walsByIdRecoveredQueues.remove(src.getQueueId());<a name="line.564"></a>
+<span class="sourceLineNo">565</span> return true;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> }<a name="line.566"></a>
+<span class="sourceLineNo">567</span><a name="line.567"></a>
+<span class="sourceLineNo">568</span> void finishRecoveredSource(ReplicationSourceInterface src) {<a name="line.568"></a>
+<span class="sourceLineNo">569</span> synchronized (oldsources) {<a name="line.569"></a>
+<span class="sourceLineNo">570</span> if (!removeRecoveredSource(src)) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span> return;<a name="line.571"></a>
+<span class="sourceLineNo">572</span> }<a name="line.572"></a>
+<span class="sourceLineNo">573</span> }<a name="line.573"></a>
+<span class="sourceLineNo">574</span> LOG.info("Finished recovering queue {} with the following stats: {}", src.getQueueId(),<a name="line.574"></a>
+<span class="sourceLineNo">575</span> src.getStats());<a name="line.575"></a>
+<span class="sourceLineNo">576</span> }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span> /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span> * Clear the metrics and related replication queue of the specified old source<a name="line.579"></a>
+<span class="sourceLineNo">580</span> * @param src source to clear<a name="line.580"></a>
+<span class="sourceLineNo">581</span> */<a name="line.581"></a>
+<span class="sourceLineNo">582</span> void removeSource(ReplicationSourceInterface src) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.info("Done with the queue " + src.getQueueId());<a name="line.583"></a>
+<span class="sourceLineNo">584</span> this.sources.remove(src.getPeerId());<a name="line.584"></a>
+<span class="sourceLineNo">585</span> // Delete queue from storage and memory<a name="line.585"></a>
+<span class="sourceLineNo">586</span> deleteQueue(src.getQueueId());<a name="line.586"></a>
+<span class="sourceLineNo">587</span> this.walsById.remove(src.getQueueId());<a name="line.587"></a>
+<span class="sourceLineNo">588</span><a name="line.588"></a>
+<span class="sourceLineNo">589</span> }<a name="line.589"></a>
+<span class="sourceLineNo">590</span><a name="line.590"></a>
+<span class="sourceLineNo">591</span> /**<a name="line.591"></a>
+<span class="sourceLineNo">592</span> * Delete a complete queue of wals associated with a replication source<a name="line.592"></a>
+<span class="sourceLineNo">593</span> * @param queueId the id of replication queue to delete<a name="line.593"></a>
+<span class="sourceLineNo">594</span> */<a name="line.594"></a>
+<span class="sourceLineNo">595</span> private void deleteQueue(String queueId) {<a name="line.595"></a>
+<span class="sourceLineNo">596</span> abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.596"></a>
+<span class="sourceLineNo">597</span> }<a name="line.597"></a>
+<span class="sourceLineNo">598</span><a name="line.598"></a>
+<span class="sourceLineNo">599</span> @FunctionalInterface<a name="line.599"></a>
+<span class="sourceLineNo">600</span> private interface ReplicationQueueOperation {<a name="line.600"></a>
+<span class="sourceLineNo">601</span> void exec() throws ReplicationException;<a name="line.601"></a>
+<span class="sourceLineNo">602</span> }<a name="line.602"></a>
+<span class="sourceLineNo">603</span><a name="line.603"></a>
+<span class="sourceLineNo">604</span> /**<a name="line.604"></a>
+<span class="sourceLineNo">605</span> * Refresh replication source will terminate the old source first, then the source thread will be<a name="line.605"></a>
+<span class="sourceLineNo">606</span> * interrupted. Need to handle it instead of abort the region server.<a name="line.606"></a>
+<span class="sourceLineNo">607</span> */<a name="line.607"></a>
+<span class="sourceLineNo">608</span> private void interruptOrAbortWhenFail(ReplicationQueueOperation op) {<a name="line.608"></a>
+<span class="sourceLineNo">609</span> try {<a name="line.609"></a>
+<span class="sourceLineNo">610</span> op.exec();<a name="line.610"></a>
+<span class="sourceLineNo">611</span> } catch (ReplicationException e) {<a name="line.611"></a>
+<span class="sourceLineNo">612</span> if (e.getCause() != null && e.getCause() instanceof KeeperException.SystemErrorException<a name="line.612"></a>
+<span class="sourceLineNo">613</span> && e.getCause().getCause() != null && e.getCause()<a name="line.613"></a>
+<span class="sourceLineNo">614</span> .getCause() instanceof InterruptedException) {<a name="line.614"></a>
+<span class="sourceLineNo">615</span> // ReplicationRuntimeException(a RuntimeException) is thrown out here. The reason is<a name="line.615"></a>
+<span class="sourceLineNo">616</span> // that thread is interrupted deep down in the stack, it should pass the following<a name="line.616"></a>
+<span class="sourceLineNo">617</span> // processing logic and propagate to the most top layer which can handle this exception<a name="line.617"></a>
+<span class="sourceLineNo">618</span> // properly. In this specific case, the top layer is ReplicationSourceShipper#run().<a name="line.618"></a>
+<span class="sourceLineNo">619</span> throw new ReplicationRuntimeException(<a name="line.619"></a>
+<span class="sourceLineNo">620</span> "Thread is interrupted, the replication source may be terminated",<a name="line.620"></a>
+<span class="sourceLineNo">621</span> e.getCause().getCause());<a name="line.621"></a>
+<span class="sourceLineNo">622</span> }<a name="line.622"></a>
+<span class="sourceLineNo">623</span> server.abort("Failed to operate on replication queue", e);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span> }<a name="line.625"></a>
+<span class="sourceLineNo">626</span><a name="line.626"></a>
+<span class="sourceLineNo">627</span> private void abortWhenFail(ReplicationQueueOperation op) {<a name="line.627"></a>
+<span class="sourceLineNo">628</span> try {<a name="line.628"></a>
+<span class="sourceLineNo">629</span> op.exec();<a name="line.629"></a>
+<span class="sourceLineNo">630</span> } catch (ReplicationException e) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> server.abort("Failed to operate on replication queue", e);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> }<a name="line.632"></a>
+<span class="sourceLineNo">633</span> }<a name="line.633"></a>
+<span class="sourceLineNo">634</span><a name="line.634"></a>
+<span class="sourceLineNo">635</span> private void throwIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.635"></a>
+<span class="sourceLineNo">636</span> try {<a name="line.636"></a>
+<span class="sourceLineNo">637</span> op.exec();<a name="line.637"></a>
+<span class="sourceLineNo">638</span> } catch (ReplicationException e) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> throw new IOException(e);<a name="line.639"></a>
+<span class="sourceLineNo">640</span> }<a name="line.640"></a>
+<span class="sourceLineNo">641</span> }<a name="line.641"></a>
+<span class="sourceLineNo">642</span><a name="line.642"></a>
+<span class="sourceLineNo">643</span> private void abortAndThrowIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.643"></a>
+<span class="sourceLineNo">644</span> try {<a name="line.644"></a>
+<span class="sourceLineNo">645</span> op.exec();<a name="line.645"></a>
+<span class="sourceLineNo">646</span> } catch (ReplicationException e) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span> server.abort("Failed to operate on replication queue", e);<a name="line.647"></a>
+<span class="sourceLineNo">648</span> throw new IOException(e);<a name="line.648"></a>
+<span class="sourceLineNo">649</span> }<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * This method will log the current position to storage. And also clean old logs from the<a name="line.653"></a>
+<span class="sourceLineNo">654</span> * replication queue.<a name="line.654"></a>
+<span class="sourceLineNo">655</span> * @param source the replication source<a name="line.655"></a>
+<span class="sourceLineNo">656</span> * @param entryBatch the wal entry batch we just shipped<a name="line.656"></a>
+<span class="sourceLineNo">657</span> */<a name="line.657"></a>
+<span class="sourceLineNo">658</span> public void logPositionAndCleanOldLogs(ReplicationSourceInterface source,<a name="line.658"></a>
+<span class="sourceLineNo">659</span> WALEntryBatch entryBatch) {<a name="line.659"></a>
+<span class="sourceLineNo">660</span> String fileName = entryBatch.getLastWalPath().getName();<a name="line.660"></a>
+<span class="sourceLineNo">661</span> interruptOrAbortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(),<a name="line.661"></a>
+<span class="sourceLineNo">662</span> source.getQueueId(), fileName, entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds()));<a name="line.662"></a>
+<span class="sourceLineNo">663</span> cleanOldLogs(fileName, entryBatch.isEndOfFile(), source);<a name="line.663"></a>
+<span class="sourceLineNo">664</span> }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span> /**<a name="line.666"></a>
+<span class="sourceLineNo">667</span> * Cleans a log file and all older logs from replication queue. Called when we are sure that a log<a name="line.667"></a>
+<span class="sourceLineNo">668</span> * file is closed and has no more entries.<a name="line.668"></a>
+<span class="sourceLineNo">669</span> * @param log Path to the log<a name="line.669"></a>
+<span class="sourceLineNo">670</span> * @param inclusive whether we should also remove the given log file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> * @param source the replication source<a name="line.671"></a>
+<span class="sourceLineNo">672</span> */<a name="line.672"></a>
+<span class="sourceLineNo">673</span> void cleanOldLogs(String log, boolean inclusive, ReplicationSourceInterface source) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(log);<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (source.isRecovered()) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> NavigableSet<String> wals = walsByIdRecoveredQueues.get(source.getQueueId()).get(logPrefix);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> if (wals != null) {<a name="line.677"></a>
+<span class="sourceLineNo">678</span> NavigableSet<String> walsToRemove = wals.headSet(log, inclusive);<a name="line.678"></a>
+<span class="sourceLineNo">679</span> if (walsToRemove.isEmpty()) {<a name="line.679"></a>
+<span class="sourceLineNo">680</span> return;<a name="line.680"></a>
+<span class="sourceLineNo">681</span> }<a name="line.681"></a>
+<span class="sourceLineNo">682</span> cleanOldLogs(walsToRemove, source);<a name="line.682"></a>
+<span class="sourceLineNo">683</span> walsToRemove.clear();<a name="line.683"></a>
+<span class="sourceLineNo">684</span> }<a name="line.684"></a>
+<span class="sourceLineNo">685</span> } else {<a name="line.685"></a>
+<span class="sourceLineNo">686</span> NavigableSet<String> wals;<a name="line.686"></a>
+<span class="sourceLineNo">687</span> NavigableSet<String> walsToRemove;<a name="line.687"></a>
+<span class="sourceLineNo">688</span> // synchronized on walsById to avoid race with preLogRoll<a name="line.688"></a>
+<span class="sourceLineNo">689</span> synchronized (this.walsById) {<a name="line.689"></a>
+<span class="sourceLineNo">690</span> wals = walsById.get(source.getQueueId()).get(logPrefix);<a name="line.690"></a>
+<span class="sourceLineNo">691</span> if (wals == null) {<a name="line.691"></a>
+<span class="sourceLineNo">692</span> return;<a name="line.692"></a>
+<span class="sourceLineNo">693</span> }<a name="line.693"></a>
+<span class="sourceLineNo">694</span> walsToRemove = wals.headSet(log, inclusive);<a name="line.694"></a>
+<span class="sourceLineNo">695</span> if (walsToRemove.isEmpty()) {<a name="line.695"></a>
+<span class="sourceLineNo">696</span> return;<a name="line.696"></a>
+<span class="sourceLineNo">697</span> }<a name="line.697"></a>
+<span class="sourceLineNo">698</span> walsToRemove = new TreeSet<>(walsToRemove);<a name="line.698"></a>
+<span class="sourceLineNo">699</span> }<a name="line.699"></a>
+<span class="sourceLineNo">700</span> // cleanOldLogs may spend some time, especially for sync replication where we may want to<a name="line.700"></a>
+<span class="sourceLineNo">701</span> // remove remote wals as the remote cluster may have already been down, so we do it outside<a name="line.701"></a>
+<span class="sourceLineNo">702</span> // the lock to avoid block preLogRoll<a name="line.702"></a>
+<span class="sourceLineNo">703</span> cleanOldLogs(walsToRemove, source);<a name="line.703"></a>
+<span class="sourceLineNo">704</span> // now let's remove the files in the set<a name="line.704"></a>
+<span class="sourceLineNo">705</span> synchronized (this.walsById) {<a name="line.705"></a>
+<span class="sourceLineNo">706</span> wals.removeAll(walsToRemove);<a name="line.706"></a>
+<span class="sourceLineNo">707</span> }<a name="line.707"></a>
+<span class="sourceLineNo">708</span> }<a name="line.708"></a>
+<span class="sourceLineNo">709</span> }<a name="line.709"></a>
+<span class="sourceLineNo">710</span><a name="line.710"></a>
+<span class="sourceLineNo">711</span> private void removeRemoteWALs(String peerId, String remoteWALDir, Collection<String> wals)<a name="line.711"></a>
+<span class="sourceLineNo">712</span> throws IOException {<a name="line.712"></a>
+<span class="sourceLineNo">713</span> Path remoteWALDirForPeer = ReplicationUtils.getPeerRemoteWALDir(remoteWALDir, peerId);<a name="line.713"></a>
+<span class="sourceLineNo">714</span> FileSystem fs = ReplicationUtils.getRemoteWALFileSystem(conf, remoteWALDir);<a name="line.714"></a>
+<span class="sourceLineNo">715</span> for (String wal : wals) {<a name="line.715"></a>
+<span class="sourceLineNo">716</span> Path walFile = new Path(remoteWALDirForPeer, wal);<a name="line.716"></a>
+<span class="sourceLineNo">717</span> try {<a name="line.717"></a>
+<span class="sourceLineNo">718</span> if (!fs.delete(walFile, false) && fs.exists(walFile)) {<a name="line.718"></a>
+<span class="sourceLineNo">719</span> throw new IOException("Can not delete " + walFile);<a name="line.719"></a>
+<span class="sourceLineNo">720</span> }<a name="line.720"></a>
+<span class="sourceLineNo">721</span> } catch (FileNotFoundException e) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span> // Just ignore since this means the file has already been deleted.<a name="line.722"></a>
+<span class="sourceLineNo">723</span> // The javadoc of the FileSystem.delete methods does not specify the behavior of deleting an<a name="line.723"></a>
+<span class="sourceLineNo">724</span> // inexistent file, so here we deal with both, i.e, check the return value of the<a name="line.724"></a>
+<span class="sourceLineNo">725</span> // FileSystem.delete, and also catch FNFE.<a name="line.725"></a>
+<span class="sourceLineNo">726</span> LOG.debug("The remote wal {} has already been deleted?", walFile, e);<a name="line.726"></a>
+<span class="sourceLineNo">727</span> }<a name="line.727"></a>
+<span class="sourceLineNo">728</span> }<a name="line.728"></a>
+<span class="sourceLineNo">729</span> }<a name="line.729"></a>
+<span class="sourceLineNo">730</span><a name="line.730"></a>
+<span class="sourceLineNo">731</span> private void cleanOldLogs(NavigableSet<String> wals, ReplicationSourceInterface source) {<a name="line.731"></a>
+<span class="sourceLineNo">732</span> LOG.debug("Removing {} logs in the list: {}", wals.size(), wals);<a name="line.732"></a>
+<span class="sourceLineNo">733</span> // The intention here is that, we want to delete the remote wal files ASAP as it may effect the<a name="line.733"></a>
+<span class="sourceLineNo">734</span> // failover time if you want to transit the remote cluster from S to A. And the infinite retry<a name="line.734"></a>
+<span class="sourceLineNo">735</span> // is not a problem, as if we can not contact with the remote HDFS cluster, then usually we can<a name="line.735"></a>
+<span class="sourceLineNo">736</span> // not contact with the HBase cluster either, so the replication will be blocked either.<a name="line.736"></a>
+<span class="sourceLineNo">737</span> if (source.isSyncReplication()) {<a name="line.737"></a>
+<span class="sourceLineNo">738</span> String peerId = source.getPeerId();<a name="line.738"></a>
+<span class="sourceLineNo">739</span> String remoteWALDir = source.getPeer().getPeerConfig().getRemoteWALDir();<a name="line.739"></a>
+<span class="sourceLineNo">740</span> // Filter out the wals need to be removed from the remote directory. Its name should be the<a name="line.740"></a>
+<span class="sourceLineNo">741</span> // special format, and also, the peer id in its name should match the peer id for the<a name="line.741"></a>
+<span class="sourceLineNo">742</span> // replication source.<a name="line.742"></a>
+<span class="sourceLineNo">743</span> List<String> remoteWals = wals.stream().filter(w -> SyncReplicationWALProvider<a name="line.743"></a>
+<span class="sourceLineNo">744</span> .getSyncReplicationPeerIdFromWALName(w).map(peerId::equals).orElse(false))<a name="line.744"></a>
+<span class="sourceLineNo">745</span> .collect(Collectors.toList());<a name="line.745"></a>
+<span class="sourceLineNo">746</span> LOG.debug("Removing {} logs from remote dir {} in the list: {}", remoteWals.size(),<a name="line.746"></a>
+<span class="sourceLineNo">747</span> remoteWALDir, remoteWals);<a name="line.747"></a>
+<span class="sourceLineNo">748</span> if (!remoteWals.isEmpty()) {<a name="line.748"></a>
+<span class="sourceLineNo">749</span> for (int sleepMultiplier = 0;;) {<a name="line.749"></a>
+<span class="sourceLineNo">750</span> try {<a name="line.750"></a>
+<span class="sourceLineNo">751</span> removeRemoteWALs(peerId, remoteWALDir, remoteWals);<a name="line.751"></a>
+<span class="sourceLineNo">752</span> break;<a name="line.752"></a>
+<span class="sourceLineNo">753</span> } catch (IOException e) {<a name="line.753"></a>
+<span class="sourceLineNo">754</span> LOG.warn("Failed to delete remote wals from remote dir {} for peer {}", remoteWALDir,<a name="line.754"></a>
+<span class="sourceLineNo">755</span> peerId);<a name="line.755"></a>
+<span class="sourceLineNo">756</span> }<a name="line.756"></a>
+<span class="sourceLineNo">757</span> if (!source.isSourceActive()) {<a name="line.757"></a>
+<span class="sourceLineNo">758</span> // skip the following operations<a name="line.758"></a>
+<span class="sourceLineNo">759</span> return;<a name="line.759"></a>
+<span class="sourceLineNo">760</span> }<a name="line.760"></a>
+<span class="sourceLineNo">761</span> if (ReplicationUtils.sleepForRetries("Failed to delete remote wals", sleepForRetries,<a name="line.761"></a>
+<span class="sourceLineNo">762</span> sleepMultiplier, maxRetriesMultiplier)) {<a name="line.762"></a>
+<span class="sourceLineNo">763</span> sleepMultiplier++;<a name="line.763"></a>
+<span class="sourceLineNo">764</span> }<a name="line.764"></a>
+<span class="sourceLineNo">765</span> }<a name="line.765"></a>
+<span class="sourceLineNo">766</span> }<a name="line.766"></a>
+<span class="sourceLineNo">767</span> }<a name="line.767"></a>
+<span class="sourceLineNo">768</span> String queueId = source.getQueueId();<a name="line.768"></a>
+<span class="sourceLineNo">769</span> for (String wal : wals) {<a name="line.769"></a>
+<span class="sourceLineNo">770</span> interruptOrAbortWhenFail(<a name="line.770"></a>
+<span class="sourceLineNo">771</span> () -> this.queueStorage.removeWAL(server.getServerName(), queueId, wal));<a name="line.771"></a>
+<span class="sourceLineNo">772</span> }<a name="line.772"></a>
+<span class="sourceLineNo">773</span> }<a name="line.773"></a>
+<span class="sourceLineNo">774</span><a name="line.774"></a>
+<span class="sourceLineNo">775</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.775"></a>
+<span class="sourceLineNo">776</span> public void preLogRoll(Path newLog) throws IOException {<a name="line.776"></a>
+<span class="sourceLineNo">777</span> String logName = newLog.getName();<a name="line.777"></a>
+<span class="sourceLineNo">778</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);<a name="line.778"></a>
+<span class="sourceLineNo">779</span> // synchronized on latestPaths to avoid the new open source miss the new log<a name="line.779"></a>
+<span class="sourceLineNo">780</span> synchronized (this.latestPaths) {<a name="line.780"></a>
+<span class="sourceLineNo">781</span> // Add log to queue storage<a name="line.781"></a>
+<span class="sourceLineNo">782</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.782"></a>
+<span class="sourceLineNo">783</span> // If record log to queue storage failed, abort RS and throw exception to make log roll<a name="line.783"></a>
+<span class="sourceLineNo">784</span> // failed<a name="line.784"></a>
+<span class="sourceLineNo">785</span> abortAndThrowIOExceptionWhenFail(<a name="line.785"></a>
+<span class="sourceLineNo">786</span> () -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));<a name="line.786"></a>
+<span class="sourceLineNo">787</span> }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span> // synchronized on walsById to avoid race with cleanOldLogs<a name="line.789"></a>
+<span class="sourceLineNo">790</span> synchronized (this.walsById) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span> // Update walsById map<a name="line.791"></a>
+<span class="sourceLineNo">792</span> for (Map.Entry<String, Map<String, NavigableSet<String>>> entry : this.walsById<a name="line.792"></a>
+<span class="sourceLineNo">793</span> .entrySet()) {<a name="line.793"></a>
+<span class="sourceLineNo">794</span> String peerId = entry.getKey();<a name="line.794"></a>
+<span class="sourceLineNo">795</span> Map<String, NavigableSet<String>> walsByPrefix = entry.getValue();<a name="line.795"></a>
+<span class="sourceLineNo">796</span> boolean existingPrefix = false;<a name="line.796"></a>
+<span class="sourceLineNo">797</span> for (Map.Entry<String, NavigableSet<String>> walsEntry : walsByPrefix.entrySet()) {<a name="line.797"></a>
+<span class="sourceLineNo">798</span> SortedSet<String> wals = walsEntry.getValue();<a name="line.798"></a>
+<span class="sourceLineNo">799</span> if (this.sources.isEmpty()) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span> // If there's no slaves, don't need to keep the old wals since<a name="line.800"></a>
+<span class="sourceLineNo">801</span> // we only consider the last one when a new slave comes in<a name="line.801"></a>
+<span class="sourceLineNo">802</span> wals.clear();<a name="line.802"></a>
+<span class="sourceLineNo">803</span> }<a name="line.803"></a>
+<span class="sourceLineNo">804</span> if (logPrefix.equals(walsEntry.getKey())) {<a name="line.804"></a>
+<span class="sourceLineNo">805</span> wals.add(logName);<a name="line.805"></a>
+<span class="sourceLineNo">806</span> existingPrefix = true;<a name="line.806"></a>
+<span class="sourceLineNo">807</span> }<a name="line.807"></a>
+<span class="sourceLineNo">808</span> }<a name="line.808"></a>
+<span class="sourceLineNo">809</span> if (!existingPrefix) {<a name="line.809"></a>
+<span class="sourceLineNo">810</span> // The new log belongs to a new group, add it into this peer<a name="line.810"></a>
+<span class="sourceLineNo">811</span> LOG.debug("Start tracking logs for wal group {} for peer {}", logPrefix, peerId);<a name="line.811"></a>
+<span class="sourceLineNo">812</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.812"></a>
+<span class="sourceLineNo">813</span> wals.add(logName);<a name="line.813"></a>
+<span class="sourceLineNo">814</span> walsByPrefix.put(logPrefix, wals);<a name="line.814"></a>
+<span class="sourceLineNo">815</span> }<a name="line.815"></a>
+<span class="sourceLineNo">816</span> }<a name="line.816"></a>
+<span class="sourceLineNo">817</span> }<a name="line.817"></a>
+<span class="sourceLineNo">818</span><a name="line.818"></a>
+<span class="sourceLineNo">819</span> // Add to latestPaths<a name="line.819"></a>
+<span class="sourceLineNo">820</span> latestPaths.put(logPrefix, newLog);<a name="line.820"></a>
+<span class="sourceLineNo">821</span> }<a name="line.821"></a>
+<span class="sourceLineNo">822</span> }<a name="line.822"></a>
+<span class="sourceLineNo">823</span><a name="line.823"></a>
+<span class="sourceLineNo">824</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.824"></a>
+<span class="sourceLineNo">825</span> public void postLogRoll(Path newLog) throws IOException {<a name="line.825"></a>
+<span class="sourceLineNo">826</span> // This only updates the sources we own, not the recovered ones<a name="line.826"></a>
+<span class="sourceLineNo">827</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.827"></a>
+<span class="sourceLineNo">828</span> source.enqueueLog(newLog);<a name="line.828"></a>
+<span class="sourceLineNo">829</span> LOG.trace("Enqueued {} to source {} while performing postLogRoll operation.",<a name="line.829"></a>
+<span class="sourceLineNo">830</span> newLog, source.getQueueId());<a name="line.830"></a>
+<span class="sourceLineNo">831</span> }<a name="line.831"></a>
+<span class="sourceLineNo">832</span> }<a name="line.832"></a>
+<span class="sourceLineNo">833</span><a name="line.833"></a>
+<span class="sourceLineNo">834</span> @Override<a name="line.834"></a>
+<span class="sourceLineNo">835</span> public void regionServerRemoved(String regionserver) {<a name="line.835"></a>
+<span class="sourceLineNo">836</span> transferQueues(ServerName.valueOf(regionserver));<a name="line.836"></a>
+<span class="sourceLineNo">837</span> }<a name="line.837"></a>
+<span class="sourceLineNo">838</span><a name="line.838"></a>
+<span class="sourceLineNo">839</span> /**<a name="line.839"></a>
+<span class="sourceLineNo">840</span> * Transfer all the queues of the specified to this region server. First it tries to grab a lock<a name="line.840"></a>
+<span class="sourceLineNo">841</span> * and if it works it will move the old queues and finally will delete the old queues.<a name="line.841"></a>
+<span class="sourceLineNo">842</span> * <p><a name="line.842"></a>
+<span class="sourceLineNo">843</span> * It creates one old source for any type of source of the old rs.<a name="line.843"></a>
+<span class="sourceLineNo">844</span> */<a name="line.844"></a>
+<span class="sourceLineNo">845</span> private void transferQueues(ServerName deadRS) {<a name="line.845"></a>
+<span class="sourceLineNo">846</span> if (server.getServerName().equals(deadRS)) {<a name="line.846"></a>
+<span class="sourceLineNo">847</span> // it's just us, give up<a name="line.847"></a>
+<span class="sourceLineNo">848</span> return;<a name="line.848"></a>
+<span class="sourceLineNo">849</span> }<a name="line.849"></a>
+<span class="sourceLineNo">850</span> NodeFailoverWorker transfer = new NodeFailoverWorker(deadRS);<a name="line.850"></a>
+<span class="sourceLineNo">851</span> try {<a name="line.851"></a>
+<span class="sourceLineNo">852</span> this.executor.execute(transfer);<a name="line.852"></a>
+<span class="sourceLineNo">853</span> } catch (RejectedExecutionException ex) {<a name="line.853"></a>
+<span class="sourceLineNo">854</span> CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)<a name="line.854"></a>
+<span class="sourceLineNo">855</span> .getGlobalSource().incrFailedRecoveryQueue();<a name="line.855"></a>
+<span class="sourceLineNo">856</span> LOG.info("Cancelling the transfer of " + deadRS + " because of " + ex.getMessage());<a name="line.856"></a>
+<span class="sourceLineNo">857</span> }<a name="line.857"></a>
+<span class="sourceLineNo">858</span> }<a name="line.858"></a>
+<span class="sourceLineNo">859</span><a name="line.859"></a>
+<span class="sourceLineNo">860</span> /**<a name="line.860"></a>
+<span class="sourceLineNo">861</span> * Class responsible to setup new ReplicationSources to take care of the queues from dead region<a name="line.861"></a>
+<span class="sourceLineNo">862</span> * servers.<a name="line.862"></a>
+<span class="sourceLineNo">863</span> */<a name="line.863"></a>
+<span class="sourceLineNo">864</span> class NodeFailoverWorker extends Thread {<a name="line.864"></a>
+<span class="sourceLineNo">865</span><a name="line.865"></a>
+<span class="sourceLineNo">866</span> private final ServerName deadRS;<a name="line.866"></a>
+<span class="sourceLineNo">867</span> // After claim the queues from dead region server, the NodeFailoverWorker will skip to start<a name="line.867"></a>
+<span class="sourceLineNo">868</span> // the RecoveredReplicationSource if the peer has been removed. but there's possible that<a name="line.868"></a>
+<span class="sourceLineNo">869</span> // remove a peer with peerId = 2 and add a peer with peerId = 2 again during the<a name="line.869"></a>
+<span class="sourceLineNo">870</span> // NodeFailoverWorker. So we need a deep copied <peerId, peer> map to decide whether we<a name="line.870"></a>
+<span class="sourceLineNo">871</span> // should start the RecoveredReplicationSource. If the latest peer is not the old peer when<a name="line.871"></a>
+<span class="sourceLineNo">872</span> // NodeFailoverWorker begin, we should skip to start the RecoveredReplicationSource, Otherwise<a name="line.872"></a>
+<span class="sourceLineNo">873</span> // the rs will abort (See HBASE-20475).<a name="line.873"></a>
+<span class="sourceLineNo">874</span> private final Map<String, ReplicationPeerImpl> peersSnapshot;<a name="line.874"></a>
+<span class="sourceLineNo">875</span><a name="line.875"></a>
+<span class="sourceLineNo">876</span> public NodeFailoverWorker(ServerName deadRS) {<a name="line.876"></a>
+<span class="sourceLineNo">877</span> super("Failover-for-" + deadRS);<a name="line.877"></a>
+<span class="sourceLineNo">878</span> this.deadRS = deadRS;<a name="line.878"></a>
+<span class="sourceLineNo">879</span> peersSnapshot = new HashMap<>(replicationPeers.getPeerCache());<a name="line.879"></a>
+<span class="sourceLineNo">880</span> }<a name="line.880"></a>
+<span class="sourceLineNo">881</span><a name="line.881"></a>
+<span class="sourceLineNo">882</span> private boolean isOldPeer(String peerId, ReplicationPeerImpl newPeerRef) {<a name="line.882"></a>
+<span class="sourceLineNo">883</span> ReplicationPeerImpl oldPeerRef = peersSnapshot.get(peerId);<a name="line.883"></a>
+<span class="sourceLineNo">884</span> return oldPeerRef != null && oldPeerRef == newPeerRef;<a name="line.884"></a>
+<span class="sourceLineNo">885</span> }<a name="line.885"></a>
+<span class="sourceLineNo">886</span><a name="line.886"></a>
+<span class="sourceLineNo">887</span> @Override<a name="line.887"></a>
+<span class="sourceLineNo">888</span> public void run() {<a name="line.888"></a>
+<span class="sourceLineNo">889</span> // Wait a bit before transferring the queues, we may be shutting down.<a name="line.889"></a>
+<span class="sourceLineNo">890</span> // This sleep may not be enough in some cases.<a name="line.890"></a>
+<span class="sourceLineNo">891</span> try {<a name="line.891"></a>
+<span class="sourceLineNo">892</span> Thread.sleep(sleepBeforeFailover +<a name="line.892"></a>
+<span class="sourceLineNo">893</span> (long) (ThreadLocalRandom.current().nextFloat() * sleepBeforeFailover));<a name="line.893"></a>
+<span class="sourceLineNo">894</span> } catch (InterruptedException e) {<a name="line.894"></a>
+<span class="sourceLineNo">895</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.895"></a>
+<span class="sourceLineNo">896</span> Thread.currentThread().interrupt();<a name="line.896"></a>
+<span class="sourceLineNo">897</span> }<a name="line.897"></a>
+<span class="sourceLineNo">898</span> // We try to lock that rs' queue directory<a name="line.898"></a>
+<span class="sourceLineNo">899</span> if (server.isStopped()) {<a name="line.899"></a>
+<span class="sourceLineNo">900</span> LOG.info("Not transferring queue since we are shutting down");<a name="line.900"></a>
+<span class="sourceLineNo">901</span> return;<a name="line.901"></a>
+<span class="sourceLineNo">902</span> }<a name="line.902"></a>
+<span class="sourceLineNo">903</span> Map<String, Set<String>> newQueues = new HashMap<>();<a name="line.903"></a>
+<span class="sourceLineNo">904</span> try {<a name="line.904"></a>
+<span class="sourceLineNo">905</span> List<String> queues = queueStorage.getAllQueues(deadRS);<a name="line.905"></a>
+<span class="sourceLineNo">906</span> while (!queues.isEmpty()) {<a name="line.906"></a>
+<span class="sourceLineNo">907</span> Pair<String, SortedSet<String>> peer = queueStorage.claimQueue(deadRS,<a name="line.907"></a>
+<span class="sourceLineNo">908</span> queues.get(ThreadLocalRandom.current().nextInt(queues.size())), server.getServerName());<a name="line.908"></a>
+<span class="sourceLineNo">909</span> long sleep = sleepBeforeFailover / 2;<a name="line.909"></a>
+<span class="sourceLineNo">910</span> if (!peer.getSecond().isEmpty()) {<a name="line.910"></a>
+<span class="sourceLineNo">911</span> newQueues.put(peer.getFirst(), peer.getSecond());<a name="line.911"></a>
+<span class="sourceLineNo">912</span> sleep = sleepBeforeFailover;<a name="line.912"></a>
+<span class="sourceLineNo">913</span> }<a name="line.913"></a>
+<span class="sourceLineNo">914</span> try {<a name="line.914"></a>
+<span class="sourceLineNo">915</span> Thread.sleep(sleep);<a name="line.915"></a>
+<span class="sourceLineNo">916</span> } catch (InterruptedException e) {<a name="line.916"></a>
+<span class="sourceLineNo">917</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.917"></a>
+<span class="sourceLineNo">918</span> Thread.currentThread().interrupt();<a name="line.918"></a>
+<span class="sourceLineNo">919</span> }<a name="line.919"></a>
+<span class="sourceLineNo">920</span> queues = queueStorage.getAllQueues(deadRS);<a name="line.920"></a>
+<span class="sourceLineNo">921</span> }<a name="line.921"></a>
+<span class="sourceLineNo">922</span> if (queues.isEmpty()) {<a name="line.922"></a>
+<span class="sourceLineNo">923</span> queueStorage.removeReplicatorIfQueueIsEmpty(deadRS);<a name="line.923"></a>
+<span class="sourceLineNo">924</span> }<a name="line.924"></a>
+<span class="sourceLineNo">925</span> } catch (ReplicationException e) {<a name="line.925"></a>
+<span class="sourceLineNo">926</span> LOG.error(String.format("ReplicationException: cannot claim dead region (%s)'s " +<a name="line.926"></a>
+<span class="sourceLineNo">927</span> "replication queue. Znode : (%s)" +<a name="line.927"></a>
+<span class="sourceLineNo">928</span> " Possible solution: check if znode size exceeds jute.maxBuffer value. " +<a name="line.928"></a>
+<span class="sourceLineNo">929</span> " If so, increase it for both client and server side." + e), deadRS,<a name="line.929"></a>
+<span class="sourceLineNo">930</span> queueStorage.getRsNode(deadRS));<a name="line.930"></a>
+<span class="sourceLineNo">931</span> server.abort("Failed to claim queue from dead regionserver.", e);<a name="line.931"></a>
+<span class="sourceLineNo">932</span> return;<a name="line.932"></a>
+<span class="sourceLineNo">933</span> }<a name="line.933"></a>
+<span class="sourceLineNo">934</span> // Copying over the failed queue is completed.<a name="line.934"></a>
+<span class="sourceLineNo">935</span> if (newQueues.isEmpty()) {<a name="line.935"></a>
+<span class="sourceLineNo">936</span> // We either didn't get the lock or the failed region server didn't have any outstanding<a name="line.936"></a>
+<span class="sourceLineNo">937</span> // WALs to replicate, so we are done.<a name="line.937"></a>
+<span class="sourceLineNo">938</span> return;<a name="line.938"></a>
+<span class="sourceLineNo">939</span> }<a name="line.939"></a>
+<span class="sourceLineNo">940</span><a name="line.940"></a>
+<span class="sourceLineNo">941</span> for (Map.Entry<String, Set<String>> entry : newQueues.entrySet()) {<a name="line.941"></a>
+<span class="sourceLineNo">942</span> String queueId = entry.getKey();<a name="line.942"></a>
+<span class="sourceLineNo">943</span> Set<String> walsSet = entry.getValue();<a name="line.943"></a>
+<span class="sourceLineNo">944</span> try {<a name="line.944"></a>
+<span class="sourceLineNo">945</span> // there is not an actual peer defined corresponding to peerId for the failover.<a name="line.945"></a>
+<span class="sourceLineNo">946</span> ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);<a name="line.946"></a>
+<span class="sourceLineNo">947</span> String actualPeerId = replicationQueueInfo.getPeerId();<a name="line.947"></a>
+<span class="sourceLineNo">948</span><a name="line.948"></a>
+<span class="sourceLineNo">949</span> ReplicationPeerImpl peer = replicationPeers.getPeer(actualPeerId);<a name="line.949"></a>
+<span class="sourceLineNo">950</span> if (peer == null || !isOldPeer(actualPeerId, peer)) {<a name="line.950"></a>
+<span class="sourceLineNo">951</span> LOG.warn("Skipping failover for peer {} of node {}, peer is null", actualPeerId,<a name="line.951"></a>
+<span class="sourceLineNo">952</span> deadRS);<a name="line.952"></a>
+<span class="sourceLineNo">953</span> abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.953"></a>
+<span class="sourceLineNo">954</span> continue;<a name="line.954"></a>
+<span class="sourceLineNo">955</span> }<a name="line.955"></a>
+<span class="sourceLineNo">956</span> if (server instanceof ReplicationSyncUp.DummyServer<a name="line.956"></a>
+<span class="sourceLineNo">957</span> && peer.getPeerState().equals(PeerState.DISABLED)) {<a name="line.957"></a>
+<span class="sourceLineNo">958</span> LOG.warn("Peer {} is disabled. ReplicationSyncUp tool will skip "<a name="line.958"></a>
+<span class="sourceLineNo">959</span> + "replicating data to this peer.",<a name="line.959"></a>
+<span class="sourceLineNo">960</span> actualPeerId);<a name="line.960"></a>
+<span class="sourceLineNo">961</span> continue;<a name="line.961"></a>
+<span class="sourceLineNo">962</span> }<a name="line.962"></a>
+<span class="sourceLineNo">963</span><a name="line.963"></a>
+<span class="sourceLineNo">964</span> ReplicationSourceInterface src = createSource(queueId, peer);<a name="line.964"></a>
+<span class="sourceLineNo">965</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.965"></a>
+<span class="sourceLineNo">966</span> synchronized (oldsources) {<a name="line.966"></a>
+<span class="sourceLineNo">967</span> peer = replicationPeers.getPeer(src.getPeerId());<a name="line.967"></a>
+<span class="sourceLineNo">968</span> if (peer == null || !isOldPeer(src.getPeerId(), peer)) {<a name="line.968"></a>
+<span class="sourceLineNo">969</span> src.terminate("Recovered queue doesn't belong to any current peer");<a name="line.969"></a>
+<span class="sourceLineNo">970</span> deleteQueue(queueId);<a name="line.970"></a>
+<span class="sourceLineNo">971</span> continue;<a name="line.971"></a>
+<span class="sourceLineNo">972</span> }<a name="line.972"></a>
+<span class="sourceLineNo">973</span> // Do not setup recovered queue if a sync replication peer is in STANDBY state, or is<a name="line.973"></a>
+<span class="sourceLineNo">974</span> // transiting to STANDBY state. The only exception is we are in STANDBY state and<a name="line.974"></a>
+<span class="sourceLineNo">975</span> // transiting to DA, under this state we will replay the remote WAL and they need to be<a name="line.975"></a>
+<span class="sourceLineNo">976</span> // replicated back.<a name="line.976"></a>
+<span class="sourceLineNo">977</span> if (peer.getPeerConfig().isSyncReplication()) {<a name="line.977"></a>
+<span class="sourceLineNo">978</span> Pair<SyncReplicationState, SyncReplicationState> stateAndNewState =<a name="line.978"></a>
+<span class="sourceLineNo">979</span> peer.getSyncReplicationStateAndNewState();<a name="line.979"></a>
+<span class="sourceLineNo">980</span> if ((stateAndNewState.getFirst().equals(SyncReplicationState.STANDBY) &&<a name="line.980"></a>
+<span class="sourceLineNo">981</span> stateAndNewState.getSecond().equals(SyncReplicationState.NONE)) ||<a name="line.981"></a>
+<span class="sourceLineNo">982</span> stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)) {<a name="line.982"></a>
+<span class="sourceLineNo">983</span> src.terminate("Sync replication peer is in STANDBY state");<a name="line.983"></a>
+<span class="sourceLineNo">984</span> deleteQueue(queueId);<a name="line.984"></a>
+<span class="sourceLineNo">985</span> continue;<a name="line.985"></a>
+<span class="sourceLineNo">986</span> }<a name="line.986"></a>
+<span class="sourceLineNo">987</span> }<a name="line.987"></a>
+<span class="sourceLineNo">988</span> // track sources in walsByIdRecoveredQueues<a name="line.988"></a>
+<span class="sourceLineNo">989</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.989"></a>
+<span class="sourceLineNo">990</span> walsByIdRecoveredQueues.put(queueId, walsByGroup);<a name="line.990"></a>
+<span class="sourceLineNo">991</span> for (String wal : walsSet) {<a name="line.991"></a>
+<span class="sourceLineNo">992</span> String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal);<a name="line.992"></a>
+<span class="sourceLineNo">993</span> NavigableSet<String> wals = walsByGroup.get(walPrefix);<a name="line.993"></a>
+<span class="sourceLineNo">994</span> if (wals == null) {<a name="line.994"></a>
+<span class="sourceLineNo">995</span> wals = new TreeSet<>();<a name="line.995"></a>
+<span class="sourceLineNo">996</span> walsByGroup.put(walPrefix, wals);<a name="line.996"></a>
+<span class="sourceLineNo">997</span> }<a name="line.997"></a>
+<span class="sourceLineNo">998</span> wals.add(wal);<a name="line.998"></a>
+<span class="sourceLineNo">999</span> }<a name="line.999"></a>
+<span class="sourceLineNo">1000</span> oldsources.add(src);<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span> LOG.info("Added source for recovered queue {}", src.getQueueId());<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span> for (String wal : walsSet) {<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span> LOG.trace("Enqueueing log from recovered queue for source: " + src.getQueueId());<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span> src.enqueueLog(new Path(oldLogDir, wal));<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span> src.startup();<a name="line.1006"></a>
+<span class="sourceLineNo">1007</span> }<a name="line.1007"></a>
+<span class="sourceLineNo">1008</span> } catch (IOException e) {<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span> // TODO manage it<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span> LOG.error("Failed creating a source", e);<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span> }<a name="line.1011"></a>
+<span class="sourceLineNo">1012</span> }<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span> }<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span><a name="line.1015"></a>
+<span class="sourceLineNo">1016</span> /**<a name="line.1016"></a>
+<span class="sourceLineNo">1017</span> * Terminate the replication on this region server<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span> */<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span> public void join() {<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span> this.executor.shutdown();<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span> source.terminate("Region server is closing");<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
+<span class="sourceLineNo">1024</span> }<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span><a name="line.1025"></a>
+<span class="sourceLineNo">1026</span> /**<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span> * Get a copy of the wals of the normal sources on this rs<a name="line.1027"></a>
+<span class="sourceLineNo">1028</span> * @return a sorted set of wal names<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span> */<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span> public Map<String, Map<String, NavigableSet<String>>> getWALs() {<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span> return Collections.unmodifiableMap(walsById);<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span><a name="line.1033"></a>
+<span class="sourceLineNo">1034</span> /**<a name="line.1034"></a>
+<span class="sourceLineNo">1035</span> * Get a copy of the wals of the recovered sources on this rs<a name="line.1035"></a>
+<span class="sourceLineNo">1036</span> * @return a sorted set of wal names<a name="line.1036"></a>
+<span class="sourceLineNo">1037</span> */<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span> Map<String, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span> return Collections.unmodifiableMap(walsByIdRecoveredQueues);<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span> }<a name="line.1040"></a>
+<span class="sourceLineNo">1041</span><a name="line.1041"></a>
+<span class="sourceLineNo">1042</span> /**<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span> * Get a list of all the normal sources of this rs<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span> * @return list of all normal sources<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span> */<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span> public List<ReplicationSourceInterface> getSources() {<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span> return new ArrayList<>(this.sources.values());<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span> }<a name="line.1048"></a>
+<span class="sourceLineNo">1049</span><a name="line.1049"></a>
+<span class="sourceLineNo">1050</span> /**<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span> * Get a list of all the recovered sources of this rs<a name="line.1051"></a>
+<span class="sourceLineNo">1052</span> * @return list of all recovered sources<a name="line.1052"></a>
+<span class="sourceLineNo">1053</span> */<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span> public List<ReplicationSourceInterface> getOldSources() {<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span> return this.oldsources;<a name="line.1055"></a>
+<span class="sourceLineNo">1056</span> }<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span><a name="line.1057"></a>
+<span class="sourceLineNo">1058</span> /**<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span> * Get the normal source for a given peer<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span> * @return the normal source for the give peer if it exists, otherwise null.<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span> */<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span> public ReplicationSourceInterface getSource(String peerId) {<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span> return this.sources.get(peerId);<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span> }<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span><a name="line.1065"></a>
+<span class="sourceLineNo">1066</span> List<String> getAllQueues() throws IOException {<a name="line.1066"></a>
+<span class="sourceLineNo">1067</span> List<String> allQueues = Collections.emptyList();<a name="line.1067"></a>
+<span class="sourceLineNo">1068</span> try {<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span> allQueues = queueStorage.getAllQueues(server.getServerName());<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span> } catch (ReplicationException e) {<a name="line.1070"></a>
+<span class="sourceLineNo">1071</span> throw new IOException(e);<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span> }<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span> return allQueues;<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span><a name="line.1075"></a>
+<span class="sourceLineNo">1076</span> int getSizeOfLatestPath() {<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span> synchronized (latestPaths) {<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span> return latestPaths.size();<a name="line.1078"></a>
+<span class="sourceLineNo">1079</span> }<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span> }<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span><a name="line.1081"></a>
+<span class="sourceLineNo">1082</span> Set<Path> getLastestPath() {<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span> synchronized (latestPaths) {<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span> return Sets.newHashSet(latestPaths.values());<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span> }<a name="line.1085"></a>
+<span class="sourceLineNo">1086</span> }<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span><a name="line.1087"></a>
+<span class="sourceLineNo">1088</span> public AtomicLong getTotalBufferUsed() {<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span> return totalBufferUsed;<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span> }<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span><a name="line.1091"></a>
+<span class="sourceLineNo">1092</span> /**<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span> * Returns the maximum size in bytes of edits held in memory which are pending replication<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span> * across all sources inside this RegionServer.<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span> */<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span> public long getTotalBufferLimit() {<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span> return totalBufferLimit;<a name="line.1097"></a>
+<span class="sourceLineNo">1098</span> }<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span><a name="line.1099"></a>
+<span class="sourceLineNo">1100</span> /**<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span> * Get the directory where wals are archived<a name="line.1101"></a>
+<span class="sourceLineNo">1102</span> * @return the directory where wals are archived<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span> */<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span> public Path getOldLogDir() {<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span> return this.oldLogDir;<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span> }<a name="line.1106"></a>
+<span class="sourceLineNo">1107</span><a name="line.1107"></a>
+<span class="sourceLineNo">1108</span> /**<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span> * Get the directory where wals are stored by their RSs<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span> * @return the directory where wals are stored by their RSs<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span> */<a name="line.1111"></a>
+<span class="sourceLineNo">1112</span> public Path getLogDir() {<a name="line.1112"></a>
+<span class="sourceLineNo">1113</span> return this.logDir;<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span> }<a name="line.1114"></a>
+<span class="sourceLineNo">1115</span><a name="line.1115"></a>
+<span class="sourceLineNo">1116</span> /**<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span> * Get the handle on the local file system<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span> * @return Handle on the local file system<a name="line.1118"></a>
+<span class="sourceLineNo">1119</span> */<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span> public FileSystem getFs() {<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span> return this.fs;<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span> }<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span><a name="line.1123"></a>
+<span class="sourceLineNo">1124</span> /**<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span> * Get the ReplicationPeers used by this ReplicationSourceManager<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span> * @return the ReplicationPeers used by this ReplicationSourceManager<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span> */<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span> public ReplicationPeers getReplicationPeers() {<a name="line.1128"></a>
+<span class="sourceLineNo">1129</span> return this.replicationPeers;<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span> }<a name="line.1130"></a>
+<span class="sourceLineNo">1131</span><a name="line.1131"></a>
+<span class="sourceLineNo">1132</span> /**<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span> * Get a string representation of all the sources' metrics<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span> */<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span> public String getStats() {<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span> StringBuilder stats = new StringBuilder();<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span> // Print stats that apply across all Replication Sources<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span> stats.append("Global stats: ");<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span> stats.append("WAL Edits Buffer Used=").append(getTotalBufferUsed().get()).append("B, Limit=")<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span> .append(getTotalBufferLimit()).append("B\n");<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span> stats.append("Normal source for cluster " + source.getPeerId() + ": ");<a name="line.1142"></a>
+<span class="sourceLineNo">1143</span> stats.append(source.getStats() + "\n");<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span> }<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span> for (ReplicationSourceInterface oldSource : oldsources) {<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span> stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerId() + ": ");<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span> stats.append(oldSource.getStats() + "\n");<a name="line.1147"></a>
+<span class="sourceLineNo">1148</span> }<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span> return stats.toString();<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span> }<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span><a name="line.1151"></a>
+<span class="sourceLineNo">1152</span> public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span> throws IOException {<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span> throwIOExceptionWhenFail(() -> source.addHFileRefs(tableName, family, pairs));<a name="line.1155"></a>
+<span class="sourceLineNo">1156</span> }<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span> }<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span><a name="line.1158"></a>
+<span class="sourceLineNo">1159</span> public void cleanUpHFileRefs(String peerId, List<String> files) {<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span> interruptOrAbortWhenFail(() -> this.queueStorage.removeHFileRefs(peerId, files));<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span> }<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span><a name="line.1162"></a>
+<span class="sourceLineNo">1163</span> int activeFailoverTaskCount() {<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span> return executor.getActiveCount();<a name="line.1164"></a>
+<span class="sourceLineNo">1165</span> }<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span><a name="line.1166"></a>
+<span class="sourceLineNo">1167</span> MetricsReplicationGlobalSourceSource getGlobalMetrics() {<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span> return this.globalMetrics;<a name="line.1168"></a>
+<span class="sourceLineNo">1169</span> }<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span><a name="line.1170"></a>
+<span class="sourceLineNo">1171</span> /**<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span> * Add an hbase:meta Catalog replication source. Called on open of an hbase:meta Region.<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span> * Create it once only. If exists already, use the existing one.<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span> * @see #removeCatalogReplicationSource(RegionInfo)<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span> * @see #addSource(String) This is specialization on the addSource method.<a name="line.1175"></a>
+<span class="sourceLineNo">1176</span> */<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span> public ReplicationSourceInterface addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span> throws IOException {<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span> // Poor-man's putIfAbsent<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span> synchronized (this.catalogReplicationSource) {<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span> ReplicationSourceInterface rs = this.catalogReplicationSource.get();<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span> return rs != null ? rs :<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span> this.catalogReplicationSource.getAndSet(createCatalogReplicationSource(regionInfo));<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span> }<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span> }<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span><a name="line.1186"></a>
+<span class="sourceLineNo">1187</span> /**<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span> * Remove the hbase:meta Catalog replication source.<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span> * Called when we close hbase:meta.<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span> * @see #addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span> */<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span> public void removeCatalogReplicationSource(RegionInfo regionInfo) {<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span> // Nothing to do. Leave any CatalogReplicationSource in place in case an hbase:meta Region<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span> // comes back to this server.<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span> }<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span><a name="line.1196"></a>
+<span class="sourceLineNo">1197</span> /**<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span> * Create, initialize, and start the Catalog ReplicationSource.<a name="line.1198"></a>
+<span class="sourceLineNo">1199</span> * Presumes called one-time only (caller must ensure one-time only call).<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span> * This ReplicationSource is NOT created via {@link ReplicationSourceFactory}.<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span> * @see #addSource(String) This is a specialization of the addSource call.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span> * @see #catalogReplicationSource for a note on this ReplicationSource's lifecycle (and more on<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span> * why the special handling).<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span> */<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span> private ReplicationSourceInterface createCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span> throws IOException {<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span> // Instantiate meta walProvider. Instantiated here or over in the #warmupRegion call made by the<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span> // Master on a 'move' operation. Need to do extra work if we did NOT instantiate the provider.<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span> WALProvider walProvider = this.walFactory.getMetaWALProvider();<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span> boolean instantiate = walProvider == null;<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span> if (instantiate) {<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span> walProvider = this.walFactory.getMetaProvider();<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span> }<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span> // Here we do a specialization on what {@link ReplicationSourceFactory} does. There is no need<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span> // for persisting offset into WALs up in zookeeper (via ReplicationQueueInfo) as the catalog<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span> // read replicas feature that makes use of the source does a reset on a crash of the WAL<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span> // source process. See "4.1 Skip maintaining zookeeper replication queue (offsets/WALs)" in the<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span> // design doc attached to HBASE-18070 'Enable memstore replication for meta replica' for detail.<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span> CatalogReplicationSourcePeer peer = new CatalogReplicationSourcePeer(this.conf,<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span> this.clusterId.toString(), "meta_" + ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER);<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span> final ReplicationSourceInterface crs = new CatalogReplicationSource();<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span> crs.init(conf, fs, this, new NoopReplicationQueueStorage(), peer, server, peer.getId(),<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span> clusterId, walProvider.getWALFileLengthProvider(), new MetricsSource(peer.getId()));<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span> // Add listener on the provider so we can pick up the WAL to replicate on roll.<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span> WALActionsListener listener = new WALActionsListener() {<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span> @Override public void postLogRoll(Path oldPath, Path newPath) throws IOException {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span> crs.enqueueLog(newPath);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span> }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span> };<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span> walProvider.addWALActionsListener(listener);<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span> if (!instantiate) {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span> // If we did not instantiate provider, need to add our listener on already-created WAL<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span> // instance too (listeners are passed by provider to WAL instance on creation but if provider<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span> // created already, our listener add above is missed). And add the current WAL file to the<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span> // Replication Source so it can start replicating it.<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span> WAL wal = walProvider.getWAL(regionInfo);<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span> wal.registerWALActionsListener(listener);<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span> crs.enqueueLog(((AbstractFSWAL)wal).getCurrentFileName());<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span> }<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span> return crs.startup();<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span> }<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span>}<a name="line.1242"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html
index cf69e73..26166d2 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.html
@@ -291,964 +291,963 @@
<span class="sourceLineNo">283</span> if (currentReplicators == null || currentReplicators.isEmpty()) {<a name="line.283"></a>
<span class="sourceLineNo">284</span> return;<a name="line.284"></a>
<span class="sourceLineNo">285</span> }<a name="line.285"></a>
-<span class="sourceLineNo">286</span> List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers().stream()<a name="line.286"></a>
-<span class="sourceLineNo">287</span> .map(ServerName::valueOf).collect(Collectors.toList());<a name="line.287"></a>
-<span class="sourceLineNo">288</span> LOG.info(<a name="line.288"></a>
-<span class="sourceLineNo">289</span> "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);<a name="line.289"></a>
-<span class="sourceLineNo">290</span><a name="line.290"></a>
-<span class="sourceLineNo">291</span> // Look if there's anything to process after a restart<a name="line.291"></a>
-<span class="sourceLineNo">292</span> for (ServerName rs : currentReplicators) {<a name="line.292"></a>
-<span class="sourceLineNo">293</span> if (!otherRegionServers.contains(rs)) {<a name="line.293"></a>
-<span class="sourceLineNo">294</span> transferQueues(rs);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> }<a name="line.295"></a>
-<span class="sourceLineNo">296</span> }<a name="line.296"></a>
-<span class="sourceLineNo">297</span> }<a name="line.297"></a>
-<span class="sourceLineNo">298</span><a name="line.298"></a>
-<span class="sourceLineNo">299</span> /**<a name="line.299"></a>
-<span class="sourceLineNo">300</span> * <ol><a name="line.300"></a>
-<span class="sourceLineNo">301</span> * <li>Add peer to replicationPeers</li><a name="line.301"></a>
-<span class="sourceLineNo">302</span> * <li>Add the normal source and related replication queue</li><a name="line.302"></a>
-<span class="sourceLineNo">303</span> * <li>Add HFile Refs</li><a name="line.303"></a>
-<span class="sourceLineNo">304</span> * </ol><a name="line.304"></a>
-<span class="sourceLineNo">305</span> * @param peerId the id of replication peer<a name="line.305"></a>
-<span class="sourceLineNo">306</span> */<a name="line.306"></a>
-<span class="sourceLineNo">307</span> public void addPeer(String peerId) throws IOException {<a name="line.307"></a>
-<span class="sourceLineNo">308</span> boolean added = false;<a name="line.308"></a>
-<span class="sourceLineNo">309</span> try {<a name="line.309"></a>
-<span class="sourceLineNo">310</span> added = this.replicationPeers.addPeer(peerId);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> } catch (ReplicationException e) {<a name="line.311"></a>
-<span class="sourceLineNo">312</span> throw new IOException(e);<a name="line.312"></a>
-<span class="sourceLineNo">313</span> }<a name="line.313"></a>
-<span class="sourceLineNo">314</span> if (added) {<a name="line.314"></a>
-<span class="sourceLineNo">315</span> addSource(peerId);<a name="line.315"></a>
-<span class="sourceLineNo">316</span> if (replicationForBulkLoadDataEnabled) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span> throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));<a name="line.317"></a>
-<span class="sourceLineNo">318</span> }<a name="line.318"></a>
-<span class="sourceLineNo">319</span> }<a name="line.319"></a>
-<span class="sourceLineNo">320</span> }<a name="line.320"></a>
-<span class="sourceLineNo">321</span><a name="line.321"></a>
-<span class="sourceLineNo">322</span> /**<a name="line.322"></a>
-<span class="sourceLineNo">323</span> * <ol><a name="line.323"></a>
-<span class="sourceLineNo">324</span> * <li>Remove peer for replicationPeers</li><a name="line.324"></a>
-<span class="sourceLineNo">325</span> * <li>Remove all the recovered sources for the specified id and related replication queues</li><a name="line.325"></a>
-<span class="sourceLineNo">326</span> * <li>Remove the normal source and related replication queue</li><a name="line.326"></a>
-<span class="sourceLineNo">327</span> * <li>Remove HFile Refs</li><a name="line.327"></a>
-<span class="sourceLineNo">328</span> * </ol><a name="line.328"></a>
-<span class="sourceLineNo">329</span> * @param peerId the id of the replication peer<a name="line.329"></a>
-<span class="sourceLineNo">330</span> */<a name="line.330"></a>
-<span class="sourceLineNo">331</span> public void removePeer(String peerId) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> ReplicationPeer peer = replicationPeers.removePeer(peerId);<a name="line.332"></a>
-<span class="sourceLineNo">333</span> String terminateMessage = "Replication stream was removed by a user";<a name="line.333"></a>
-<span class="sourceLineNo">334</span> List<ReplicationSourceInterface> oldSourcesToDelete = new ArrayList<>();<a name="line.334"></a>
-<span class="sourceLineNo">335</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.335"></a>
-<span class="sourceLineNo">336</span> // see NodeFailoverWorker.run<a name="line.336"></a>
-<span class="sourceLineNo">337</span> synchronized (this.oldsources) {<a name="line.337"></a>
-<span class="sourceLineNo">338</span> // First close all the recovered sources for this peer<a name="line.338"></a>
-<span class="sourceLineNo">339</span> for (ReplicationSourceInterface src : oldsources) {<a name="line.339"></a>
-<span class="sourceLineNo">340</span> if (peerId.equals(src.getPeerId())) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span> oldSourcesToDelete.add(src);<a name="line.341"></a>
-<span class="sourceLineNo">342</span> }<a name="line.342"></a>
-<span class="sourceLineNo">343</span> }<a name="line.343"></a>
-<span class="sourceLineNo">344</span> for (ReplicationSourceInterface src : oldSourcesToDelete) {<a name="line.344"></a>
-<span class="sourceLineNo">345</span> src.terminate(terminateMessage);<a name="line.345"></a>
-<span class="sourceLineNo">346</span> removeRecoveredSource(src);<a name="line.346"></a>
-<span class="sourceLineNo">347</span> }<a name="line.347"></a>
-<span class="sourceLineNo">348</span> }<a name="line.348"></a>
-<span class="sourceLineNo">349</span> LOG.info(<a name="line.349"></a>
-<span class="sourceLineNo">350</span> "Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());<a name="line.350"></a>
-<span class="sourceLineNo">351</span> // Now close the normal source for this peer<a name="line.351"></a>
-<span class="sourceLineNo">352</span> ReplicationSourceInterface srcToRemove = this.sources.get(peerId);<a name="line.352"></a>
-<span class="sourceLineNo">353</span> if (srcToRemove != null) {<a name="line.353"></a>
-<span class="sourceLineNo">354</span> srcToRemove.terminate(terminateMessage);<a name="line.354"></a>
-<span class="sourceLineNo">355</span> removeSource(srcToRemove);<a name="line.355"></a>
-<span class="sourceLineNo">356</span> } else {<a name="line.356"></a>
-<span class="sourceLineNo">357</span> // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup<a name="line.357"></a>
-<span class="sourceLineNo">358</span> // Delete queue from storage and memory and queue id is same with peer id for normal<a name="line.358"></a>
-<span class="sourceLineNo">359</span> // source<a name="line.359"></a>
-<span class="sourceLineNo">360</span> deleteQueue(peerId);<a name="line.360"></a>
-<span class="sourceLineNo">361</span> this.walsById.remove(peerId);<a name="line.361"></a>
-<span class="sourceLineNo">362</span> }<a name="line.362"></a>
-<span class="sourceLineNo">363</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> if (peerConfig.isSyncReplication()) {<a name="line.364"></a>
-<span class="sourceLineNo">365</span> syncReplicationPeerMappingManager.remove(peerId, peerConfig);<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> // Remove HFile Refs<a name="line.367"></a>
-<span class="sourceLineNo">368</span> abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span> /**<a name="line.371"></a>
-<span class="sourceLineNo">372</span> * @return a new 'classic' user-space replication source.<a name="line.372"></a>
-<span class="sourceLineNo">373</span> * @param queueId the id of the replication queue to associate the ReplicationSource with.<a name="line.373"></a>
-<span class="sourceLineNo">374</span> * @see #createCatalogReplicationSource(RegionInfo) for creating a ReplicationSource for meta.<a name="line.374"></a>
-<span class="sourceLineNo">375</span> */<a name="line.375"></a>
-<span class="sourceLineNo">376</span> private ReplicationSourceInterface createSource(String queueId, ReplicationPeer replicationPeer)<a name="line.376"></a>
-<span class="sourceLineNo">377</span> throws IOException {<a name="line.377"></a>
-<span class="sourceLineNo">378</span> ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueId);<a name="line.378"></a>
-<span class="sourceLineNo">379</span> // Init the just created replication source. Pass the default walProvider's wal file length<a name="line.379"></a>
-<span class="sourceLineNo">380</span> // provider. Presumption is we replicate user-space Tables only. For hbase:meta region replica<a name="line.380"></a>
-<span class="sourceLineNo">381</span> // replication, see #createCatalogReplicationSource().<a name="line.381"></a>
-<span class="sourceLineNo">382</span> WALFileLengthProvider walFileLengthProvider =<a name="line.382"></a>
-<span class="sourceLineNo">383</span> this.walFactory.getWALProvider() != null?<a name="line.383"></a>
-<span class="sourceLineNo">384</span> this.walFactory.getWALProvider().getWALFileLengthProvider() : p -> OptionalLong.empty();<a name="line.384"></a>
-<span class="sourceLineNo">385</span> src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId,<a name="line.385"></a>
-<span class="sourceLineNo">386</span> walFileLengthProvider, new MetricsSource(queueId));<a name="line.386"></a>
-<span class="sourceLineNo">387</span> return src;<a name="line.387"></a>
-<span class="sourceLineNo">388</span> }<a name="line.388"></a>
-<span class="sourceLineNo">389</span><a name="line.389"></a>
-<span class="sourceLineNo">390</span> /**<a name="line.390"></a>
-<span class="sourceLineNo">391</span> * Add a normal source for the given peer on this region server. Meanwhile, add new replication<a name="line.391"></a>
-<span class="sourceLineNo">392</span> * queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal<a name="line.392"></a>
-<span class="sourceLineNo">393</span> * group and do replication<a name="line.393"></a>
-<span class="sourceLineNo">394</span> * @param peerId the id of the replication peer<a name="line.394"></a>
-<span class="sourceLineNo">395</span> * @return the source that was created<a name="line.395"></a>
-<span class="sourceLineNo">396</span> */<a name="line.396"></a>
-<span class="sourceLineNo">397</span> ReplicationSourceInterface addSource(String peerId) throws IOException {<a name="line.397"></a>
-<span class="sourceLineNo">398</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.398"></a>
-<span class="sourceLineNo">399</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.399"></a>
-<span class="sourceLineNo">400</span> // synchronized on latestPaths to avoid missing the new log<a name="line.400"></a>
-<span class="sourceLineNo">401</span> synchronized (this.latestPaths) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> this.sources.put(peerId, src);<a name="line.402"></a>
-<span class="sourceLineNo">403</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.403"></a>
-<span class="sourceLineNo">404</span> this.walsById.put(peerId, walsByGroup);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> // Add the latest wal to that source's queue<a name="line.405"></a>
-<span class="sourceLineNo">406</span> if (!latestPaths.isEmpty()) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span> for (Map.Entry<String, Path> walPrefixAndPath : latestPaths.entrySet()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> Path walPath = walPrefixAndPath.getValue();<a name="line.408"></a>
-<span class="sourceLineNo">409</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.409"></a>
-<span class="sourceLineNo">410</span> wals.add(walPath.getName());<a name="line.410"></a>
-<span class="sourceLineNo">411</span> walsByGroup.put(walPrefixAndPath.getKey(), wals);<a name="line.411"></a>
-<span class="sourceLineNo">412</span> // Abort RS and throw exception to make add peer failed<a name="line.412"></a>
-<span class="sourceLineNo">413</span> abortAndThrowIOExceptionWhenFail(<a name="line.413"></a>
-<span class="sourceLineNo">414</span> () -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName()));<a name="line.414"></a>
-<span class="sourceLineNo">415</span> src.enqueueLog(walPath);<a name="line.415"></a>
-<span class="sourceLineNo">416</span> LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());<a name="line.416"></a>
-<span class="sourceLineNo">417</span> }<a name="line.417"></a>
-<span class="sourceLineNo">418</span> }<a name="line.418"></a>
-<span class="sourceLineNo">419</span> }<a name="line.419"></a>
-<span class="sourceLineNo">420</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.420"></a>
-<span class="sourceLineNo">421</span> if (peerConfig.isSyncReplication()) {<a name="line.421"></a>
-<span class="sourceLineNo">422</span> syncReplicationPeerMappingManager.add(peer.getId(), peerConfig);<a name="line.422"></a>
-<span class="sourceLineNo">423</span> }<a name="line.423"></a>
-<span class="sourceLineNo">424</span> src.startup();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> return src;<a name="line.425"></a>
-<span class="sourceLineNo">426</span> }<a name="line.426"></a>
-<span class="sourceLineNo">427</span><a name="line.427"></a>
-<span class="sourceLineNo">428</span> /**<a name="line.428"></a>
-<span class="sourceLineNo">429</span> * <p><a name="line.429"></a>
-<span class="sourceLineNo">430</span> * This is used when we transit a sync replication peer to {@link SyncReplicationState#STANDBY}.<a name="line.430"></a>
-<span class="sourceLineNo">431</span> * </p><a name="line.431"></a>
-<span class="sourceLineNo">432</span> * <p><a name="line.432"></a>
-<span class="sourceLineNo">433</span> * When transiting to {@link SyncReplicationState#STANDBY}, we can remove all the pending wal<a name="line.433"></a>
-<span class="sourceLineNo">434</span> * files for a replication peer as we do not need to replicate them any more. And this is<a name="line.434"></a>
-<span class="sourceLineNo">435</span> * necessary, otherwise when we transit back to {@link SyncReplicationState#DOWNGRADE_ACTIVE}<a name="line.435"></a>
-<span class="sourceLineNo">436</span> * later, the stale data will be replicated again and cause inconsistency.<a name="line.436"></a>
-<span class="sourceLineNo">437</span> * </p><a name="line.437"></a>
-<span class="sourceLineNo">438</span> * <p><a name="line.438"></a>
-<span class="sourceLineNo">439</span> * See HBASE-20426 for more details.<a name="line.439"></a>
-<span class="sourceLineNo">440</span> * </p><a name="line.440"></a>
-<span class="sourceLineNo">441</span> * @param peerId the id of the sync replication peer<a name="line.441"></a>
-<span class="sourceLineNo">442</span> */<a name="line.442"></a>
-<span class="sourceLineNo">443</span> public void drainSources(String peerId) throws IOException, ReplicationException {<a name="line.443"></a>
-<span class="sourceLineNo">444</span> String terminateMessage = "Sync replication peer " + peerId +<a name="line.444"></a>
-<span class="sourceLineNo">445</span> " is transiting to STANDBY. Will close the previous replication source and open a new one";<a name="line.445"></a>
-<span class="sourceLineNo">446</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.446"></a>
-<span class="sourceLineNo">447</span> assert peer.getPeerConfig().isSyncReplication();<a name="line.447"></a>
-<span class="sourceLineNo">448</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.448"></a>
-<span class="sourceLineNo">449</span> // synchronized here to avoid race with preLogRoll where we add new log to source and also<a name="line.449"></a>
-<span class="sourceLineNo">450</span> // walsById.<a name="line.450"></a>
-<span class="sourceLineNo">451</span> ReplicationSourceInterface toRemove;<a name="line.451"></a>
-<span class="sourceLineNo">452</span> Map<String, NavigableSet<String>> wals = new HashMap<>();<a name="line.452"></a>
-<span class="sourceLineNo">453</span> synchronized (latestPaths) {<a name="line.453"></a>
-<span class="sourceLineNo">454</span> toRemove = sources.put(peerId, src);<a name="line.454"></a>
-<span class="sourceLineNo">455</span> if (toRemove != null) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.456"></a>
-<span class="sourceLineNo">457</span> toRemove.terminate(terminateMessage);<a name="line.457"></a>
-<span class="sourceLineNo">458</span> toRemove.getSourceMetrics().clear();<a name="line.458"></a>
-<span class="sourceLineNo">459</span> }<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // Here we make a copy of all the remaining wal files and then delete them from the<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // replication queue storage after releasing the lock. It is not safe to just remove the old<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // map from walsById since later we may fail to delete them from the replication queue<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // storage, and when we retry next time, we can not know the wal files that need to be deleted<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // from the replication queue storage.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> walsById.get(peerId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));<a name="line.465"></a>
-<span class="sourceLineNo">466</span> }<a name="line.466"></a>
-<span class="sourceLineNo">467</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.467"></a>
-<span class="sourceLineNo">468</span> src.startup();<a name="line.468"></a>
-<span class="sourceLineNo">469</span> for (NavigableSet<String> walsByGroup : wals.values()) {<a name="line.469"></a>
-<span class="sourceLineNo">470</span> for (String wal : walsByGroup) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> queueStorage.removeWAL(server.getServerName(), peerId, wal);<a name="line.471"></a>
-<span class="sourceLineNo">472</span> }<a name="line.472"></a>
-<span class="sourceLineNo">473</span> }<a name="line.473"></a>
-<span class="sourceLineNo">474</span> synchronized (walsById) {<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Map<String, NavigableSet<String>> oldWals = walsById.get(peerId);<a name="line.475"></a>
-<span class="sourceLineNo">476</span> wals.forEach((k, v) -> {<a name="line.476"></a>
-<span class="sourceLineNo">477</span> NavigableSet<String> walsByGroup = oldWals.get(k);<a name="line.477"></a>
-<span class="sourceLineNo">478</span> if (walsByGroup != null) {<a name="line.478"></a>
-<span class="sourceLineNo">479</span> walsByGroup.removeAll(v);<a name="line.479"></a>
-<span class="sourceLineNo">480</span> }<a name="line.480"></a>
-<span class="sourceLineNo">481</span> });<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // synchronized on oldsources to avoid race with NodeFailoverWorker. Since NodeFailoverWorker is<a name="line.483"></a>
-<span class="sourceLineNo">484</span> // a background task, we will delete the file from replication queue storage under the lock to<a name="line.484"></a>
-<span class="sourceLineNo">485</span> // simplify the logic.<a name="line.485"></a>
-<span class="sourceLineNo">486</span> synchronized (this.oldsources) {<a name="line.486"></a>
-<span class="sourceLineNo">487</span> for (Iterator<ReplicationSourceInterface> iter = oldsources.iterator(); iter.hasNext();) {<a name="line.487"></a>
-<span class="sourceLineNo">488</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.488"></a>
-<span class="sourceLineNo">489</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.489"></a>
-<span class="sourceLineNo">490</span> String queueId = oldSource.getQueueId();<a name="line.490"></a>
-<span class="sourceLineNo">491</span> oldSource.terminate(terminateMessage);<a name="line.491"></a>
-<span class="sourceLineNo">492</span> oldSource.getSourceMetrics().clear();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> queueStorage.removeQueue(server.getServerName(), queueId);<a name="line.493"></a>
-<span class="sourceLineNo">494</span> walsByIdRecoveredQueues.remove(queueId);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> iter.remove();<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span> }<a name="line.497"></a>
-<span class="sourceLineNo">498</span> }<a name="line.498"></a>
-<span class="sourceLineNo">499</span> }<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span> /**<a name="line.501"></a>
-<span class="sourceLineNo">502</span> * Close the previous replication sources of this peer id and open new sources to trigger the new<a name="line.502"></a>
-<span class="sourceLineNo">503</span> * replication state changes or new replication config changes. Here we don't need to change<a name="line.503"></a>
-<span class="sourceLineNo">504</span> * replication queue storage and only to enqueue all logs to the new replication source<a name="line.504"></a>
-<span class="sourceLineNo">505</span> * @param peerId the id of the replication peer<a name="line.505"></a>
-<span class="sourceLineNo">506</span> */<a name="line.506"></a>
-<span class="sourceLineNo">507</span> public void refreshSources(String peerId) throws IOException {<a name="line.507"></a>
-<span class="sourceLineNo">508</span> String terminateMessage = "Peer " + peerId +<a name="line.508"></a>
-<span class="sourceLineNo">509</span> " state or config changed. Will close the previous replication source and open a new one";<a name="line.509"></a>
-<span class="sourceLineNo">510</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.510"></a>
-<span class="sourceLineNo">511</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.511"></a>
-<span class="sourceLineNo">512</span> // synchronized on latestPaths to avoid missing the new log<a name="line.512"></a>
-<span class="sourceLineNo">513</span> synchronized (this.latestPaths) {<a name="line.513"></a>
-<span class="sourceLineNo">514</span> ReplicationSourceInterface toRemove = this.sources.put(peerId, src);<a name="line.514"></a>
-<span class="sourceLineNo">515</span> if (toRemove != null) {<a name="line.515"></a>
-<span class="sourceLineNo">516</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.516"></a>
-<span class="sourceLineNo">517</span> // Do not clear metrics<a name="line.517"></a>
-<span class="sourceLineNo">518</span> toRemove.terminate(terminateMessage, null, false);<a name="line.518"></a>
-<span class="sourceLineNo">519</span> }<a name="line.519"></a>
-<span class="sourceLineNo">520</span> for (NavigableSet<String> walsByGroup : walsById.get(peerId).values()) {<a name="line.520"></a>
-<span class="sourceLineNo">521</span> walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));<a name="line.521"></a>
-<span class="sourceLineNo">522</span> }<a name="line.522"></a>
-<span class="sourceLineNo">523</span> }<a name="line.523"></a>
-<span class="sourceLineNo">524</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.524"></a>
-<span class="sourceLineNo">525</span> src.startup();<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> List<ReplicationSourceInterface> toStartup = new ArrayList<>();<a name="line.527"></a>
-<span class="sourceLineNo">528</span> // synchronized on oldsources to avoid race with NodeFailoverWorker<a name="line.528"></a>
-<span class="sourceLineNo">529</span> synchronized (this.oldsources) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span> List<String> previousQueueIds = new ArrayList<>();<a name="line.530"></a>
-<span class="sourceLineNo">531</span> for (Iterator<ReplicationSourceInterface> iter = this.oldsources.iterator(); iter<a name="line.531"></a>
-<span class="sourceLineNo">532</span> .hasNext();) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.534"></a>
-<span class="sourceLineNo">535</span> previousQueueIds.add(oldSource.getQueueId());<a name="line.535"></a>
-<span class="sourceLineNo">536</span> oldSource.terminate(terminateMessage);<a name="line.536"></a>
-<span class="sourceLineNo">537</span> iter.remove();<a name="line.537"></a>
-<span class="sourceLineNo">538</span> }<a name="line.538"></a>
-<span class="sourceLineNo">539</span> }<a name="line.539"></a>
-<span class="sourceLineNo">540</span> for (String queueId : previousQueueIds) {<a name="line.540"></a>
-<span class="sourceLineNo">541</span> ReplicationSourceInterface recoveredReplicationSource = createSource(queueId, peer);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> this.oldsources.add(recoveredReplicationSource);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {<a name="line.543"></a>
-<span class="sourceLineNo">544</span> walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span> toStartup.add(recoveredReplicationSource);<a name="line.546"></a>
-<span class="sourceLineNo">547</span> }<a name="line.547"></a>
-<span class="sourceLineNo">548</span> }<a name="line.548"></a>
-<span class="sourceLineNo">549</span> for (ReplicationSourceInterface replicationSource : toStartup) {<a name="line.549"></a>
-<span class="sourceLineNo">550</span> replicationSource.startup();<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span> }<a name="line.552"></a>
-<span class="sourceLineNo">553</span><a name="line.553"></a>
-<span class="sourceLineNo">554</span> /**<a name="line.554"></a>
-<span class="sourceLineNo">555</span> * Clear the metrics and related replication queue of the specified old source<a name="line.555"></a>
-<span class="sourceLineNo">556</span> * @param src source to clear<a name="line.556"></a>
-<span class="sourceLineNo">557</span> */<a name="line.557"></a>
-<span class="sourceLineNo">558</span> private boolean removeRecoveredSource(ReplicationSourceInterface src) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> if (!this.oldsources.remove(src)) {<a name="line.559"></a>
-<span class="sourceLineNo">560</span> return false;<a name="line.560"></a>
-<span class="sourceLineNo">561</span> }<a name="line.561"></a>
-<span class="sourceLineNo">562</span> LOG.info("Done with the recovered queue {}", src.getQueueId());<a name="line.562"></a>
-<span class="sourceLineNo">563</span> // Delete queue from storage and memory<a name="line.563"></a>
-<span class="sourceLineNo">564</span> deleteQueue(src.getQueueId());<a name="line.564"></a>
-<span class="sourceLineNo">565</span> this.walsByIdRecoveredQueues.remove(src.getQueueId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> return true;<a name="line.566"></a>
-<span class="sourceLineNo">567</span> }<a name="line.567"></a>
-<span class="sourceLineNo">568</span><a name="line.568"></a>
-<span class="sourceLineNo">569</span> void finishRecoveredSource(ReplicationSourceInterface src) {<a name="line.569"></a>
-<span class="sourceLineNo">570</span> synchronized (oldsources) {<a name="line.570"></a>
-<span class="sourceLineNo">571</span> if (!removeRecoveredSource(src)) {<a name="line.571"></a>
-<span class="sourceLineNo">572</span> return;<a name="line.572"></a>
-<span class="sourceLineNo">573</span> }<a name="line.573"></a>
-<span class="sourceLineNo">574</span> }<a name="line.574"></a>
-<span class="sourceLineNo">575</span> LOG.info("Finished recovering queue {} with the following stats: {}", src.getQueueId(),<a name="line.575"></a>
-<span class="sourceLineNo">576</span> src.getStats());<a name="line.576"></a>
-<span class="sourceLineNo">577</span> }<a name="line.577"></a>
-<span class="sourceLineNo">578</span><a name="line.578"></a>
-<span class="sourceLineNo">579</span> /**<a name="line.579"></a>
-<span class="sourceLineNo">580</span> * Clear the metrics and related replication queue of the specified old source<a name="line.580"></a>
-<span class="sourceLineNo">581</span> * @param src source to clear<a name="line.581"></a>
-<span class="sourceLineNo">582</span> */<a name="line.582"></a>
-<span class="sourceLineNo">583</span> void removeSource(ReplicationSourceInterface src) {<a name="line.583"></a>
-<span class="sourceLineNo">584</span> LOG.info("Done with the queue " + src.getQueueId());<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.sources.remove(src.getPeerId());<a name="line.585"></a>
-<span class="sourceLineNo">586</span> // Delete queue from storage and memory<a name="line.586"></a>
-<span class="sourceLineNo">587</span> deleteQueue(src.getQueueId());<a name="line.587"></a>
-<span class="sourceLineNo">588</span> this.walsById.remove(src.getQueueId());<a name="line.588"></a>
-<span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span> }<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> /**<a name="line.592"></a>
-<span class="sourceLineNo">593</span> * Delete a complete queue of wals associated with a replication source<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * @param queueId the id of replication queue to delete<a name="line.594"></a>
-<span class="sourceLineNo">595</span> */<a name="line.595"></a>
-<span class="sourceLineNo">596</span> private void deleteQueue(String queueId) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span> abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.597"></a>
-<span class="sourceLineNo">598</span> }<a name="line.598"></a>
-<span class="sourceLineNo">599</span><a name="line.599"></a>
-<span class="sourceLineNo">600</span> @FunctionalInterface<a name="line.600"></a>
-<span class="sourceLineNo">601</span> private interface ReplicationQueueOperation {<a name="line.601"></a>
-<span class="sourceLineNo">602</span> void exec() throws ReplicationException;<a name="line.602"></a>
-<span class="sourceLineNo">603</span> }<a name="line.603"></a>
-<span class="sourceLineNo">604</span><a name="line.604"></a>
-<span class="sourceLineNo">605</span> /**<a name="line.605"></a>
-<span class="sourceLineNo">606</span> * Refresh replication source will terminate the old source first, then the source thread will be<a name="line.606"></a>
-<span class="sourceLineNo">607</span> * interrupted. Need to handle it instead of abort the region server.<a name="line.607"></a>
-<span class="sourceLineNo">608</span> */<a name="line.608"></a>
-<span class="sourceLineNo">609</span> private void interruptOrAbortWhenFail(ReplicationQueueOperation op) {<a name="line.609"></a>
-<span class="sourceLineNo">610</span> try {<a name="line.610"></a>
-<span class="sourceLineNo">611</span> op.exec();<a name="line.611"></a>
-<span class="sourceLineNo">612</span> } catch (ReplicationException e) {<a name="line.612"></a>
-<span class="sourceLineNo">613</span> if (e.getCause() != null && e.getCause() instanceof KeeperException.SystemErrorException<a name="line.613"></a>
-<span class="sourceLineNo">614</span> && e.getCause().getCause() != null && e.getCause()<a name="line.614"></a>
-<span class="sourceLineNo">615</span> .getCause() instanceof InterruptedException) {<a name="line.615"></a>
-<span class="sourceLineNo">616</span> // ReplicationRuntimeException(a RuntimeException) is thrown out here. The reason is<a name="line.616"></a>
-<span class="sourceLineNo">617</span> // that thread is interrupted deep down in the stack, it should pass the following<a name="line.617"></a>
-<span class="sourceLineNo">618</span> // processing logic and propagate to the most top layer which can handle this exception<a name="line.618"></a>
-<span class="sourceLineNo">619</span> // properly. In this specific case, the top layer is ReplicationSourceShipper#run().<a name="line.619"></a>
-<span class="sourceLineNo">620</span> throw new ReplicationRuntimeException(<a name="line.620"></a>
-<span class="sourceLineNo">621</span> "Thread is interrupted, the replication source may be terminated",<a name="line.621"></a>
-<span class="sourceLineNo">622</span> e.getCause().getCause());<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span> server.abort("Failed to operate on replication queue", e);<a name="line.624"></a>
-<span class="sourceLineNo">625</span> }<a name="line.625"></a>
-<span class="sourceLineNo">626</span> }<a name="line.626"></a>
-<span class="sourceLineNo">627</span><a name="line.627"></a>
-<span class="sourceLineNo">628</span> private void abortWhenFail(ReplicationQueueOperation op) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> op.exec();<a name="line.630"></a>
-<span class="sourceLineNo">631</span> } catch (ReplicationException e) {<a name="line.631"></a>
-<span class="sourceLineNo">632</span> server.abort("Failed to operate on replication queue", e);<a name="line.632"></a>
-<span class="sourceLineNo">633</span> }<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span><a name="line.635"></a>
-<span class="sourceLineNo">636</span> private void throwIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.636"></a>
-<span class="sourceLineNo">637</span> try {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> op.exec();<a name="line.638"></a>
-<span class="sourceLineNo">639</span> } catch (ReplicationException e) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> throw new IOException(e);<a name="line.640"></a>
-<span class="sourceLineNo">641</span> }<a name="line.641"></a>
-<span class="sourceLineNo">642</span> }<a name="line.642"></a>
-<span class="sourceLineNo">643</span><a name="line.643"></a>
-<span class="sourceLineNo">644</span> private void abortAndThrowIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> try {<a name="line.645"></a>
-<span class="sourceLineNo">646</span> op.exec();<a name="line.646"></a>
-<span class="sourceLineNo">647</span> } catch (ReplicationException e) {<a name="line.647"></a>
-<span class="sourceLineNo">648</span> server.abort("Failed to operate on replication queue", e);<a name="line.648"></a>
-<span class="sourceLineNo">649</span> throw new IOException(e);<a name="line.649"></a>
-<span class="sourceLineNo">650</span> }<a name="line.650"></a>
-<span class="sourceLineNo">651</span> }<a name="line.651"></a>
-<span class="sourceLineNo">652</span><a name="line.652"></a>
-<span class="sourceLineNo">653</span> /**<a name="line.653"></a>
-<span class="sourceLineNo">654</span> * This method will log the current position to storage. And also clean old logs from the<a name="line.654"></a>
-<span class="sourceLineNo">655</span> * replication queue.<a name="line.655"></a>
-<span class="sourceLineNo">656</span> * @param source the replication source<a name="line.656"></a>
-<span class="sourceLineNo">657</span> * @param entryBatch the wal entry batch we just shipped<a name="line.657"></a>
-<span class="sourceLineNo">658</span> */<a name="line.658"></a>
-<span class="sourceLineNo">659</span> public void logPositionAndCleanOldLogs(ReplicationSourceInterface source,<a name="line.659"></a>
-<span class="sourceLineNo">660</span> WALEntryBatch entryBatch) {<a name="line.660"></a>
-<span class="sourceLineNo">661</span> String fileName = entryBatch.getLastWalPath().getName();<a name="line.661"></a>
-<span class="sourceLineNo">662</span> interruptOrAbortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(),<a name="line.662"></a>
-<span class="sourceLineNo">663</span> source.getQueueId(), fileName, entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds()));<a name="line.663"></a>
-<span class="sourceLineNo">664</span> cleanOldLogs(fileName, entryBatch.isEndOfFile(), source);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> }<a name="line.665"></a>
-<span class="sourceLineNo">666</span><a name="line.666"></a>
-<span class="sourceLineNo">667</span> /**<a name="line.667"></a>
-<span class="sourceLineNo">668</span> * Cleans a log file and all older logs from replication queue. Called when we are sure that a log<a name="line.668"></a>
-<span class="sourceLineNo">669</span> * file is closed and has no more entries.<a name="line.669"></a>
-<span class="sourceLineNo">670</span> * @param log Path to the log<a name="line.670"></a>
-<span class="sourceLineNo">671</span> * @param inclusive whether we should also remove the given log file<a name="line.671"></a>
-<span class="sourceLineNo">672</span> * @param source the replication source<a name="line.672"></a>
-<span class="sourceLineNo">673</span> */<a name="line.673"></a>
-<span class="sourceLineNo">674</span> void cleanOldLogs(String log, boolean inclusive, ReplicationSourceInterface source) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(log);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> if (source.isRecovered()) {<a name="line.676"></a>
-<span class="sourceLineNo">677</span> NavigableSet<String> wals = walsByIdRecoveredQueues.get(source.getQueueId()).get(logPrefix);<a name="line.677"></a>
-<span class="sourceLineNo">678</span> if (wals != null) {<a name="line.678"></a>
-<span class="sourceLineNo">679</span> NavigableSet<String> walsToRemove = wals.headSet(log, inclusive);<a name="line.679"></a>
-<span class="sourceLineNo">680</span> if (walsToRemove.isEmpty()) {<a name="line.680"></a>
-<span class="sourceLineNo">681</span> return;<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> cleanOldLogs(walsToRemove, source);<a name="line.683"></a>
-<span class="sourceLineNo">684</span> walsToRemove.clear();<a name="line.684"></a>
-<span class="sourceLineNo">685</span> }<a name="line.685"></a>
-<span class="sourceLineNo">686</span> } else {<a name="line.686"></a>
-<span class="sourceLineNo">687</span> NavigableSet<String> wals;<a name="line.687"></a>
-<span class="sourceLineNo">688</span> NavigableSet<String> walsToRemove;<a name="line.688"></a>
-<span class="sourceLineNo">689</span> // synchronized on walsById to avoid race with preLogRoll<a name="line.689"></a>
-<span class="sourceLineNo">690</span> synchronized (this.walsById) {<a name="line.690"></a>
-<span class="sourceLineNo">691</span> wals = walsById.get(source.getQueueId()).get(logPrefix);<a name="line.691"></a>
-<span class="sourceLineNo">692</span> if (wals == null) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span> return;<a name="line.693"></a>
-<span class="sourceLineNo">694</span> }<a name="line.694"></a>
-<span class="sourceLineNo">695</span> walsToRemove = wals.headSet(log, inclusive);<a name="line.695"></a>
-<span class="sourceLineNo">696</span> if (walsToRemove.isEmpty()) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span> return;<a name="line.697"></a>
-<span class="sourceLineNo">698</span> }<a name="line.698"></a>
-<span class="sourceLineNo">699</span> walsToRemove = new TreeSet<>(walsToRemove);<a name="line.699"></a>
-<span class="sourceLineNo">700</span> }<a name="line.700"></a>
-<span class="sourceLineNo">701</span> // cleanOldLogs may spend some time, especially for sync replication where we may want to<a name="line.701"></a>
-<span class="sourceLineNo">702</span> // remove remote wals as the remote cluster may have already been down, so we do it outside<a name="line.702"></a>
-<span class="sourceLineNo">703</span> // the lock to avoid block preLogRoll<a name="line.703"></a>
-<span class="sourceLineNo">704</span> cleanOldLogs(walsToRemove, source);<a name="line.704"></a>
-<span class="sourceLineNo">705</span> // now let's remove the files in the set<a name="line.705"></a>
-<span class="sourceLineNo">706</span> synchronized (this.walsById) {<a name="line.706"></a>
-<span class="sourceLineNo">707</span> wals.removeAll(walsToRemove);<a name="line.707"></a>
-<span class="sourceLineNo">708</span> }<a name="line.708"></a>
-<span class="sourceLineNo">709</span> }<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> private void removeRemoteWALs(String peerId, String remoteWALDir, Collection<String> wals)<a name="line.712"></a>
-<span class="sourceLineNo">713</span> throws IOException {<a name="line.713"></a>
-<span class="sourceLineNo">714</span> Path remoteWALDirForPeer = ReplicationUtils.getPeerRemoteWALDir(remoteWALDir, peerId);<a name="line.714"></a>
-<span class="sourceLineNo">715</span> FileSystem fs = ReplicationUtils.getRemoteWALFileSystem(conf, remoteWALDir);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> for (String wal : wals) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> Path walFile = new Path(remoteWALDirForPeer, wal);<a name="line.717"></a>
-<span class="sourceLineNo">718</span> try {<a name="line.718"></a>
-<span class="sourceLineNo">719</span> if (!fs.delete(walFile, false) && fs.exists(walFile)) {<a name="line.719"></a>
-<span class="sourceLineNo">720</span> throw new IOException("Can not delete " + walFile);<a name="line.720"></a>
-<span class="sourceLineNo">721</span> }<a name="line.721"></a>
-<span class="sourceLineNo">722</span> } catch (FileNotFoundException e) {<a name="line.722"></a>
-<span class="sourceLineNo">723</span> // Just ignore since this means the file has already been deleted.<a name="line.723"></a>
-<span class="sourceLineNo">724</span> // The javadoc of the FileSystem.delete methods does not specify the behavior of deleting an<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // inexistent file, so here we deal with both, i.e, check the return value of the<a name="line.725"></a>
-<span class="sourceLineNo">726</span> // FileSystem.delete, and also catch FNFE.<a name="line.726"></a>
-<span class="sourceLineNo">727</span> LOG.debug("The remote wal {} has already been deleted?", walFile, e);<a name="line.727"></a>
-<span class="sourceLineNo">728</span> }<a name="line.728"></a>
-<span class="sourceLineNo">729</span> }<a name="line.729"></a>
-<span class="sourceLineNo">730</span> }<a name="line.730"></a>
-<span class="sourceLineNo">731</span><a name="line.731"></a>
-<span class="sourceLineNo">732</span> private void cleanOldLogs(NavigableSet<String> wals, ReplicationSourceInterface source) {<a name="line.732"></a>
-<span class="sourceLineNo">733</span> LOG.debug("Removing {} logs in the list: {}", wals.size(), wals);<a name="line.733"></a>
-<span class="sourceLineNo">734</span> // The intention here is that, we want to delete the remote wal files ASAP as it may effect the<a name="line.734"></a>
-<span class="sourceLineNo">735</span> // failover time if you want to transit the remote cluster from S to A. And the infinite retry<a name="line.735"></a>
-<span class="sourceLineNo">736</span> // is not a problem, as if we can not contact with the remote HDFS cluster, then usually we can<a name="line.736"></a>
-<span class="sourceLineNo">737</span> // not contact with the HBase cluster either, so the replication will be blocked either.<a name="line.737"></a>
-<span class="sourceLineNo">738</span> if (source.isSyncReplication()) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span> String peerId = source.getPeerId();<a name="line.739"></a>
-<span class="sourceLineNo">740</span> String remoteWALDir = source.getPeer().getPeerConfig().getRemoteWALDir();<a name="line.740"></a>
-<span class="sourceLineNo">741</span> // Filter out the wals need to be removed from the remote directory. Its name should be the<a name="line.741"></a>
-<span class="sourceLineNo">742</span> // special format, and also, the peer id in its name should match the peer id for the<a name="line.742"></a>
-<span class="sourceLineNo">743</span> // replication source.<a name="line.743"></a>
-<span class="sourceLineNo">744</span> List<String> remoteWals = wals.stream().filter(w -> SyncReplicationWALProvider<a name="line.744"></a>
-<span class="sourceLineNo">745</span> .getSyncReplicationPeerIdFromWALName(w).map(peerId::equals).orElse(false))<a name="line.745"></a>
-<span class="sourceLineNo">746</span> .collect(Collectors.toList());<a name="line.746"></a>
-<span class="sourceLineNo">747</span> LOG.debug("Removing {} logs from remote dir {} in the list: {}", remoteWals.size(),<a name="line.747"></a>
-<span class="sourceLineNo">748</span> remoteWALDir, remoteWals);<a name="line.748"></a>
-<span class="sourceLineNo">749</span> if (!remoteWals.isEmpty()) {<a name="line.749"></a>
-<span class="sourceLineNo">750</span> for (int sleepMultiplier = 0;;) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> try {<a name="line.751"></a>
-<span class="sourceLineNo">752</span> removeRemoteWALs(peerId, remoteWALDir, remoteWals);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> break;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> LOG.warn("Failed to delete remote wals from remote dir {} for peer {}", remoteWALDir,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> peerId);<a name="line.756"></a>
-<span class="sourceLineNo">757</span> }<a name="line.757"></a>
-<span class="sourceLineNo">758</span> if (!source.isSourceActive()) {<a name="line.758"></a>
-<span class="sourceLineNo">759</span> // skip the following operations<a name="line.759"></a>
-<span class="sourceLineNo">760</span> return;<a name="line.760"></a>
-<span class="sourceLineNo">761</span> }<a name="line.761"></a>
-<span class="sourceLineNo">762</span> if (ReplicationUtils.sleepForRetries("Failed to delete remote wals", sleepForRetries,<a name="line.762"></a>
-<span class="sourceLineNo">763</span> sleepMultiplier, maxRetriesMultiplier)) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> sleepMultiplier++;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> }<a name="line.766"></a>
-<span class="sourceLineNo">767</span> }<a name="line.767"></a>
-<span class="sourceLineNo">768</span> }<a name="line.768"></a>
-<span class="sourceLineNo">769</span> String queueId = source.getQueueId();<a name="line.769"></a>
-<span class="sourceLineNo">770</span> for (String wal : wals) {<a name="line.770"></a>
-<span class="sourceLineNo">771</span> interruptOrAbortWhenFail(<a name="line.771"></a>
-<span class="sourceLineNo">772</span> () -> this.queueStorage.removeWAL(server.getServerName(), queueId, wal));<a name="line.772"></a>
-<span class="sourceLineNo">773</span> }<a name="line.773"></a>
-<span class="sourceLineNo">774</span> }<a name="line.774"></a>
-<span class="sourceLineNo">775</span><a name="line.775"></a>
-<span class="sourceLineNo">776</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.776"></a>
-<span class="sourceLineNo">777</span> public void preLogRoll(Path newLog) throws IOException {<a name="line.777"></a>
-<span class="sourceLineNo">778</span> String logName = newLog.getName();<a name="line.778"></a>
-<span class="sourceLineNo">779</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);<a name="line.779"></a>
-<span class="sourceLineNo">780</span> // synchronized on latestPaths to avoid the new open source miss the new log<a name="line.780"></a>
-<span class="sourceLineNo">781</span> synchronized (this.latestPaths) {<a name="line.781"></a>
-<span class="sourceLineNo">782</span> // Add log to queue storage<a name="line.782"></a>
-<span class="sourceLineNo">783</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span> // If record log to queue storage failed, abort RS and throw exception to make log roll<a name="line.784"></a>
-<span class="sourceLineNo">785</span> // failed<a name="line.785"></a>
-<span class="sourceLineNo">786</span> abortAndThrowIOExceptionWhenFail(<a name="line.786"></a>
-<span class="sourceLineNo">787</span> () -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));<a name="line.787"></a>
-<span class="sourceLineNo">788</span> }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span> // synchronized on walsById to avoid race with cleanOldLogs<a name="line.790"></a>
-<span class="sourceLineNo">791</span> synchronized (this.walsById) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span> // Update walsById map<a name="line.792"></a>
-<span class="sourceLineNo">793</span> for (Map.Entry<String, Map<String, NavigableSet<String>>> entry : this.walsById<a name="line.793"></a>
-<span class="sourceLineNo">794</span> .entrySet()) {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> String peerId = entry.getKey();<a name="line.795"></a>
-<span class="sourceLineNo">796</span> Map<String, NavigableSet<String>> walsByPrefix = entry.getValue();<a name="line.796"></a>
-<span class="sourceLineNo">797</span> boolean existingPrefix = false;<a name="line.797"></a>
-<span class="sourceLineNo">798</span> for (Map.Entry<String, NavigableSet<String>> walsEntry : walsByPrefix.entrySet()) {<a name="line.798"></a>
-<span class="sourceLineNo">799</span> SortedSet<String> wals = walsEntry.getValue();<a name="line.799"></a>
-<span class="sourceLineNo">800</span> if (this.sources.isEmpty()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span> // If there's no slaves, don't need to keep the old wals since<a name="line.801"></a>
-<span class="sourceLineNo">802</span> // we only consider the last one when a new slave comes in<a name="line.802"></a>
-<span class="sourceLineNo">803</span> wals.clear();<a name="line.803"></a>
-<span class="sourceLineNo">804</span> }<a name="line.804"></a>
-<span class="sourceLineNo">805</span> if (logPrefix.equals(walsEntry.getKey())) {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> wals.add(logName);<a name="line.806"></a>
-<span class="sourceLineNo">807</span> existingPrefix = true;<a name="line.807"></a>
-<span class="sourceLineNo">808</span> }<a name="line.808"></a>
-<span class="sourceLineNo">809</span> }<a name="line.809"></a>
-<span class="sourceLineNo">810</span> if (!existingPrefix) {<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // The new log belongs to a new group, add it into this peer<a name="line.811"></a>
-<span class="sourceLineNo">812</span> LOG.debug("Start tracking logs for wal group {} for peer {}", logPrefix, peerId);<a name="line.812"></a>
-<span class="sourceLineNo">813</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.813"></a>
-<span class="sourceLineNo">814</span> wals.add(logName);<a name="line.814"></a>
-<span class="sourceLineNo">815</span> walsByPrefix.put(logPrefix, wals);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span> }<a name="line.817"></a>
-<span class="sourceLineNo">818</span> }<a name="line.818"></a>
-<span class="sourceLineNo">819</span><a name="line.819"></a>
-<span class="sourceLineNo">820</span> // Add to latestPaths<a name="line.820"></a>
-<span class="sourceLineNo">821</span> latestPaths.put(logPrefix, newLog);<a name="line.821"></a>
-<span class="sourceLineNo">822</span> }<a name="line.822"></a>
-<span class="sourceLineNo">823</span> }<a name="line.823"></a>
-<span class="sourceLineNo">824</span><a name="line.824"></a>
-<span class="sourceLineNo">825</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.825"></a>
-<span class="sourceLineNo">826</span> public void postLogRoll(Path newLog) throws IOException {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> // This only updates the sources we own, not the recovered ones<a name="line.827"></a>
-<span class="sourceLineNo">828</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.828"></a>
-<span class="sourceLineNo">829</span> source.enqueueLog(newLog);<a name="line.829"></a>
-<span class="sourceLineNo">830</span> LOG.trace("Enqueued {} to source {} while performing postLogRoll operation.",<a name="line.830"></a>
-<span class="sourceLineNo">831</span> newLog, source.getQueueId());<a name="line.831"></a>
-<span class="sourceLineNo">832</span> }<a name="line.832"></a>
-<span class="sourceLineNo">833</span> }<a name="line.833"></a>
-<span class="sourceLineNo">834</span><a name="line.834"></a>
-<span class="sourceLineNo">835</span> @Override<a name="line.835"></a>
-<span class="sourceLineNo">836</span> public void regionServerRemoved(String regionserver) {<a name="line.836"></a>
-<span class="sourceLineNo">837</span> transferQueues(ServerName.valueOf(regionserver));<a name="line.837"></a>
-<span class="sourceLineNo">838</span> }<a name="line.838"></a>
-<span class="sourceLineNo">839</span><a name="line.839"></a>
-<span class="sourceLineNo">840</span> /**<a name="line.840"></a>
-<span class="sourceLineNo">841</span> * Transfer all the queues of the specified to this region server. First it tries to grab a lock<a name="line.841"></a>
-<span class="sourceLineNo">842</span> * and if it works it will move the old queues and finally will delete the old queues.<a name="line.842"></a>
-<span class="sourceLineNo">843</span> * <p><a name="line.843"></a>
-<span class="sourceLineNo">844</span> * It creates one old source for any type of source of the old rs.<a name="line.844"></a>
-<span class="sourceLineNo">845</span> */<a name="line.845"></a>
-<span class="sourceLineNo">846</span> private void transferQueues(ServerName deadRS) {<a name="line.846"></a>
-<span class="sourceLineNo">847</span> if (server.getServerName().equals(deadRS)) {<a name="line.847"></a>
-<span class="sourceLineNo">848</span> // it's just us, give up<a name="line.848"></a>
-<span class="sourceLineNo">849</span> return;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> }<a name="line.850"></a>
-<span class="sourceLineNo">851</span> NodeFailoverWorker transfer = new NodeFailoverWorker(deadRS);<a name="line.851"></a>
-<span class="sourceLineNo">852</span> try {<a name="line.852"></a>
-<span class="sourceLineNo">853</span> this.executor.execute(transfer);<a name="line.853"></a>
-<span class="sourceLineNo">854</span> } catch (RejectedExecutionException ex) {<a name="line.854"></a>
-<span class="sourceLineNo">855</span> CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)<a name="line.855"></a>
-<span class="sourceLineNo">856</span> .getGlobalSource().incrFailedRecoveryQueue();<a name="line.856"></a>
-<span class="sourceLineNo">857</span> LOG.info("Cancelling the transfer of " + deadRS + " because of " + ex.getMessage());<a name="line.857"></a>
-<span class="sourceLineNo">858</span> }<a name="line.858"></a>
-<span class="sourceLineNo">859</span> }<a name="line.859"></a>
-<span class="sourceLineNo">860</span><a name="line.860"></a>
-<span class="sourceLineNo">861</span> /**<a name="line.861"></a>
-<span class="sourceLineNo">862</span> * Class responsible to setup new ReplicationSources to take care of the queues from dead region<a name="line.862"></a>
-<span class="sourceLineNo">863</span> * servers.<a name="line.863"></a>
-<span class="sourceLineNo">864</span> */<a name="line.864"></a>
-<span class="sourceLineNo">865</span> class NodeFailoverWorker extends Thread {<a name="line.865"></a>
-<span class="sourceLineNo">866</span><a name="line.866"></a>
-<span class="sourceLineNo">867</span> private final ServerName deadRS;<a name="line.867"></a>
-<span class="sourceLineNo">868</span> // After claim the queues from dead region server, the NodeFailoverWorker will skip to start<a name="line.868"></a>
-<span class="sourceLineNo">869</span> // the RecoveredReplicationSource if the peer has been removed. but there's possible that<a name="line.869"></a>
-<span class="sourceLineNo">870</span> // remove a peer with peerId = 2 and add a peer with peerId = 2 again during the<a name="line.870"></a>
-<span class="sourceLineNo">871</span> // NodeFailoverWorker. So we need a deep copied <peerId, peer> map to decide whether we<a name="line.871"></a>
-<span class="sourceLineNo">872</span> // should start the RecoveredReplicationSource. If the latest peer is not the old peer when<a name="line.872"></a>
-<span class="sourceLineNo">873</span> // NodeFailoverWorker begin, we should skip to start the RecoveredReplicationSource, Otherwise<a name="line.873"></a>
-<span class="sourceLineNo">874</span> // the rs will abort (See HBASE-20475).<a name="line.874"></a>
-<span class="sourceLineNo">875</span> private final Map<String, ReplicationPeerImpl> peersSnapshot;<a name="line.875"></a>
-<span class="sourceLineNo">876</span><a name="line.876"></a>
-<span class="sourceLineNo">877</span> public NodeFailoverWorker(ServerName deadRS) {<a name="line.877"></a>
-<span class="sourceLineNo">878</span> super("Failover-for-" + deadRS);<a name="line.878"></a>
-<span class="sourceLineNo">879</span> this.deadRS = deadRS;<a name="line.879"></a>
-<span class="sourceLineNo">880</span> peersSnapshot = new HashMap<>(replicationPeers.getPeerCache());<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span><a name="line.882"></a>
-<span class="sourceLineNo">883</span> private boolean isOldPeer(String peerId, ReplicationPeerImpl newPeerRef) {<a name="line.883"></a>
-<span class="sourceLineNo">884</span> ReplicationPeerImpl oldPeerRef = peersSnapshot.get(peerId);<a name="line.884"></a>
-<span class="sourceLineNo">885</span> return oldPeerRef != null && oldPeerRef == newPeerRef;<a name="line.885"></a>
-<span class="sourceLineNo">886</span> }<a name="line.886"></a>
-<span class="sourceLineNo">887</span><a name="line.887"></a>
-<span class="sourceLineNo">888</span> @Override<a name="line.888"></a>
-<span class="sourceLineNo">889</span> public void run() {<a name="line.889"></a>
-<span class="sourceLineNo">890</span> // Wait a bit before transferring the queues, we may be shutting down.<a name="line.890"></a>
-<span class="sourceLineNo">891</span> // This sleep may not be enough in some cases.<a name="line.891"></a>
-<span class="sourceLineNo">892</span> try {<a name="line.892"></a>
-<span class="sourceLineNo">893</span> Thread.sleep(sleepBeforeFailover +<a name="line.893"></a>
-<span class="sourceLineNo">894</span> (long) (ThreadLocalRandom.current().nextFloat() * sleepBeforeFailover));<a name="line.894"></a>
-<span class="sourceLineNo">895</span> } catch (InterruptedException e) {<a name="line.895"></a>
-<span class="sourceLineNo">896</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.896"></a>
-<span class="sourceLineNo">897</span> Thread.currentThread().interrupt();<a name="line.897"></a>
-<span class="sourceLineNo">898</span> }<a name="line.898"></a>
-<span class="sourceLineNo">899</span> // We try to lock that rs' queue directory<a name="line.899"></a>
-<span class="sourceLineNo">900</span> if (server.isStopped()) {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> LOG.info("Not transferring queue since we are shutting down");<a name="line.901"></a>
-<span class="sourceLineNo">902</span> return;<a name="line.902"></a>
-<span class="sourceLineNo">903</span> }<a name="line.903"></a>
-<span class="sourceLineNo">904</span> Map<String, Set<String>> newQueues = new HashMap<>();<a name="line.904"></a>
-<span class="sourceLineNo">905</span> try {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> List<String> queues = queueStorage.getAllQueues(deadRS);<a name="line.906"></a>
-<span class="sourceLineNo">907</span> while (!queues.isEmpty()) {<a name="line.907"></a>
-<span class="sourceLineNo">908</span> Pair<String, SortedSet<String>> peer = queueStorage.claimQueue(deadRS,<a name="line.908"></a>
-<span class="sourceLineNo">909</span> queues.get(ThreadLocalRandom.current().nextInt(queues.size())), server.getServerName());<a name="line.909"></a>
-<span class="sourceLineNo">910</span> long sleep = sleepBeforeFailover / 2;<a name="line.910"></a>
-<span class="sourceLineNo">911</span> if (!peer.getSecond().isEmpty()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> newQueues.put(peer.getFirst(), peer.getSecond());<a name="line.912"></a>
-<span class="sourceLineNo">913</span> sleep = sleepBeforeFailover;<a name="line.913"></a>
-<span class="sourceLineNo">914</span> }<a name="line.914"></a>
-<span class="sourceLineNo">915</span> try {<a name="line.915"></a>
-<span class="sourceLineNo">916</span> Thread.sleep(sleep);<a name="line.916"></a>
-<span class="sourceLineNo">917</span> } catch (InterruptedException e) {<a name="line.917"></a>
-<span class="sourceLineNo">918</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.918"></a>
-<span class="sourceLineNo">919</span> Thread.currentThread().interrupt();<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span> queues = queueStorage.getAllQueues(deadRS);<a name="line.921"></a>
-<span class="sourceLineNo">922</span> }<a name="line.922"></a>
-<span class="sourceLineNo">923</span> if (queues.isEmpty()) {<a name="line.923"></a>
-<span class="sourceLineNo">924</span> queueStorage.removeReplicatorIfQueueIsEmpty(deadRS);<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span> } catch (ReplicationException e) {<a name="line.926"></a>
-<span class="sourceLineNo">927</span> LOG.error(String.format("ReplicationException: cannot claim dead region (%s)'s " +<a name="line.927"></a>
-<span class="sourceLineNo">928</span> "replication queue. Znode : (%s)" +<a name="line.928"></a>
-<span class="sourceLineNo">929</span> " Possible solution: check if znode size exceeds jute.maxBuffer value. " +<a name="line.929"></a>
-<span class="sourceLineNo">930</span> " If so, increase it for both client and server side." + e), deadRS,<a name="line.930"></a>
-<span class="sourceLineNo">931</span> queueStorage.getRsNode(deadRS));<a name="line.931"></a>
-<span class="sourceLineNo">932</span> server.abort("Failed to claim queue from dead regionserver.", e);<a name="line.932"></a>
-<span class="sourceLineNo">933</span> return;<a name="line.933"></a>
-<span class="sourceLineNo">934</span> }<a name="line.934"></a>
-<span class="sourceLineNo">935</span> // Copying over the failed queue is completed.<a name="line.935"></a>
-<span class="sourceLineNo">936</span> if (newQueues.isEmpty()) {<a name="line.936"></a>
-<span class="sourceLineNo">937</span> // We either didn't get the lock or the failed region server didn't have any outstanding<a name="line.937"></a>
-<span class="sourceLineNo">938</span> // WALs to replicate, so we are done.<a name="line.938"></a>
-<span class="sourceLineNo">939</span> return;<a name="line.939"></a>
-<span class="sourceLineNo">940</span> }<a name="line.940"></a>
-<span class="sourceLineNo">941</span><a name="line.941"></a>
-<span class="sourceLineNo">942</span> for (Map.Entry<String, Set<String>> entry : newQueues.entrySet()) {<a name="line.942"></a>
-<span class="sourceLineNo">943</span> String queueId = entry.getKey();<a name="line.943"></a>
-<span class="sourceLineNo">944</span> Set<String> walsSet = entry.getValue();<a name="line.944"></a>
-<span class="sourceLineNo">945</span> try {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> // there is not an actual peer defined corresponding to peerId for the failover.<a name="line.946"></a>
-<span class="sourceLineNo">947</span> ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);<a name="line.947"></a>
-<span class="sourceLineNo">948</span> String actualPeerId = replicationQueueInfo.getPeerId();<a name="line.948"></a>
-<span class="sourceLineNo">949</span><a name="line.949"></a>
-<span class="sourceLineNo">950</span> ReplicationPeerImpl peer = replicationPeers.getPeer(actualPeerId);<a name="line.950"></a>
-<span class="sourceLineNo">951</span> if (peer == null || !isOldPeer(actualPeerId, peer)) {<a name="line.951"></a>
-<span class="sourceLineNo">952</span> LOG.warn("Skipping failover for peer {} of node {}, peer is null", actualPeerId,<a name="line.952"></a>
-<span class="sourceLineNo">953</span> deadRS);<a name="line.953"></a>
-<span class="sourceLineNo">954</span> abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.954"></a>
-<span class="sourceLineNo">955</span> continue;<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> if (server instanceof ReplicationSyncUp.DummyServer<a name="line.957"></a>
-<span class="sourceLineNo">958</span> && peer.getPeerState().equals(PeerState.DISABLED)) {<a name="line.958"></a>
-<span class="sourceLineNo">959</span> LOG.warn("Peer {} is disabled. ReplicationSyncUp tool will skip "<a name="line.959"></a>
-<span class="sourceLineNo">960</span> + "replicating data to this peer.",<a name="line.960"></a>
-<span class="sourceLineNo">961</span> actualPeerId);<a name="line.961"></a>
-<span class="sourceLineNo">962</span> continue;<a name="line.962"></a>
-<span class="sourceLineNo">963</span> }<a name="line.963"></a>
-<span class="sourceLineNo">964</span><a name="line.964"></a>
-<span class="sourceLineNo">965</span> ReplicationSourceInterface src = createSource(queueId, peer);<a name="line.965"></a>
-<span class="sourceLineNo">966</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.966"></a>
-<span class="sourceLineNo">967</span> synchronized (oldsources) {<a name="line.967"></a>
-<span class="sourceLineNo">968</span> peer = replicationPeers.getPeer(src.getPeerId());<a name="line.968"></a>
-<span class="sourceLineNo">969</span> if (peer == null || !isOldPeer(src.getPeerId(), peer)) {<a name="line.969"></a>
-<span class="sourceLineNo">970</span> src.terminate("Recovered queue doesn't belong to any current peer");<a name="line.970"></a>
-<span class="sourceLineNo">971</span> deleteQueue(queueId);<a name="line.971"></a>
-<span class="sourceLineNo">972</span> continue;<a name="line.972"></a>
-<span class="sourceLineNo">973</span> }<a name="line.973"></a>
-<span class="sourceLineNo">974</span> // Do not setup recovered queue if a sync replication peer is in STANDBY state, or is<a name="line.974"></a>
-<span class="sourceLineNo">975</span> // transiting to STANDBY state. The only exception is we are in STANDBY state and<a name="line.975"></a>
-<span class="sourceLineNo">976</span> // transiting to DA, under this state we will replay the remote WAL and they need to be<a name="line.976"></a>
-<span class="sourceLineNo">977</span> // replicated back.<a name="line.977"></a>
-<span class="sourceLineNo">978</span> if (peer.getPeerConfig().isSyncReplication()) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> Pair<SyncReplicationState, SyncReplicationState> stateAndNewState =<a name="line.979"></a>
-<span class="sourceLineNo">980</span> peer.getSyncReplicationStateAndNewState();<a name="line.980"></a>
-<span class="sourceLineNo">981</span> if ((stateAndNewState.getFirst().equals(SyncReplicationState.STANDBY) &&<a name="line.981"></a>
-<span class="sourceLineNo">982</span> stateAndNewState.getSecond().equals(SyncReplicationState.NONE)) ||<a name="line.982"></a>
-<span class="sourceLineNo">983</span> stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)) {<a name="line.983"></a>
-<span class="sourceLineNo">984</span> src.terminate("Sync replication peer is in STANDBY state");<a name="line.984"></a>
-<span class="sourceLineNo">985</span> deleteQueue(queueId);<a name="line.985"></a>
-<span class="sourceLineNo">986</span> continue;<a name="line.986"></a>
-<span class="sourceLineNo">987</span> }<a name="line.987"></a>
-<span class="sourceLineNo">988</span> }<a name="line.988"></a>
-<span class="sourceLineNo">989</span> // track sources in walsByIdRecoveredQueues<a name="line.989"></a>
-<span class="sourceLineNo">990</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.990"></a>
-<span class="sourceLineNo">991</span> walsByIdRecoveredQueues.put(queueId, walsByGroup);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> for (String wal : walsSet) {<a name="line.992"></a>
-<span class="sourceLineNo">993</span> String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal);<a name="line.993"></a>
-<span class="sourceLineNo">994</span> NavigableSet<String> wals = walsByGroup.get(walPrefix);<a name="line.994"></a>
-<span class="sourceLineNo">995</span> if (wals == null) {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> wals = new TreeSet<>();<a name="line.996"></a>
-<span class="sourceLineNo">997</span> walsByGroup.put(walPrefix, wals);<a name="line.997"></a>
-<span class="sourceLineNo">998</span> }<a name="line.998"></a>
-<span class="sourceLineNo">999</span> wals.add(wal);<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> }<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> oldsources.add(src);<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span> LOG.info("Added source for recovered queue {}", src.getQueueId());<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> for (String wal : walsSet) {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> LOG.trace("Enqueueing log from recovered queue for source: " + src.getQueueId());<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> src.enqueueLog(new Path(oldLogDir, wal));<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span> }<a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> src.startup();<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> }<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> } catch (IOException e) {<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> // TODO manage it<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span> LOG.error("Failed creating a source", e);<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> }<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> }<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> }<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span><a name="line.1016"></a>
-<span class="sourceLineNo">1017</span> /**<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> * Terminate the replication on this region server<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> */<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> public void join() {<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.executor.shutdown();<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> source.terminate("Region server is closing");<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span> }<a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> }<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span><a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> /**<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> * Get a copy of the wals of the normal sources on this rs<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> * @return a sorted set of wal names<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> */<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> public Map<String, Map<String, NavigableSet<String>>> getWALs() {<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> return Collections.unmodifiableMap(walsById);<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> /**<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> * Get a copy of the wals of the recovered sources on this rs<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> * @return a sorted set of wal names<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> */<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> Map<String, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> return Collections.unmodifiableMap(walsByIdRecoveredQueues);<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> }<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span><a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> /**<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> * Get a list of all the normal sources of this rs<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> * @return list of all normal sources<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> */<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> public List<ReplicationSourceInterface> getSources() {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> return new ArrayList<>(this.sources.values());<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> }<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span><a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> /**<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> * Get a list of all the recovered sources of this rs<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> * @return list of all recovered sources<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> */<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span> public List<ReplicationSourceInterface> getOldSources() {<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> return this.oldsources;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> }<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span><a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> /**<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> * Get the normal source for a given peer<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> * @return the normal source for the give peer if it exists, otherwise null.<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> */<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> public ReplicationSourceInterface getSource(String peerId) {<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span> return this.sources.get(peerId);<a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> }<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span><a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> List<String> getAllQueues() throws IOException {<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span> List<String> allQueues = Collections.emptyList();<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> try {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> allQueues = queueStorage.getAllQueues(server.getServerName());<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> } catch (ReplicationException e) {<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> throw new IOException(e);<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> return allQueues;<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> int getSizeOfLatestPath() {<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> synchronized (latestPaths) {<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> return latestPaths.size();<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> }<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> }<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> Set<Path> getLastestPath() {<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> synchronized (latestPaths) {<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span> return Sets.newHashSet(latestPaths.values());<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> }<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> }<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span><a name="line.1088"></a>
-<span class="sourceLineNo">1089</span> public AtomicLong getTotalBufferUsed() {<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span> return totalBufferUsed;<a name="line.1090"></a>
-<span class="sourceLineNo">1091</span> }<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span><a name="line.1092"></a>
-<span class="sourceLineNo">1093</span> /**<a name="line.1093"></a>
-<span class="sourceLineNo">1094</span> * Returns the maximum size in bytes of edits held in memory which are pending replication<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span> * across all sources inside this RegionServer.<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span> */<a name="line.1096"></a>
-<span class="sourceLineNo">1097</span> public long getTotalBufferLimit() {<a name="line.1097"></a>
-<span class="sourceLineNo">1098</span> return totalBufferLimit;<a name="line.1098"></a>
-<span class="sourceLineNo">1099</span> }<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span><a name="line.1100"></a>
-<span class="sourceLineNo">1101</span> /**<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span> * Get the directory where wals are archived<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span> * @return the directory where wals are archived<a name="line.1103"></a>
-<span class="sourceLineNo">1104</span> */<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span> public Path getOldLogDir() {<a name="line.1105"></a>
-<span class="sourceLineNo">1106</span> return this.oldLogDir;<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span> }<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span><a name="line.1108"></a>
-<span class="sourceLineNo">1109</span> /**<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span> * Get the directory where wals are stored by their RSs<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span> * @return the directory where wals are stored by their RSs<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span> */<a name="line.1112"></a>
-<span class="sourceLineNo">1113</span> public Path getLogDir() {<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span> return this.logDir;<a name="line.1114"></a>
-<span class="sourceLineNo">1115</span> }<a name="line.1115"></a>
-<span class="sourceLineNo">1116</span><a name="line.1116"></a>
-<span class="sourceLineNo">1117</span> /**<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span> * Get the handle on the local file system<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span> * @return Handle on the local file system<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span> */<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span> public FileSystem getFs() {<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span> return this.fs;<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span> }<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span><a name="line.1124"></a>
-<span class="sourceLineNo">1125</span> /**<a name="line.1125"></a>
-<span class="sourceLineNo">1126</span> * Get the ReplicationPeers used by this ReplicationSourceManager<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span> * @return the ReplicationPeers used by this ReplicationSourceManager<a name="line.1127"></a>
-<span class="sourceLineNo">1128</span> */<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span> public ReplicationPeers getReplicationPeers() {<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span> return this.replicationPeers;<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span> }<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span><a name="line.1132"></a>
-<span class="sourceLineNo">1133</span> /**<a name="line.1133"></a>
-<span class="sourceLineNo">1134</span> * Get a string representation of all the sources' metrics<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span> */<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span> public String getStats() {<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span> StringBuilder stats = new StringBuilder();<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span> // Print stats that apply across all Replication Sources<a name="line.1138"></a>
-<span class="sourceLineNo">1139</span> stats.append("Global stats: ");<a name="line.1139"></a>
-<span class="sourceLineNo">1140</span> stats.append("WAL Edits Buffer Used=").append(getTotalBufferUsed().get()).append("B, Limit=")<a name="line.1140"></a>
-<span class="sourceLineNo">1141</span> .append(getTotalBufferLimit()).append("B\n");<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span> stats.append("Normal source for cluster " + source.getPeerId() + ": ");<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span> stats.append(source.getStats() + "\n");<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span> }<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span> for (ReplicationSourceInterface oldSource : oldsources) {<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span> stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerId() + ": ");<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span> stats.append(oldSource.getStats() + "\n");<a name="line.1148"></a>
-<span class="sourceLineNo">1149</span> }<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span> return stats.toString();<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span> }<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span><a name="line.1152"></a>
-<span class="sourceLineNo">1153</span> public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span> throws IOException {<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span> throwIOExceptionWhenFail(() -> source.addHFileRefs(tableName, family, pairs));<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span> }<a name="line.1157"></a>
-<span class="sourceLineNo">1158</span> }<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span><a name="line.1159"></a>
-<span class="sourceLineNo">1160</span> public void cleanUpHFileRefs(String peerId, List<String> files) {<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span> interruptOrAbortWhenFail(() -> this.queueStorage.removeHFileRefs(peerId, files));<a name="line.1161"></a>
-<span class="sourceLineNo">1162</span> }<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span><a name="line.1163"></a>
-<span class="sourceLineNo">1164</span> int activeFailoverTaskCount() {<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span> return executor.getActiveCount();<a name="line.1165"></a>
-<span class="sourceLineNo">1166</span> }<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span><a name="line.1167"></a>
-<span class="sourceLineNo">1168</span> MetricsReplicationGlobalSourceSource getGlobalMetrics() {<a name="line.1168"></a>
-<span class="sourceLineNo">1169</span> return this.globalMetrics;<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span> }<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span><a name="line.1171"></a>
-<span class="sourceLineNo">1172</span> /**<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span> * Add an hbase:meta Catalog replication source. Called on open of an hbase:meta Region.<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span> * Create it once only. If exists already, use the existing one.<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span> * @see #removeCatalogReplicationSource(RegionInfo)<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span> * @see #addSource(String) This is specialization on the addSource method.<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span> */<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span> public ReplicationSourceInterface addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span> throws IOException {<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span> // Poor-man's putIfAbsent<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span> synchronized (this.catalogReplicationSource) {<a name="line.1181"></a>
-<span class="sourceLineNo">1182</span> ReplicationSourceInterface rs = this.catalogReplicationSource.get();<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span> return rs != null ? rs :<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span> this.catalogReplicationSource.getAndSet(createCatalogReplicationSource(regionInfo));<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span> }<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span> }<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span><a name="line.1187"></a>
-<span class="sourceLineNo">1188</span> /**<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span> * Remove the hbase:meta Catalog replication source.<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span> * Called when we close hbase:meta.<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span> * @see #addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span> */<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span> public void removeCatalogReplicationSource(RegionInfo regionInfo) {<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span> // Nothing to do. Leave any CatalogReplicationSource in place in case an hbase:meta Region<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span> // comes back to this server.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span> }<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span><a name="line.1197"></a>
-<span class="sourceLineNo">1198</span> /**<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span> * Create, initialize, and start the Catalog ReplicationSource.<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span> * Presumes called one-time only (caller must ensure one-time only call).<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span> * This ReplicationSource is NOT created via {@link ReplicationSourceFactory}.<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span> * @see #addSource(String) This is a specialization of the addSource call.<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span> * @see #catalogReplicationSource for a note on this ReplicationSource's lifecycle (and more on<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span> * why the special handling).<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span> */<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span> private ReplicationSourceInterface createCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span> throws IOException {<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span> // Instantiate meta walProvider. Instantiated here or over in the #warmupRegion call made by the<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span> // Master on a 'move' operation. Need to do extra work if we did NOT instantiate the provider.<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span> WALProvider walProvider = this.walFactory.getMetaWALProvider();<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span> boolean instantiate = walProvider == null;<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span> if (instantiate) {<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span> walProvider = this.walFactory.getMetaProvider();<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span> }<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span> // Here we do a specialization on what {@link ReplicationSourceFactory} does. There is no need<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span> // for persisting offset into WALs up in zookeeper (via ReplicationQueueInfo) as the catalog<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span> // read replicas feature that makes use of the source does a reset on a crash of the WAL<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span> // source process. See "4.1 Skip maintaining zookeeper replication queue (offsets/WALs)" in the<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span> // design doc attached to HBASE-18070 'Enable memstore replication for meta replica' for detail.<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span> CatalogReplicationSourcePeer peer = new CatalogReplicationSourcePeer(this.conf,<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span> this.clusterId.toString(), "meta_" + ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER);<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span> final ReplicationSourceInterface crs = new CatalogReplicationSource();<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span> crs.init(conf, fs, this, new NoopReplicationQueueStorage(), peer, server, peer.getId(),<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span> clusterId, walProvider.getWALFileLengthProvider(), new MetricsSource(peer.getId()));<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span> // Add listener on the provider so we can pick up the WAL to replicate on roll.<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span> WALActionsListener listener = new WALActionsListener() {<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span> @Override public void postLogRoll(Path oldPath, Path newPath) throws IOException {<a name="line.1227"></a>
-<span class="sourceLineNo">1228</span> crs.enqueueLog(newPath);<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span> }<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span> };<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span> walProvider.addWALActionsListener(listener);<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span> if (!instantiate) {<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span> // If we did not instantiate provider, need to add our listener on already-created WAL<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span> // instance too (listeners are passed by provider to WAL instance on creation but if provider<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span> // created already, our listener add above is missed). And add the current WAL file to the<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span> // Replication Source so it can start replicating it.<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span> WAL wal = walProvider.getWAL(regionInfo);<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span> wal.registerWALActionsListener(listener);<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span> crs.enqueueLog(((AbstractFSWAL)wal).getCurrentFileName());<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span> }<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span> return crs.startup();<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span> }<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>}<a name="line.1243"></a>
+<span class="sourceLineNo">286</span> List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers();<a name="line.286"></a>
+<span class="sourceLineNo">287</span> LOG.info(<a name="line.287"></a>
+<span class="sourceLineNo">288</span> "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);<a name="line.288"></a>
+<span class="sourceLineNo">289</span><a name="line.289"></a>
+<span class="sourceLineNo">290</span> // Look if there's anything to process after a restart<a name="line.290"></a>
+<span class="sourceLineNo">291</span> for (ServerName rs : currentReplicators) {<a name="line.291"></a>
+<span class="sourceLineNo">292</span> if (!otherRegionServers.contains(rs)) {<a name="line.292"></a>
+<span class="sourceLineNo">293</span> transferQueues(rs);<a name="line.293"></a>
+<span class="sourceLineNo">294</span> }<a name="line.294"></a>
+<span class="sourceLineNo">295</span> }<a name="line.295"></a>
+<span class="sourceLineNo">296</span> }<a name="line.296"></a>
+<span class="sourceLineNo">297</span><a name="line.297"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * <ol><a name="line.299"></a>
+<span class="sourceLineNo">300</span> * <li>Add peer to replicationPeers</li><a name="line.300"></a>
+<span class="sourceLineNo">301</span> * <li>Add the normal source and related replication queue</li><a name="line.301"></a>
+<span class="sourceLineNo">302</span> * <li>Add HFile Refs</li><a name="line.302"></a>
+<span class="sourceLineNo">303</span> * </ol><a name="line.303"></a>
+<span class="sourceLineNo">304</span> * @param peerId the id of replication peer<a name="line.304"></a>
+<span class="sourceLineNo">305</span> */<a name="line.305"></a>
+<span class="sourceLineNo">306</span> public void addPeer(String peerId) throws IOException {<a name="line.306"></a>
+<span class="sourceLineNo">307</span> boolean added = false;<a name="line.307"></a>
+<span class="sourceLineNo">308</span> try {<a name="line.308"></a>
+<span class="sourceLineNo">309</span> added = this.replicationPeers.addPeer(peerId);<a name="line.309"></a>
+<span class="sourceLineNo">310</span> } catch (ReplicationException e) {<a name="line.310"></a>
+<span class="sourceLineNo">311</span> throw new IOException(e);<a name="line.311"></a>
+<span class="sourceLineNo">312</span> }<a name="line.312"></a>
+<span class="sourceLineNo">313</span> if (added) {<a name="line.313"></a>
+<span class="sourceLineNo">314</span> addSource(peerId);<a name="line.314"></a>
+<span class="sourceLineNo">315</span> if (replicationForBulkLoadDataEnabled) {<a name="line.315"></a>
+<span class="sourceLineNo">316</span> throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));<a name="line.316"></a>
+<span class="sourceLineNo">317</span> }<a name="line.317"></a>
+<span class="sourceLineNo">318</span> }<a name="line.318"></a>
+<span class="sourceLineNo">319</span> }<a name="line.319"></a>
+<span class="sourceLineNo">320</span><a name="line.320"></a>
+<span class="sourceLineNo">321</span> /**<a name="line.321"></a>
+<span class="sourceLineNo">322</span> * <ol><a name="line.322"></a>
+<span class="sourceLineNo">323</span> * <li>Remove peer for replicationPeers</li><a name="line.323"></a>
+<span class="sourceLineNo">324</span> * <li>Remove all the recovered sources for the specified id and related replication queues</li><a name="line.324"></a>
+<span class="sourceLineNo">325</span> * <li>Remove the normal source and related replication queue</li><a name="line.325"></a>
+<span class="sourceLineNo">326</span> * <li>Remove HFile Refs</li><a name="line.326"></a>
+<span class="sourceLineNo">327</span> * </ol><a name="line.327"></a>
+<span class="sourceLineNo">328</span> * @param peerId the id of the replication peer<a name="line.328"></a>
+<span class="sourceLineNo">329</span> */<a name="line.329"></a>
+<span class="sourceLineNo">330</span> public void removePeer(String peerId) {<a name="line.330"></a>
+<span class="sourceLineNo">331</span> ReplicationPeer peer = replicationPeers.removePeer(peerId);<a name="line.331"></a>
+<span class="sourceLineNo">332</span> String terminateMessage = "Replication stream was removed by a user";<a name="line.332"></a>
+<span class="sourceLineNo">333</span> List<ReplicationSourceInterface> oldSourcesToDelete = new ArrayList<>();<a name="line.333"></a>
+<span class="sourceLineNo">334</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.334"></a>
+<span class="sourceLineNo">335</span> // see NodeFailoverWorker.run<a name="line.335"></a>
+<span class="sourceLineNo">336</span> synchronized (this.oldsources) {<a name="line.336"></a>
+<span class="sourceLineNo">337</span> // First close all the recovered sources for this peer<a name="line.337"></a>
+<span class="sourceLineNo">338</span> for (ReplicationSourceInterface src : oldsources) {<a name="line.338"></a>
+<span class="sourceLineNo">339</span> if (peerId.equals(src.getPeerId())) {<a name="line.339"></a>
+<span class="sourceLineNo">340</span> oldSourcesToDelete.add(src);<a name="line.340"></a>
+<span class="sourceLineNo">341</span> }<a name="line.341"></a>
+<span class="sourceLineNo">342</span> }<a name="line.342"></a>
+<span class="sourceLineNo">343</span> for (ReplicationSourceInterface src : oldSourcesToDelete) {<a name="line.343"></a>
+<span class="sourceLineNo">344</span> src.terminate(terminateMessage);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> removeRecoveredSource(src);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> }<a name="line.346"></a>
+<span class="sourceLineNo">347</span> }<a name="line.347"></a>
+<span class="sourceLineNo">348</span> LOG.info(<a name="line.348"></a>
+<span class="sourceLineNo">349</span> "Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());<a name="line.349"></a>
+<span class="sourceLineNo">350</span> // Now close the normal source for this peer<a name="line.350"></a>
+<span class="sourceLineNo">351</span> ReplicationSourceInterface srcToRemove = this.sources.get(peerId);<a name="line.351"></a>
+<span class="sourceLineNo">352</span> if (srcToRemove != null) {<a name="line.352"></a>
+<span class="sourceLineNo">353</span> srcToRemove.terminate(terminateMessage);<a name="line.353"></a>
+<span class="sourceLineNo">354</span> removeSource(srcToRemove);<a name="line.354"></a>
+<span class="sourceLineNo">355</span> } else {<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // Delete queue from storage and memory and queue id is same with peer id for normal<a name="line.357"></a>
+<span class="sourceLineNo">358</span> // source<a name="line.358"></a>
+<span class="sourceLineNo">359</span> deleteQueue(peerId);<a name="line.359"></a>
+<span class="sourceLineNo">360</span> this.walsById.remove(peerId);<a name="line.360"></a>
+<span class="sourceLineNo">361</span> }<a name="line.361"></a>
+<span class="sourceLineNo">362</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.362"></a>
+<span class="sourceLineNo">363</span> if (peerConfig.isSyncReplication()) {<a name="line.363"></a>
+<span class="sourceLineNo">364</span> syncReplicationPeerMappingManager.remove(peerId, peerConfig);<a name="line.364"></a>
+<span class="sourceLineNo">365</span> }<a name="line.365"></a>
+<span class="sourceLineNo">366</span> // Remove HFile Refs<a name="line.366"></a>
+<span class="sourceLineNo">367</span> abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));<a name="line.367"></a>
+<span class="sourceLineNo">368</span> }<a name="line.368"></a>
+<span class="sourceLineNo">369</span><a name="line.369"></a>
+<span class="sourceLineNo">370</span> /**<a name="line.370"></a>
+<span class="sourceLineNo">371</span> * @return a new 'classic' user-space replication source.<a name="line.371"></a>
+<span class="sourceLineNo">372</span> * @param queueId the id of the replication queue to associate the ReplicationSource with.<a name="line.372"></a>
+<span class="sourceLineNo">373</span> * @see #createCatalogReplicationSource(RegionInfo) for creating a ReplicationSource for meta.<a name="line.373"></a>
+<span class="sourceLineNo">374</span> */<a name="line.374"></a>
+<span class="sourceLineNo">375</span> private ReplicationSourceInterface createSource(String queueId, ReplicationPeer replicationPeer)<a name="line.375"></a>
+<span class="sourceLineNo">376</span> throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueId);<a name="line.377"></a>
+<span class="sourceLineNo">378</span> // Init the just created replication source. Pass the default walProvider's wal file length<a name="line.378"></a>
+<span class="sourceLineNo">379</span> // provider. Presumption is we replicate user-space Tables only. For hbase:meta region replica<a name="line.379"></a>
+<span class="sourceLineNo">380</span> // replication, see #createCatalogReplicationSource().<a name="line.380"></a>
+<span class="sourceLineNo">381</span> WALFileLengthProvider walFileLengthProvider =<a name="line.381"></a>
+<span class="sourceLineNo">382</span> this.walFactory.getWALProvider() != null?<a name="line.382"></a>
+<span class="sourceLineNo">383</span> this.walFactory.getWALProvider().getWALFileLengthProvider() : p -> OptionalLong.empty();<a name="line.383"></a>
+<span class="sourceLineNo">384</span> src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId,<a name="line.384"></a>
+<span class="sourceLineNo">385</span> walFileLengthProvider, new MetricsSource(queueId));<a name="line.385"></a>
+<span class="sourceLineNo">386</span> return src;<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> /**<a name="line.389"></a>
+<span class="sourceLineNo">390</span> * Add a normal source for the given peer on this region server. Meanwhile, add new replication<a name="line.390"></a>
+<span class="sourceLineNo">391</span> * queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal<a name="line.391"></a>
+<span class="sourceLineNo">392</span> * group and do replication<a name="line.392"></a>
+<span class="sourceLineNo">393</span> * @param peerId the id of the replication peer<a name="line.393"></a>
+<span class="sourceLineNo">394</span> * @return the source that was created<a name="line.394"></a>
+<span class="sourceLineNo">395</span> */<a name="line.395"></a>
+<span class="sourceLineNo">396</span> ReplicationSourceInterface addSource(String peerId) throws IOException {<a name="line.396"></a>
+<span class="sourceLineNo">397</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.397"></a>
+<span class="sourceLineNo">398</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.398"></a>
+<span class="sourceLineNo">399</span> // synchronized on latestPaths to avoid missing the new log<a name="line.399"></a>
+<span class="sourceLineNo">400</span> synchronized (this.latestPaths) {<a name="line.400"></a>
+<span class="sourceLineNo">401</span> this.sources.put(peerId, src);<a name="line.401"></a>
+<span class="sourceLineNo">402</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.402"></a>
+<span class="sourceLineNo">403</span> this.walsById.put(peerId, walsByGroup);<a name="line.403"></a>
+<span class="sourceLineNo">404</span> // Add the latest wal to that source's queue<a name="line.404"></a>
+<span class="sourceLineNo">405</span> if (!latestPaths.isEmpty()) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span> for (Map.Entry<String, Path> walPrefixAndPath : latestPaths.entrySet()) {<a name="line.406"></a>
+<span class="sourceLineNo">407</span> Path walPath = walPrefixAndPath.getValue();<a name="line.407"></a>
+<span class="sourceLineNo">408</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.408"></a>
+<span class="sourceLineNo">409</span> wals.add(walPath.getName());<a name="line.409"></a>
+<span class="sourceLineNo">410</span> walsByGroup.put(walPrefixAndPath.getKey(), wals);<a name="line.410"></a>
+<span class="sourceLineNo">411</span> // Abort RS and throw exception to make add peer failed<a name="line.411"></a>
+<span class="sourceLineNo">412</span> abortAndThrowIOExceptionWhenFail(<a name="line.412"></a>
+<span class="sourceLineNo">413</span> () -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName()));<a name="line.413"></a>
+<span class="sourceLineNo">414</span> src.enqueueLog(walPath);<a name="line.414"></a>
+<span class="sourceLineNo">415</span> LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());<a name="line.415"></a>
+<span class="sourceLineNo">416</span> }<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span> ReplicationPeerConfig peerConfig = peer.getPeerConfig();<a name="line.419"></a>
+<span class="sourceLineNo">420</span> if (peerConfig.isSyncReplication()) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span> syncReplicationPeerMappingManager.add(peer.getId(), peerConfig);<a name="line.421"></a>
+<span class="sourceLineNo">422</span> }<a name="line.422"></a>
+<span class="sourceLineNo">423</span> src.startup();<a name="line.423"></a>
+<span class="sourceLineNo">424</span> return src;<a name="line.424"></a>
+<span class="sourceLineNo">425</span> }<a name="line.425"></a>
+<span class="sourceLineNo">426</span><a name="line.426"></a>
+<span class="sourceLineNo">427</span> /**<a name="line.427"></a>
+<span class="sourceLineNo">428</span> * <p><a name="line.428"></a>
+<span class="sourceLineNo">429</span> * This is used when we transit a sync replication peer to {@link SyncReplicationState#STANDBY}.<a name="line.429"></a>
+<span class="sourceLineNo">430</span> * </p><a name="line.430"></a>
+<span class="sourceLineNo">431</span> * <p><a name="line.431"></a>
+<span class="sourceLineNo">432</span> * When transiting to {@link SyncReplicationState#STANDBY}, we can remove all the pending wal<a name="line.432"></a>
+<span class="sourceLineNo">433</span> * files for a replication peer as we do not need to replicate them any more. And this is<a name="line.433"></a>
+<span class="sourceLineNo">434</span> * necessary, otherwise when we transit back to {@link SyncReplicationState#DOWNGRADE_ACTIVE}<a name="line.434"></a>
+<span class="sourceLineNo">435</span> * later, the stale data will be replicated again and cause inconsistency.<a name="line.435"></a>
+<span class="sourceLineNo">436</span> * </p><a name="line.436"></a>
+<span class="sourceLineNo">437</span> * <p><a name="line.437"></a>
+<span class="sourceLineNo">438</span> * See HBASE-20426 for more details.<a name="line.438"></a>
+<span class="sourceLineNo">439</span> * </p><a name="line.439"></a>
+<span class="sourceLineNo">440</span> * @param peerId the id of the sync replication peer<a name="line.440"></a>
+<span class="sourceLineNo">441</span> */<a name="line.441"></a>
+<span class="sourceLineNo">442</span> public void drainSources(String peerId) throws IOException, ReplicationException {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> String terminateMessage = "Sync replication peer " + peerId +<a name="line.443"></a>
+<span class="sourceLineNo">444</span> " is transiting to STANDBY. Will close the previous replication source and open a new one";<a name="line.444"></a>
+<span class="sourceLineNo">445</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.445"></a>
+<span class="sourceLineNo">446</span> assert peer.getPeerConfig().isSyncReplication();<a name="line.446"></a>
+<span class="sourceLineNo">447</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.447"></a>
+<span class="sourceLineNo">448</span> // synchronized here to avoid race with preLogRoll where we add new log to source and also<a name="line.448"></a>
+<span class="sourceLineNo">449</span> // walsById.<a name="line.449"></a>
+<span class="sourceLineNo">450</span> ReplicationSourceInterface toRemove;<a name="line.450"></a>
+<span class="sourceLineNo">451</span> Map<String, NavigableSet<String>> wals = new HashMap<>();<a name="line.451"></a>
+<span class="sourceLineNo">452</span> synchronized (latestPaths) {<a name="line.452"></a>
+<span class="sourceLineNo">453</span> toRemove = sources.put(peerId, src);<a name="line.453"></a>
+<span class="sourceLineNo">454</span> if (toRemove != null) {<a name="line.454"></a>
+<span class="sourceLineNo">455</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.455"></a>
+<span class="sourceLineNo">456</span> toRemove.terminate(terminateMessage);<a name="line.456"></a>
+<span class="sourceLineNo">457</span> toRemove.getSourceMetrics().clear();<a name="line.457"></a>
+<span class="sourceLineNo">458</span> }<a name="line.458"></a>
+<span class="sourceLineNo">459</span> // Here we make a copy of all the remaining wal files and then delete them from the<a name="line.459"></a>
+<span class="sourceLineNo">460</span> // replication queue storage after releasing the lock. It is not safe to just remove the old<a name="line.460"></a>
+<span class="sourceLineNo">461</span> // map from walsById since later we may fail to delete them from the replication queue<a name="line.461"></a>
+<span class="sourceLineNo">462</span> // storage, and when we retry next time, we can not know the wal files that need to be deleted<a name="line.462"></a>
+<span class="sourceLineNo">463</span> // from the replication queue storage.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> walsById.get(peerId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));<a name="line.464"></a>
+<span class="sourceLineNo">465</span> }<a name="line.465"></a>
+<span class="sourceLineNo">466</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.466"></a>
+<span class="sourceLineNo">467</span> src.startup();<a name="line.467"></a>
+<span class="sourceLineNo">468</span> for (NavigableSet<String> walsByGroup : wals.values()) {<a name="line.468"></a>
+<span class="sourceLineNo">469</span> for (String wal : walsByGroup) {<a name="line.469"></a>
+<span class="sourceLineNo">470</span> queueStorage.removeWAL(server.getServerName(), peerId, wal);<a name="line.470"></a>
+<span class="sourceLineNo">471</span> }<a name="line.471"></a>
+<span class="sourceLineNo">472</span> }<a name="line.472"></a>
+<span class="sourceLineNo">473</span> synchronized (walsById) {<a name="line.473"></a>
+<span class="sourceLineNo">474</span> Map<String, NavigableSet<String>> oldWals = walsById.get(peerId);<a name="line.474"></a>
+<span class="sourceLineNo">475</span> wals.forEach((k, v) -> {<a name="line.475"></a>
+<span class="sourceLineNo">476</span> NavigableSet<String> walsByGroup = oldWals.get(k);<a name="line.476"></a>
+<span class="sourceLineNo">477</span> if (walsByGroup != null) {<a name="line.477"></a>
+<span class="sourceLineNo">478</span> walsByGroup.removeAll(v);<a name="line.478"></a>
+<span class="sourceLineNo">479</span> }<a name="line.479"></a>
+<span class="sourceLineNo">480</span> });<a name="line.480"></a>
+<span class="sourceLineNo">481</span> }<a name="line.481"></a>
+<span class="sourceLineNo">482</span> // synchronized on oldsources to avoid race with NodeFailoverWorker. Since NodeFailoverWorker is<a name="line.482"></a>
+<span class="sourceLineNo">483</span> // a background task, we will delete the file from replication queue storage under the lock to<a name="line.483"></a>
+<span class="sourceLineNo">484</span> // simplify the logic.<a name="line.484"></a>
+<span class="sourceLineNo">485</span> synchronized (this.oldsources) {<a name="line.485"></a>
+<span class="sourceLineNo">486</span> for (Iterator<ReplicationSourceInterface> iter = oldsources.iterator(); iter.hasNext();) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.487"></a>
+<span class="sourceLineNo">488</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.488"></a>
+<span class="sourceLineNo">489</span> String queueId = oldSource.getQueueId();<a name="line.489"></a>
+<span class="sourceLineNo">490</span> oldSource.terminate(terminateMessage);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> oldSource.getSourceMetrics().clear();<a name="line.491"></a>
+<span class="sourceLineNo">492</span> queueStorage.removeQueue(server.getServerName(), queueId);<a name="line.492"></a>
+<span class="sourceLineNo">493</span> walsByIdRecoveredQueues.remove(queueId);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> iter.remove();<a name="line.494"></a>
+<span class="sourceLineNo">495</span> }<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span> }<a name="line.498"></a>
+<span class="sourceLineNo">499</span><a name="line.499"></a>
+<span class="sourceLineNo">500</span> /**<a name="line.500"></a>
+<span class="sourceLineNo">501</span> * Close the previous replication sources of this peer id and open new sources to trigger the new<a name="line.501"></a>
+<span class="sourceLineNo">502</span> * replication state changes or new replication config changes. Here we don't need to change<a name="line.502"></a>
+<span class="sourceLineNo">503</span> * replication queue storage and only to enqueue all logs to the new replication source<a name="line.503"></a>
+<span class="sourceLineNo">504</span> * @param peerId the id of the replication peer<a name="line.504"></a>
+<span class="sourceLineNo">505</span> */<a name="line.505"></a>
+<span class="sourceLineNo">506</span> public void refreshSources(String peerId) throws IOException {<a name="line.506"></a>
+<span class="sourceLineNo">507</span> String terminateMessage = "Peer " + peerId +<a name="line.507"></a>
+<span class="sourceLineNo">508</span> " state or config changed. Will close the previous replication source and open a new one";<a name="line.508"></a>
+<span class="sourceLineNo">509</span> ReplicationPeer peer = replicationPeers.getPeer(peerId);<a name="line.509"></a>
+<span class="sourceLineNo">510</span> ReplicationSourceInterface src = createSource(peerId, peer);<a name="line.510"></a>
+<span class="sourceLineNo">511</span> // synchronized on latestPaths to avoid missing the new log<a name="line.511"></a>
+<span class="sourceLineNo">512</span> synchronized (this.latestPaths) {<a name="line.512"></a>
+<span class="sourceLineNo">513</span> ReplicationSourceInterface toRemove = this.sources.put(peerId, src);<a name="line.513"></a>
+<span class="sourceLineNo">514</span> if (toRemove != null) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span> LOG.info("Terminate replication source for " + toRemove.getPeerId());<a name="line.515"></a>
+<span class="sourceLineNo">516</span> // Do not clear metrics<a name="line.516"></a>
+<span class="sourceLineNo">517</span> toRemove.terminate(terminateMessage, null, false);<a name="line.517"></a>
+<span class="sourceLineNo">518</span> }<a name="line.518"></a>
+<span class="sourceLineNo">519</span> for (NavigableSet<String> walsByGroup : walsById.get(peerId).values()) {<a name="line.519"></a>
+<span class="sourceLineNo">520</span> walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));<a name="line.520"></a>
+<span class="sourceLineNo">521</span> }<a name="line.521"></a>
+<span class="sourceLineNo">522</span> }<a name="line.522"></a>
+<span class="sourceLineNo">523</span> LOG.info("Startup replication source for " + src.getPeerId());<a name="line.523"></a>
+<span class="sourceLineNo">524</span> src.startup();<a name="line.524"></a>
+<span class="sourceLineNo">525</span><a name="line.525"></a>
+<span class="sourceLineNo">526</span> List<ReplicationSourceInterface> toStartup = new ArrayList<>();<a name="line.526"></a>
+<span class="sourceLineNo">527</span> // synchronized on oldsources to avoid race with NodeFailoverWorker<a name="line.527"></a>
+<span class="sourceLineNo">528</span> synchronized (this.oldsources) {<a name="line.528"></a>
+<span class="sourceLineNo">529</span> List<String> previousQueueIds = new ArrayList<>();<a name="line.529"></a>
+<span class="sourceLineNo">530</span> for (Iterator<ReplicationSourceInterface> iter = this.oldsources.iterator(); iter<a name="line.530"></a>
+<span class="sourceLineNo">531</span> .hasNext();) {<a name="line.531"></a>
+<span class="sourceLineNo">532</span> ReplicationSourceInterface oldSource = iter.next();<a name="line.532"></a>
+<span class="sourceLineNo">533</span> if (oldSource.getPeerId().equals(peerId)) {<a name="line.533"></a>
+<span class="sourceLineNo">534</span> previousQueueIds.add(oldSource.getQueueId());<a name="line.534"></a>
+<span class="sourceLineNo">535</span> oldSource.terminate(terminateMessage);<a name="line.535"></a>
+<span class="sourceLineNo">536</span> iter.remove();<a name="line.536"></a>
+<span class="sourceLineNo">537</span> }<a name="line.537"></a>
+<span class="sourceLineNo">538</span> }<a name="line.538"></a>
+<span class="sourceLineNo">539</span> for (String queueId : previousQueueIds) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span> ReplicationSourceInterface recoveredReplicationSource = createSource(queueId, peer);<a name="line.540"></a>
+<span class="sourceLineNo">541</span> this.oldsources.add(recoveredReplicationSource);<a name="line.541"></a>
+<span class="sourceLineNo">542</span> for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span> toStartup.add(recoveredReplicationSource);<a name="line.545"></a>
+<span class="sourceLineNo">546</span> }<a name="line.546"></a>
+<span class="sourceLineNo">547</span> }<a name="line.547"></a>
+<span class="sourceLineNo">548</span> for (ReplicationSourceInterface replicationSource : toStartup) {<a name="line.548"></a>
+<span class="sourceLineNo">549</span> replicationSource.startup();<a name="line.549"></a>
+<span class="sourceLineNo">550</span> }<a name="line.550"></a>
+<span class="sourceLineNo">551</span> }<a name="line.551"></a>
+<span class="sourceLineNo">552</span><a name="line.552"></a>
+<span class="sourceLineNo">553</span> /**<a name="line.553"></a>
+<span class="sourceLineNo">554</span> * Clear the metrics and related replication queue of the specified old source<a name="line.554"></a>
+<span class="sourceLineNo">555</span> * @param src source to clear<a name="line.555"></a>
+<span class="sourceLineNo">556</span> */<a name="line.556"></a>
+<span class="sourceLineNo">557</span> private boolean removeRecoveredSource(ReplicationSourceInterface src) {<a name="line.557"></a>
+<span class="sourceLineNo">558</span> if (!this.oldsources.remove(src)) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span> return false;<a name="line.559"></a>
+<span class="sourceLineNo">560</span> }<a name="line.560"></a>
+<span class="sourceLineNo">561</span> LOG.info("Done with the recovered queue {}", src.getQueueId());<a name="line.561"></a>
+<span class="sourceLineNo">562</span> // Delete queue from storage and memory<a name="line.562"></a>
+<span class="sourceLineNo">563</span> deleteQueue(src.getQueueId());<a name="line.563"></a>
+<span class="sourceLineNo">564</span> this.walsByIdRecoveredQueues.remove(src.getQueueId());<a name="line.564"></a>
+<span class="sourceLineNo">565</span> return true;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> }<a name="line.566"></a>
+<span class="sourceLineNo">567</span><a name="line.567"></a>
+<span class="sourceLineNo">568</span> void finishRecoveredSource(ReplicationSourceInterface src) {<a name="line.568"></a>
+<span class="sourceLineNo">569</span> synchronized (oldsources) {<a name="line.569"></a>
+<span class="sourceLineNo">570</span> if (!removeRecoveredSource(src)) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span> return;<a name="line.571"></a>
+<span class="sourceLineNo">572</span> }<a name="line.572"></a>
+<span class="sourceLineNo">573</span> }<a name="line.573"></a>
+<span class="sourceLineNo">574</span> LOG.info("Finished recovering queue {} with the following stats: {}", src.getQueueId(),<a name="line.574"></a>
+<span class="sourceLineNo">575</span> src.getStats());<a name="line.575"></a>
+<span class="sourceLineNo">576</span> }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span> /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span> * Clear the metrics and related replication queue of the specified old source<a name="line.579"></a>
+<span class="sourceLineNo">580</span> * @param src source to clear<a name="line.580"></a>
+<span class="sourceLineNo">581</span> */<a name="line.581"></a>
+<span class="sourceLineNo">582</span> void removeSource(ReplicationSourceInterface src) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.info("Done with the queue " + src.getQueueId());<a name="line.583"></a>
+<span class="sourceLineNo">584</span> this.sources.remove(src.getPeerId());<a name="line.584"></a>
+<span class="sourceLineNo">585</span> // Delete queue from storage and memory<a name="line.585"></a>
+<span class="sourceLineNo">586</span> deleteQueue(src.getQueueId());<a name="line.586"></a>
+<span class="sourceLineNo">587</span> this.walsById.remove(src.getQueueId());<a name="line.587"></a>
+<span class="sourceLineNo">588</span><a name="line.588"></a>
+<span class="sourceLineNo">589</span> }<a name="line.589"></a>
+<span class="sourceLineNo">590</span><a name="line.590"></a>
+<span class="sourceLineNo">591</span> /**<a name="line.591"></a>
+<span class="sourceLineNo">592</span> * Delete a complete queue of wals associated with a replication source<a name="line.592"></a>
+<span class="sourceLineNo">593</span> * @param queueId the id of replication queue to delete<a name="line.593"></a>
+<span class="sourceLineNo">594</span> */<a name="line.594"></a>
+<span class="sourceLineNo">595</span> private void deleteQueue(String queueId) {<a name="line.595"></a>
+<span class="sourceLineNo">596</span> abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.596"></a>
+<span class="sourceLineNo">597</span> }<a name="line.597"></a>
+<span class="sourceLineNo">598</span><a name="line.598"></a>
+<span class="sourceLineNo">599</span> @FunctionalInterface<a name="line.599"></a>
+<span class="sourceLineNo">600</span> private interface ReplicationQueueOperation {<a name="line.600"></a>
+<span class="sourceLineNo">601</span> void exec() throws ReplicationException;<a name="line.601"></a>
+<span class="sourceLineNo">602</span> }<a name="line.602"></a>
+<span class="sourceLineNo">603</span><a name="line.603"></a>
+<span class="sourceLineNo">604</span> /**<a name="line.604"></a>
+<span class="sourceLineNo">605</span> * Refresh replication source will terminate the old source first, then the source thread will be<a name="line.605"></a>
+<span class="sourceLineNo">606</span> * interrupted. Need to handle it instead of abort the region server.<a name="line.606"></a>
+<span class="sourceLineNo">607</span> */<a name="line.607"></a>
+<span class="sourceLineNo">608</span> private void interruptOrAbortWhenFail(ReplicationQueueOperation op) {<a name="line.608"></a>
+<span class="sourceLineNo">609</span> try {<a name="line.609"></a>
+<span class="sourceLineNo">610</span> op.exec();<a name="line.610"></a>
+<span class="sourceLineNo">611</span> } catch (ReplicationException e) {<a name="line.611"></a>
+<span class="sourceLineNo">612</span> if (e.getCause() != null && e.getCause() instanceof KeeperException.SystemErrorException<a name="line.612"></a>
+<span class="sourceLineNo">613</span> && e.getCause().getCause() != null && e.getCause()<a name="line.613"></a>
+<span class="sourceLineNo">614</span> .getCause() instanceof InterruptedException) {<a name="line.614"></a>
+<span class="sourceLineNo">615</span> // ReplicationRuntimeException(a RuntimeException) is thrown out here. The reason is<a name="line.615"></a>
+<span class="sourceLineNo">616</span> // that thread is interrupted deep down in the stack, it should pass the following<a name="line.616"></a>
+<span class="sourceLineNo">617</span> // processing logic and propagate to the most top layer which can handle this exception<a name="line.617"></a>
+<span class="sourceLineNo">618</span> // properly. In this specific case, the top layer is ReplicationSourceShipper#run().<a name="line.618"></a>
+<span class="sourceLineNo">619</span> throw new ReplicationRuntimeException(<a name="line.619"></a>
+<span class="sourceLineNo">620</span> "Thread is interrupted, the replication source may be terminated",<a name="line.620"></a>
+<span class="sourceLineNo">621</span> e.getCause().getCause());<a name="line.621"></a>
+<span class="sourceLineNo">622</span> }<a name="line.622"></a>
+<span class="sourceLineNo">623</span> server.abort("Failed to operate on replication queue", e);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span> }<a name="line.625"></a>
+<span class="sourceLineNo">626</span><a name="line.626"></a>
+<span class="sourceLineNo">627</span> private void abortWhenFail(ReplicationQueueOperation op) {<a name="line.627"></a>
+<span class="sourceLineNo">628</span> try {<a name="line.628"></a>
+<span class="sourceLineNo">629</span> op.exec();<a name="line.629"></a>
+<span class="sourceLineNo">630</span> } catch (ReplicationException e) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> server.abort("Failed to operate on replication queue", e);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> }<a name="line.632"></a>
+<span class="sourceLineNo">633</span> }<a name="line.633"></a>
+<span class="sourceLineNo">634</span><a name="line.634"></a>
+<span class="sourceLineNo">635</span> private void throwIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.635"></a>
+<span class="sourceLineNo">636</span> try {<a name="line.636"></a>
+<span class="sourceLineNo">637</span> op.exec();<a name="line.637"></a>
+<span class="sourceLineNo">638</span> } catch (ReplicationException e) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> throw new IOException(e);<a name="line.639"></a>
+<span class="sourceLineNo">640</span> }<a name="line.640"></a>
+<span class="sourceLineNo">641</span> }<a name="line.641"></a>
+<span class="sourceLineNo">642</span><a name="line.642"></a>
+<span class="sourceLineNo">643</span> private void abortAndThrowIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {<a name="line.643"></a>
+<span class="sourceLineNo">644</span> try {<a name="line.644"></a>
+<span class="sourceLineNo">645</span> op.exec();<a name="line.645"></a>
+<span class="sourceLineNo">646</span> } catch (ReplicationException e) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span> server.abort("Failed to operate on replication queue", e);<a name="line.647"></a>
+<span class="sourceLineNo">648</span> throw new IOException(e);<a name="line.648"></a>
+<span class="sourceLineNo">649</span> }<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * This method will log the current position to storage. And also clean old logs from the<a name="line.653"></a>
+<span class="sourceLineNo">654</span> * replication queue.<a name="line.654"></a>
+<span class="sourceLineNo">655</span> * @param source the replication source<a name="line.655"></a>
+<span class="sourceLineNo">656</span> * @param entryBatch the wal entry batch we just shipped<a name="line.656"></a>
+<span class="sourceLineNo">657</span> */<a name="line.657"></a>
+<span class="sourceLineNo">658</span> public void logPositionAndCleanOldLogs(ReplicationSourceInterface source,<a name="line.658"></a>
+<span class="sourceLineNo">659</span> WALEntryBatch entryBatch) {<a name="line.659"></a>
+<span class="sourceLineNo">660</span> String fileName = entryBatch.getLastWalPath().getName();<a name="line.660"></a>
+<span class="sourceLineNo">661</span> interruptOrAbortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(),<a name="line.661"></a>
+<span class="sourceLineNo">662</span> source.getQueueId(), fileName, entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds()));<a name="line.662"></a>
+<span class="sourceLineNo">663</span> cleanOldLogs(fileName, entryBatch.isEndOfFile(), source);<a name="line.663"></a>
+<span class="sourceLineNo">664</span> }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span> /**<a name="line.666"></a>
+<span class="sourceLineNo">667</span> * Cleans a log file and all older logs from replication queue. Called when we are sure that a log<a name="line.667"></a>
+<span class="sourceLineNo">668</span> * file is closed and has no more entries.<a name="line.668"></a>
+<span class="sourceLineNo">669</span> * @param log Path to the log<a name="line.669"></a>
+<span class="sourceLineNo">670</span> * @param inclusive whether we should also remove the given log file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> * @param source the replication source<a name="line.671"></a>
+<span class="sourceLineNo">672</span> */<a name="line.672"></a>
+<span class="sourceLineNo">673</span> void cleanOldLogs(String log, boolean inclusive, ReplicationSourceInterface source) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(log);<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (source.isRecovered()) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> NavigableSet<String> wals = walsByIdRecoveredQueues.get(source.getQueueId()).get(logPrefix);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> if (wals != null) {<a name="line.677"></a>
+<span class="sourceLineNo">678</span> NavigableSet<String> walsToRemove = wals.headSet(log, inclusive);<a name="line.678"></a>
+<span class="sourceLineNo">679</span> if (walsToRemove.isEmpty()) {<a name="line.679"></a>
+<span class="sourceLineNo">680</span> return;<a name="line.680"></a>
+<span class="sourceLineNo">681</span> }<a name="line.681"></a>
+<span class="sourceLineNo">682</span> cleanOldLogs(walsToRemove, source);<a name="line.682"></a>
+<span class="sourceLineNo">683</span> walsToRemove.clear();<a name="line.683"></a>
+<span class="sourceLineNo">684</span> }<a name="line.684"></a>
+<span class="sourceLineNo">685</span> } else {<a name="line.685"></a>
+<span class="sourceLineNo">686</span> NavigableSet<String> wals;<a name="line.686"></a>
+<span class="sourceLineNo">687</span> NavigableSet<String> walsToRemove;<a name="line.687"></a>
+<span class="sourceLineNo">688</span> // synchronized on walsById to avoid race with preLogRoll<a name="line.688"></a>
+<span class="sourceLineNo">689</span> synchronized (this.walsById) {<a name="line.689"></a>
+<span class="sourceLineNo">690</span> wals = walsById.get(source.getQueueId()).get(logPrefix);<a name="line.690"></a>
+<span class="sourceLineNo">691</span> if (wals == null) {<a name="line.691"></a>
+<span class="sourceLineNo">692</span> return;<a name="line.692"></a>
+<span class="sourceLineNo">693</span> }<a name="line.693"></a>
+<span class="sourceLineNo">694</span> walsToRemove = wals.headSet(log, inclusive);<a name="line.694"></a>
+<span class="sourceLineNo">695</span> if (walsToRemove.isEmpty()) {<a name="line.695"></a>
+<span class="sourceLineNo">696</span> return;<a name="line.696"></a>
+<span class="sourceLineNo">697</span> }<a name="line.697"></a>
+<span class="sourceLineNo">698</span> walsToRemove = new TreeSet<>(walsToRemove);<a name="line.698"></a>
+<span class="sourceLineNo">699</span> }<a name="line.699"></a>
+<span class="sourceLineNo">700</span> // cleanOldLogs may spend some time, especially for sync replication where we may want to<a name="line.700"></a>
+<span class="sourceLineNo">701</span> // remove remote wals as the remote cluster may have already been down, so we do it outside<a name="line.701"></a>
+<span class="sourceLineNo">702</span> // the lock to avoid block preLogRoll<a name="line.702"></a>
+<span class="sourceLineNo">703</span> cleanOldLogs(walsToRemove, source);<a name="line.703"></a>
+<span class="sourceLineNo">704</span> // now let's remove the files in the set<a name="line.704"></a>
+<span class="sourceLineNo">705</span> synchronized (this.walsById) {<a name="line.705"></a>
+<span class="sourceLineNo">706</span> wals.removeAll(walsToRemove);<a name="line.706"></a>
+<span class="sourceLineNo">707</span> }<a name="line.707"></a>
+<span class="sourceLineNo">708</span> }<a name="line.708"></a>
+<span class="sourceLineNo">709</span> }<a name="line.709"></a>
+<span class="sourceLineNo">710</span><a name="line.710"></a>
+<span class="sourceLineNo">711</span> private void removeRemoteWALs(String peerId, String remoteWALDir, Collection<String> wals)<a name="line.711"></a>
+<span class="sourceLineNo">712</span> throws IOException {<a name="line.712"></a>
+<span class="sourceLineNo">713</span> Path remoteWALDirForPeer = ReplicationUtils.getPeerRemoteWALDir(remoteWALDir, peerId);<a name="line.713"></a>
+<span class="sourceLineNo">714</span> FileSystem fs = ReplicationUtils.getRemoteWALFileSystem(conf, remoteWALDir);<a name="line.714"></a>
+<span class="sourceLineNo">715</span> for (String wal : wals) {<a name="line.715"></a>
+<span class="sourceLineNo">716</span> Path walFile = new Path(remoteWALDirForPeer, wal);<a name="line.716"></a>
+<span class="sourceLineNo">717</span> try {<a name="line.717"></a>
+<span class="sourceLineNo">718</span> if (!fs.delete(walFile, false) && fs.exists(walFile)) {<a name="line.718"></a>
+<span class="sourceLineNo">719</span> throw new IOException("Can not delete " + walFile);<a name="line.719"></a>
+<span class="sourceLineNo">720</span> }<a name="line.720"></a>
+<span class="sourceLineNo">721</span> } catch (FileNotFoundException e) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span> // Just ignore since this means the file has already been deleted.<a name="line.722"></a>
+<span class="sourceLineNo">723</span> // The javadoc of the FileSystem.delete methods does not specify the behavior of deleting an<a name="line.723"></a>
+<span class="sourceLineNo">724</span> // inexistent file, so here we deal with both, i.e, check the return value of the<a name="line.724"></a>
+<span class="sourceLineNo">725</span> // FileSystem.delete, and also catch FNFE.<a name="line.725"></a>
+<span class="sourceLineNo">726</span> LOG.debug("The remote wal {} has already been deleted?", walFile, e);<a name="line.726"></a>
+<span class="sourceLineNo">727</span> }<a name="line.727"></a>
+<span class="sourceLineNo">728</span> }<a name="line.728"></a>
+<span class="sourceLineNo">729</span> }<a name="line.729"></a>
+<span class="sourceLineNo">730</span><a name="line.730"></a>
+<span class="sourceLineNo">731</span> private void cleanOldLogs(NavigableSet<String> wals, ReplicationSourceInterface source) {<a name="line.731"></a>
+<span class="sourceLineNo">732</span> LOG.debug("Removing {} logs in the list: {}", wals.size(), wals);<a name="line.732"></a>
+<span class="sourceLineNo">733</span> // The intention here is that, we want to delete the remote wal files ASAP as it may effect the<a name="line.733"></a>
+<span class="sourceLineNo">734</span> // failover time if you want to transit the remote cluster from S to A. And the infinite retry<a name="line.734"></a>
+<span class="sourceLineNo">735</span> // is not a problem, as if we can not contact with the remote HDFS cluster, then usually we can<a name="line.735"></a>
+<span class="sourceLineNo">736</span> // not contact with the HBase cluster either, so the replication will be blocked either.<a name="line.736"></a>
+<span class="sourceLineNo">737</span> if (source.isSyncReplication()) {<a name="line.737"></a>
+<span class="sourceLineNo">738</span> String peerId = source.getPeerId();<a name="line.738"></a>
+<span class="sourceLineNo">739</span> String remoteWALDir = source.getPeer().getPeerConfig().getRemoteWALDir();<a name="line.739"></a>
+<span class="sourceLineNo">740</span> // Filter out the wals need to be removed from the remote directory. Its name should be the<a name="line.740"></a>
+<span class="sourceLineNo">741</span> // special format, and also, the peer id in its name should match the peer id for the<a name="line.741"></a>
+<span class="sourceLineNo">742</span> // replication source.<a name="line.742"></a>
+<span class="sourceLineNo">743</span> List<String> remoteWals = wals.stream().filter(w -> SyncReplicationWALProvider<a name="line.743"></a>
+<span class="sourceLineNo">744</span> .getSyncReplicationPeerIdFromWALName(w).map(peerId::equals).orElse(false))<a name="line.744"></a>
+<span class="sourceLineNo">745</span> .collect(Collectors.toList());<a name="line.745"></a>
+<span class="sourceLineNo">746</span> LOG.debug("Removing {} logs from remote dir {} in the list: {}", remoteWals.size(),<a name="line.746"></a>
+<span class="sourceLineNo">747</span> remoteWALDir, remoteWals);<a name="line.747"></a>
+<span class="sourceLineNo">748</span> if (!remoteWals.isEmpty()) {<a name="line.748"></a>
+<span class="sourceLineNo">749</span> for (int sleepMultiplier = 0;;) {<a name="line.749"></a>
+<span class="sourceLineNo">750</span> try {<a name="line.750"></a>
+<span class="sourceLineNo">751</span> removeRemoteWALs(peerId, remoteWALDir, remoteWals);<a name="line.751"></a>
+<span class="sourceLineNo">752</span> break;<a name="line.752"></a>
+<span class="sourceLineNo">753</span> } catch (IOException e) {<a name="line.753"></a>
+<span class="sourceLineNo">754</span> LOG.warn("Failed to delete remote wals from remote dir {} for peer {}", remoteWALDir,<a name="line.754"></a>
+<span class="sourceLineNo">755</span> peerId);<a name="line.755"></a>
+<span class="sourceLineNo">756</span> }<a name="line.756"></a>
+<span class="sourceLineNo">757</span> if (!source.isSourceActive()) {<a name="line.757"></a>
+<span class="sourceLineNo">758</span> // skip the following operations<a name="line.758"></a>
+<span class="sourceLineNo">759</span> return;<a name="line.759"></a>
+<span class="sourceLineNo">760</span> }<a name="line.760"></a>
+<span class="sourceLineNo">761</span> if (ReplicationUtils.sleepForRetries("Failed to delete remote wals", sleepForRetries,<a name="line.761"></a>
+<span class="sourceLineNo">762</span> sleepMultiplier, maxRetriesMultiplier)) {<a name="line.762"></a>
+<span class="sourceLineNo">763</span> sleepMultiplier++;<a name="line.763"></a>
+<span class="sourceLineNo">764</span> }<a name="line.764"></a>
+<span class="sourceLineNo">765</span> }<a name="line.765"></a>
+<span class="sourceLineNo">766</span> }<a name="line.766"></a>
+<span class="sourceLineNo">767</span> }<a name="line.767"></a>
+<span class="sourceLineNo">768</span> String queueId = source.getQueueId();<a name="line.768"></a>
+<span class="sourceLineNo">769</span> for (String wal : wals) {<a name="line.769"></a>
+<span class="sourceLineNo">770</span> interruptOrAbortWhenFail(<a name="line.770"></a>
+<span class="sourceLineNo">771</span> () -> this.queueStorage.removeWAL(server.getServerName(), queueId, wal));<a name="line.771"></a>
+<span class="sourceLineNo">772</span> }<a name="line.772"></a>
+<span class="sourceLineNo">773</span> }<a name="line.773"></a>
+<span class="sourceLineNo">774</span><a name="line.774"></a>
+<span class="sourceLineNo">775</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.775"></a>
+<span class="sourceLineNo">776</span> public void preLogRoll(Path newLog) throws IOException {<a name="line.776"></a>
+<span class="sourceLineNo">777</span> String logName = newLog.getName();<a name="line.777"></a>
+<span class="sourceLineNo">778</span> String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);<a name="line.778"></a>
+<span class="sourceLineNo">779</span> // synchronized on latestPaths to avoid the new open source miss the new log<a name="line.779"></a>
+<span class="sourceLineNo">780</span> synchronized (this.latestPaths) {<a name="line.780"></a>
+<span class="sourceLineNo">781</span> // Add log to queue storage<a name="line.781"></a>
+<span class="sourceLineNo">782</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.782"></a>
+<span class="sourceLineNo">783</span> // If record log to queue storage failed, abort RS and throw exception to make log roll<a name="line.783"></a>
+<span class="sourceLineNo">784</span> // failed<a name="line.784"></a>
+<span class="sourceLineNo">785</span> abortAndThrowIOExceptionWhenFail(<a name="line.785"></a>
+<span class="sourceLineNo">786</span> () -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));<a name="line.786"></a>
+<span class="sourceLineNo">787</span> }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span> // synchronized on walsById to avoid race with cleanOldLogs<a name="line.789"></a>
+<span class="sourceLineNo">790</span> synchronized (this.walsById) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span> // Update walsById map<a name="line.791"></a>
+<span class="sourceLineNo">792</span> for (Map.Entry<String, Map<String, NavigableSet<String>>> entry : this.walsById<a name="line.792"></a>
+<span class="sourceLineNo">793</span> .entrySet()) {<a name="line.793"></a>
+<span class="sourceLineNo">794</span> String peerId = entry.getKey();<a name="line.794"></a>
+<span class="sourceLineNo">795</span> Map<String, NavigableSet<String>> walsByPrefix = entry.getValue();<a name="line.795"></a>
+<span class="sourceLineNo">796</span> boolean existingPrefix = false;<a name="line.796"></a>
+<span class="sourceLineNo">797</span> for (Map.Entry<String, NavigableSet<String>> walsEntry : walsByPrefix.entrySet()) {<a name="line.797"></a>
+<span class="sourceLineNo">798</span> SortedSet<String> wals = walsEntry.getValue();<a name="line.798"></a>
+<span class="sourceLineNo">799</span> if (this.sources.isEmpty()) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span> // If there's no slaves, don't need to keep the old wals since<a name="line.800"></a>
+<span class="sourceLineNo">801</span> // we only consider the last one when a new slave comes in<a name="line.801"></a>
+<span class="sourceLineNo">802</span> wals.clear();<a name="line.802"></a>
+<span class="sourceLineNo">803</span> }<a name="line.803"></a>
+<span class="sourceLineNo">804</span> if (logPrefix.equals(walsEntry.getKey())) {<a name="line.804"></a>
+<span class="sourceLineNo">805</span> wals.add(logName);<a name="line.805"></a>
+<span class="sourceLineNo">806</span> existingPrefix = true;<a name="line.806"></a>
+<span class="sourceLineNo">807</span> }<a name="line.807"></a>
+<span class="sourceLineNo">808</span> }<a name="line.808"></a>
+<span class="sourceLineNo">809</span> if (!existingPrefix) {<a name="line.809"></a>
+<span class="sourceLineNo">810</span> // The new log belongs to a new group, add it into this peer<a name="line.810"></a>
+<span class="sourceLineNo">811</span> LOG.debug("Start tracking logs for wal group {} for peer {}", logPrefix, peerId);<a name="line.811"></a>
+<span class="sourceLineNo">812</span> NavigableSet<String> wals = new TreeSet<>();<a name="line.812"></a>
+<span class="sourceLineNo">813</span> wals.add(logName);<a name="line.813"></a>
+<span class="sourceLineNo">814</span> walsByPrefix.put(logPrefix, wals);<a name="line.814"></a>
+<span class="sourceLineNo">815</span> }<a name="line.815"></a>
+<span class="sourceLineNo">816</span> }<a name="line.816"></a>
+<span class="sourceLineNo">817</span> }<a name="line.817"></a>
+<span class="sourceLineNo">818</span><a name="line.818"></a>
+<span class="sourceLineNo">819</span> // Add to latestPaths<a name="line.819"></a>
+<span class="sourceLineNo">820</span> latestPaths.put(logPrefix, newLog);<a name="line.820"></a>
+<span class="sourceLineNo">821</span> }<a name="line.821"></a>
+<span class="sourceLineNo">822</span> }<a name="line.822"></a>
+<span class="sourceLineNo">823</span><a name="line.823"></a>
+<span class="sourceLineNo">824</span> // public because of we call it in TestReplicationEmptyWALRecovery<a name="line.824"></a>
+<span class="sourceLineNo">825</span> public void postLogRoll(Path newLog) throws IOException {<a name="line.825"></a>
+<span class="sourceLineNo">826</span> // This only updates the sources we own, not the recovered ones<a name="line.826"></a>
+<span class="sourceLineNo">827</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.827"></a>
+<span class="sourceLineNo">828</span> source.enqueueLog(newLog);<a name="line.828"></a>
+<span class="sourceLineNo">829</span> LOG.trace("Enqueued {} to source {} while performing postLogRoll operation.",<a name="line.829"></a>
+<span class="sourceLineNo">830</span> newLog, source.getQueueId());<a name="line.830"></a>
+<span class="sourceLineNo">831</span> }<a name="line.831"></a>
+<span class="sourceLineNo">832</span> }<a name="line.832"></a>
+<span class="sourceLineNo">833</span><a name="line.833"></a>
+<span class="sourceLineNo">834</span> @Override<a name="line.834"></a>
+<span class="sourceLineNo">835</span> public void regionServerRemoved(String regionserver) {<a name="line.835"></a>
+<span class="sourceLineNo">836</span> transferQueues(ServerName.valueOf(regionserver));<a name="line.836"></a>
+<span class="sourceLineNo">837</span> }<a name="line.837"></a>
+<span class="sourceLineNo">838</span><a name="line.838"></a>
+<span class="sourceLineNo">839</span> /**<a name="line.839"></a>
+<span class="sourceLineNo">840</span> * Transfer all the queues of the specified to this region server. First it tries to grab a lock<a name="line.840"></a>
+<span class="sourceLineNo">841</span> * and if it works it will move the old queues and finally will delete the old queues.<a name="line.841"></a>
+<span class="sourceLineNo">842</span> * <p><a name="line.842"></a>
+<span class="sourceLineNo">843</span> * It creates one old source for any type of source of the old rs.<a name="line.843"></a>
+<span class="sourceLineNo">844</span> */<a name="line.844"></a>
+<span class="sourceLineNo">845</span> private void transferQueues(ServerName deadRS) {<a name="line.845"></a>
+<span class="sourceLineNo">846</span> if (server.getServerName().equals(deadRS)) {<a name="line.846"></a>
+<span class="sourceLineNo">847</span> // it's just us, give up<a name="line.847"></a>
+<span class="sourceLineNo">848</span> return;<a name="line.848"></a>
+<span class="sourceLineNo">849</span> }<a name="line.849"></a>
+<span class="sourceLineNo">850</span> NodeFailoverWorker transfer = new NodeFailoverWorker(deadRS);<a name="line.850"></a>
+<span class="sourceLineNo">851</span> try {<a name="line.851"></a>
+<span class="sourceLineNo">852</span> this.executor.execute(transfer);<a name="line.852"></a>
+<span class="sourceLineNo">853</span> } catch (RejectedExecutionException ex) {<a name="line.853"></a>
+<span class="sourceLineNo">854</span> CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)<a name="line.854"></a>
+<span class="sourceLineNo">855</span> .getGlobalSource().incrFailedRecoveryQueue();<a name="line.855"></a>
+<span class="sourceLineNo">856</span> LOG.info("Cancelling the transfer of " + deadRS + " because of " + ex.getMessage());<a name="line.856"></a>
+<span class="sourceLineNo">857</span> }<a name="line.857"></a>
+<span class="sourceLineNo">858</span> }<a name="line.858"></a>
+<span class="sourceLineNo">859</span><a name="line.859"></a>
+<span class="sourceLineNo">860</span> /**<a name="line.860"></a>
+<span class="sourceLineNo">861</span> * Class responsible to setup new ReplicationSources to take care of the queues from dead region<a name="line.861"></a>
+<span class="sourceLineNo">862</span> * servers.<a name="line.862"></a>
+<span class="sourceLineNo">863</span> */<a name="line.863"></a>
+<span class="sourceLineNo">864</span> class NodeFailoverWorker extends Thread {<a name="line.864"></a>
+<span class="sourceLineNo">865</span><a name="line.865"></a>
+<span class="sourceLineNo">866</span> private final ServerName deadRS;<a name="line.866"></a>
+<span class="sourceLineNo">867</span> // After claim the queues from dead region server, the NodeFailoverWorker will skip to start<a name="line.867"></a>
+<span class="sourceLineNo">868</span> // the RecoveredReplicationSource if the peer has been removed. but there's possible that<a name="line.868"></a>
+<span class="sourceLineNo">869</span> // remove a peer with peerId = 2 and add a peer with peerId = 2 again during the<a name="line.869"></a>
+<span class="sourceLineNo">870</span> // NodeFailoverWorker. So we need a deep copied <peerId, peer> map to decide whether we<a name="line.870"></a>
+<span class="sourceLineNo">871</span> // should start the RecoveredReplicationSource. If the latest peer is not the old peer when<a name="line.871"></a>
+<span class="sourceLineNo">872</span> // NodeFailoverWorker begin, we should skip to start the RecoveredReplicationSource, Otherwise<a name="line.872"></a>
+<span class="sourceLineNo">873</span> // the rs will abort (See HBASE-20475).<a name="line.873"></a>
+<span class="sourceLineNo">874</span> private final Map<String, ReplicationPeerImpl> peersSnapshot;<a name="line.874"></a>
+<span class="sourceLineNo">875</span><a name="line.875"></a>
+<span class="sourceLineNo">876</span> public NodeFailoverWorker(ServerName deadRS) {<a name="line.876"></a>
+<span class="sourceLineNo">877</span> super("Failover-for-" + deadRS);<a name="line.877"></a>
+<span class="sourceLineNo">878</span> this.deadRS = deadRS;<a name="line.878"></a>
+<span class="sourceLineNo">879</span> peersSnapshot = new HashMap<>(replicationPeers.getPeerCache());<a name="line.879"></a>
+<span class="sourceLineNo">880</span> }<a name="line.880"></a>
+<span class="sourceLineNo">881</span><a name="line.881"></a>
+<span class="sourceLineNo">882</span> private boolean isOldPeer(String peerId, ReplicationPeerImpl newPeerRef) {<a name="line.882"></a>
+<span class="sourceLineNo">883</span> ReplicationPeerImpl oldPeerRef = peersSnapshot.get(peerId);<a name="line.883"></a>
+<span class="sourceLineNo">884</span> return oldPeerRef != null && oldPeerRef == newPeerRef;<a name="line.884"></a>
+<span class="sourceLineNo">885</span> }<a name="line.885"></a>
+<span class="sourceLineNo">886</span><a name="line.886"></a>
+<span class="sourceLineNo">887</span> @Override<a name="line.887"></a>
+<span class="sourceLineNo">888</span> public void run() {<a name="line.888"></a>
+<span class="sourceLineNo">889</span> // Wait a bit before transferring the queues, we may be shutting down.<a name="line.889"></a>
+<span class="sourceLineNo">890</span> // This sleep may not be enough in some cases.<a name="line.890"></a>
+<span class="sourceLineNo">891</span> try {<a name="line.891"></a>
+<span class="sourceLineNo">892</span> Thread.sleep(sleepBeforeFailover +<a name="line.892"></a>
+<span class="sourceLineNo">893</span> (long) (ThreadLocalRandom.current().nextFloat() * sleepBeforeFailover));<a name="line.893"></a>
+<span class="sourceLineNo">894</span> } catch (InterruptedException e) {<a name="line.894"></a>
+<span class="sourceLineNo">895</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.895"></a>
+<span class="sourceLineNo">896</span> Thread.currentThread().interrupt();<a name="line.896"></a>
+<span class="sourceLineNo">897</span> }<a name="line.897"></a>
+<span class="sourceLineNo">898</span> // We try to lock that rs' queue directory<a name="line.898"></a>
+<span class="sourceLineNo">899</span> if (server.isStopped()) {<a name="line.899"></a>
+<span class="sourceLineNo">900</span> LOG.info("Not transferring queue since we are shutting down");<a name="line.900"></a>
+<span class="sourceLineNo">901</span> return;<a name="line.901"></a>
+<span class="sourceLineNo">902</span> }<a name="line.902"></a>
+<span class="sourceLineNo">903</span> Map<String, Set<String>> newQueues = new HashMap<>();<a name="line.903"></a>
+<span class="sourceLineNo">904</span> try {<a name="line.904"></a>
+<span class="sourceLineNo">905</span> List<String> queues = queueStorage.getAllQueues(deadRS);<a name="line.905"></a>
+<span class="sourceLineNo">906</span> while (!queues.isEmpty()) {<a name="line.906"></a>
+<span class="sourceLineNo">907</span> Pair<String, SortedSet<String>> peer = queueStorage.claimQueue(deadRS,<a name="line.907"></a>
+<span class="sourceLineNo">908</span> queues.get(ThreadLocalRandom.current().nextInt(queues.size())), server.getServerName());<a name="line.908"></a>
+<span class="sourceLineNo">909</span> long sleep = sleepBeforeFailover / 2;<a name="line.909"></a>
+<span class="sourceLineNo">910</span> if (!peer.getSecond().isEmpty()) {<a name="line.910"></a>
+<span class="sourceLineNo">911</span> newQueues.put(peer.getFirst(), peer.getSecond());<a name="line.911"></a>
+<span class="sourceLineNo">912</span> sleep = sleepBeforeFailover;<a name="line.912"></a>
+<span class="sourceLineNo">913</span> }<a name="line.913"></a>
+<span class="sourceLineNo">914</span> try {<a name="line.914"></a>
+<span class="sourceLineNo">915</span> Thread.sleep(sleep);<a name="line.915"></a>
+<span class="sourceLineNo">916</span> } catch (InterruptedException e) {<a name="line.916"></a>
+<span class="sourceLineNo">917</span> LOG.warn("Interrupted while waiting before transferring a queue.");<a name="line.917"></a>
+<span class="sourceLineNo">918</span> Thread.currentThread().interrupt();<a name="line.918"></a>
+<span class="sourceLineNo">919</span> }<a name="line.919"></a>
+<span class="sourceLineNo">920</span> queues = queueStorage.getAllQueues(deadRS);<a name="line.920"></a>
+<span class="sourceLineNo">921</span> }<a name="line.921"></a>
+<span class="sourceLineNo">922</span> if (queues.isEmpty()) {<a name="line.922"></a>
+<span class="sourceLineNo">923</span> queueStorage.removeReplicatorIfQueueIsEmpty(deadRS);<a name="line.923"></a>
+<span class="sourceLineNo">924</span> }<a name="line.924"></a>
+<span class="sourceLineNo">925</span> } catch (ReplicationException e) {<a name="line.925"></a>
+<span class="sourceLineNo">926</span> LOG.error(String.format("ReplicationException: cannot claim dead region (%s)'s " +<a name="line.926"></a>
+<span class="sourceLineNo">927</span> "replication queue. Znode : (%s)" +<a name="line.927"></a>
+<span class="sourceLineNo">928</span> " Possible solution: check if znode size exceeds jute.maxBuffer value. " +<a name="line.928"></a>
+<span class="sourceLineNo">929</span> " If so, increase it for both client and server side." + e), deadRS,<a name="line.929"></a>
+<span class="sourceLineNo">930</span> queueStorage.getRsNode(deadRS));<a name="line.930"></a>
+<span class="sourceLineNo">931</span> server.abort("Failed to claim queue from dead regionserver.", e);<a name="line.931"></a>
+<span class="sourceLineNo">932</span> return;<a name="line.932"></a>
+<span class="sourceLineNo">933</span> }<a name="line.933"></a>
+<span class="sourceLineNo">934</span> // Copying over the failed queue is completed.<a name="line.934"></a>
+<span class="sourceLineNo">935</span> if (newQueues.isEmpty()) {<a name="line.935"></a>
+<span class="sourceLineNo">936</span> // We either didn't get the lock or the failed region server didn't have any outstanding<a name="line.936"></a>
+<span class="sourceLineNo">937</span> // WALs to replicate, so we are done.<a name="line.937"></a>
+<span class="sourceLineNo">938</span> return;<a name="line.938"></a>
+<span class="sourceLineNo">939</span> }<a name="line.939"></a>
+<span class="sourceLineNo">940</span><a name="line.940"></a>
+<span class="sourceLineNo">941</span> for (Map.Entry<String, Set<String>> entry : newQueues.entrySet()) {<a name="line.941"></a>
+<span class="sourceLineNo">942</span> String queueId = entry.getKey();<a name="line.942"></a>
+<span class="sourceLineNo">943</span> Set<String> walsSet = entry.getValue();<a name="line.943"></a>
+<span class="sourceLineNo">944</span> try {<a name="line.944"></a>
+<span class="sourceLineNo">945</span> // there is not an actual peer defined corresponding to peerId for the failover.<a name="line.945"></a>
+<span class="sourceLineNo">946</span> ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);<a name="line.946"></a>
+<span class="sourceLineNo">947</span> String actualPeerId = replicationQueueInfo.getPeerId();<a name="line.947"></a>
+<span class="sourceLineNo">948</span><a name="line.948"></a>
+<span class="sourceLineNo">949</span> ReplicationPeerImpl peer = replicationPeers.getPeer(actualPeerId);<a name="line.949"></a>
+<span class="sourceLineNo">950</span> if (peer == null || !isOldPeer(actualPeerId, peer)) {<a name="line.950"></a>
+<span class="sourceLineNo">951</span> LOG.warn("Skipping failover for peer {} of node {}, peer is null", actualPeerId,<a name="line.951"></a>
+<span class="sourceLineNo">952</span> deadRS);<a name="line.952"></a>
+<span class="sourceLineNo">953</span> abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), queueId));<a name="line.953"></a>
+<span class="sourceLineNo">954</span> continue;<a name="line.954"></a>
+<span class="sourceLineNo">955</span> }<a name="line.955"></a>
+<span class="sourceLineNo">956</span> if (server instanceof ReplicationSyncUp.DummyServer<a name="line.956"></a>
+<span class="sourceLineNo">957</span> && peer.getPeerState().equals(PeerState.DISABLED)) {<a name="line.957"></a>
+<span class="sourceLineNo">958</span> LOG.warn("Peer {} is disabled. ReplicationSyncUp tool will skip "<a name="line.958"></a>
+<span class="sourceLineNo">959</span> + "replicating data to this peer.",<a name="line.959"></a>
+<span class="sourceLineNo">960</span> actualPeerId);<a name="line.960"></a>
+<span class="sourceLineNo">961</span> continue;<a name="line.961"></a>
+<span class="sourceLineNo">962</span> }<a name="line.962"></a>
+<span class="sourceLineNo">963</span><a name="line.963"></a>
+<span class="sourceLineNo">964</span> ReplicationSourceInterface src = createSource(queueId, peer);<a name="line.964"></a>
+<span class="sourceLineNo">965</span> // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer<a name="line.965"></a>
+<span class="sourceLineNo">966</span> synchronized (oldsources) {<a name="line.966"></a>
+<span class="sourceLineNo">967</span> peer = replicationPeers.getPeer(src.getPeerId());<a name="line.967"></a>
+<span class="sourceLineNo">968</span> if (peer == null || !isOldPeer(src.getPeerId(), peer)) {<a name="line.968"></a>
+<span class="sourceLineNo">969</span> src.terminate("Recovered queue doesn't belong to any current peer");<a name="line.969"></a>
+<span class="sourceLineNo">970</span> deleteQueue(queueId);<a name="line.970"></a>
+<span class="sourceLineNo">971</span> continue;<a name="line.971"></a>
+<span class="sourceLineNo">972</span> }<a name="line.972"></a>
+<span class="sourceLineNo">973</span> // Do not setup recovered queue if a sync replication peer is in STANDBY state, or is<a name="line.973"></a>
+<span class="sourceLineNo">974</span> // transiting to STANDBY state. The only exception is we are in STANDBY state and<a name="line.974"></a>
+<span class="sourceLineNo">975</span> // transiting to DA, under this state we will replay the remote WAL and they need to be<a name="line.975"></a>
+<span class="sourceLineNo">976</span> // replicated back.<a name="line.976"></a>
+<span class="sourceLineNo">977</span> if (peer.getPeerConfig().isSyncReplication()) {<a name="line.977"></a>
+<span class="sourceLineNo">978</span> Pair<SyncReplicationState, SyncReplicationState> stateAndNewState =<a name="line.978"></a>
+<span class="sourceLineNo">979</span> peer.getSyncReplicationStateAndNewState();<a name="line.979"></a>
+<span class="sourceLineNo">980</span> if ((stateAndNewState.getFirst().equals(SyncReplicationState.STANDBY) &&<a name="line.980"></a>
+<span class="sourceLineNo">981</span> stateAndNewState.getSecond().equals(SyncReplicationState.NONE)) ||<a name="line.981"></a>
+<span class="sourceLineNo">982</span> stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)) {<a name="line.982"></a>
+<span class="sourceLineNo">983</span> src.terminate("Sync replication peer is in STANDBY state");<a name="line.983"></a>
+<span class="sourceLineNo">984</span> deleteQueue(queueId);<a name="line.984"></a>
+<span class="sourceLineNo">985</span> continue;<a name="line.985"></a>
+<span class="sourceLineNo">986</span> }<a name="line.986"></a>
+<span class="sourceLineNo">987</span> }<a name="line.987"></a>
+<span class="sourceLineNo">988</span> // track sources in walsByIdRecoveredQueues<a name="line.988"></a>
+<span class="sourceLineNo">989</span> Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();<a name="line.989"></a>
+<span class="sourceLineNo">990</span> walsByIdRecoveredQueues.put(queueId, walsByGroup);<a name="line.990"></a>
+<span class="sourceLineNo">991</span> for (String wal : walsSet) {<a name="line.991"></a>
+<span class="sourceLineNo">992</span> String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal);<a name="line.992"></a>
+<span class="sourceLineNo">993</span> NavigableSet<String> wals = walsByGroup.get(walPrefix);<a name="line.993"></a>
+<span class="sourceLineNo">994</span> if (wals == null) {<a name="line.994"></a>
+<span class="sourceLineNo">995</span> wals = new TreeSet<>();<a name="line.995"></a>
+<span class="sourceLineNo">996</span> walsByGroup.put(walPrefix, wals);<a name="line.996"></a>
+<span class="sourceLineNo">997</span> }<a name="line.997"></a>
+<span class="sourceLineNo">998</span> wals.add(wal);<a name="line.998"></a>
+<span class="sourceLineNo">999</span> }<a name="line.999"></a>
+<span class="sourceLineNo">1000</span> oldsources.add(src);<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span> LOG.info("Added source for recovered queue {}", src.getQueueId());<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span> for (String wal : walsSet) {<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span> LOG.trace("Enqueueing log from recovered queue for source: " + src.getQueueId());<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span> src.enqueueLog(new Path(oldLogDir, wal));<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span> src.startup();<a name="line.1006"></a>
+<span class="sourceLineNo">1007</span> }<a name="line.1007"></a>
+<span class="sourceLineNo">1008</span> } catch (IOException e) {<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span> // TODO manage it<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span> LOG.error("Failed creating a source", e);<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span> }<a name="line.1011"></a>
+<span class="sourceLineNo">1012</span> }<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span> }<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span><a name="line.1015"></a>
+<span class="sourceLineNo">1016</span> /**<a name="line.1016"></a>
+<span class="sourceLineNo">1017</span> * Terminate the replication on this region server<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span> */<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span> public void join() {<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span> this.executor.shutdown();<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span> source.terminate("Region server is closing");<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
+<span class="sourceLineNo">1024</span> }<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span><a name="line.1025"></a>
+<span class="sourceLineNo">1026</span> /**<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span> * Get a copy of the wals of the normal sources on this rs<a name="line.1027"></a>
+<span class="sourceLineNo">1028</span> * @return a sorted set of wal names<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span> */<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span> public Map<String, Map<String, NavigableSet<String>>> getWALs() {<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span> return Collections.unmodifiableMap(walsById);<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span><a name="line.1033"></a>
+<span class="sourceLineNo">1034</span> /**<a name="line.1034"></a>
+<span class="sourceLineNo">1035</span> * Get a copy of the wals of the recovered sources on this rs<a name="line.1035"></a>
+<span class="sourceLineNo">1036</span> * @return a sorted set of wal names<a name="line.1036"></a>
+<span class="sourceLineNo">1037</span> */<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span> Map<String, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span> return Collections.unmodifiableMap(walsByIdRecoveredQueues);<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span> }<a name="line.1040"></a>
+<span class="sourceLineNo">1041</span><a name="line.1041"></a>
+<span class="sourceLineNo">1042</span> /**<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span> * Get a list of all the normal sources of this rs<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span> * @return list of all normal sources<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span> */<a name="line.1045"></a>
+<span class="sourceLineNo">1046</span> public List<ReplicationSourceInterface> getSources() {<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span> return new ArrayList<>(this.sources.values());<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span> }<a name="line.1048"></a>
+<span class="sourceLineNo">1049</span><a name="line.1049"></a>
+<span class="sourceLineNo">1050</span> /**<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span> * Get a list of all the recovered sources of this rs<a name="line.1051"></a>
+<span class="sourceLineNo">1052</span> * @return list of all recovered sources<a name="line.1052"></a>
+<span class="sourceLineNo">1053</span> */<a name="line.1053"></a>
+<span class="sourceLineNo">1054</span> public List<ReplicationSourceInterface> getOldSources() {<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span> return this.oldsources;<a name="line.1055"></a>
+<span class="sourceLineNo">1056</span> }<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span><a name="line.1057"></a>
+<span class="sourceLineNo">1058</span> /**<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span> * Get the normal source for a given peer<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span> * @return the normal source for the give peer if it exists, otherwise null.<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span> */<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span> public ReplicationSourceInterface getSource(String peerId) {<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span> return this.sources.get(peerId);<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span> }<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span><a name="line.1065"></a>
+<span class="sourceLineNo">1066</span> List<String> getAllQueues() throws IOException {<a name="line.1066"></a>
+<span class="sourceLineNo">1067</span> List<String> allQueues = Collections.emptyList();<a name="line.1067"></a>
+<span class="sourceLineNo">1068</span> try {<a name="line.1068"></a>
+<span class="sourceLineNo">1069</span> allQueues = queueStorage.getAllQueues(server.getServerName());<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span> } catch (ReplicationException e) {<a name="line.1070"></a>
+<span class="sourceLineNo">1071</span> throw new IOException(e);<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span> }<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span> return allQueues;<a name="line.1073"></a>
+<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span><a name="line.1075"></a>
+<span class="sourceLineNo">1076</span> int getSizeOfLatestPath() {<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span> synchronized (latestPaths) {<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span> return latestPaths.size();<a name="line.1078"></a>
+<span class="sourceLineNo">1079</span> }<a name="line.1079"></a>
+<span class="sourceLineNo">1080</span> }<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span><a name="line.1081"></a>
+<span class="sourceLineNo">1082</span> Set<Path> getLastestPath() {<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span> synchronized (latestPaths) {<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span> return Sets.newHashSet(latestPaths.values());<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span> }<a name="line.1085"></a>
+<span class="sourceLineNo">1086</span> }<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span><a name="line.1087"></a>
+<span class="sourceLineNo">1088</span> public AtomicLong getTotalBufferUsed() {<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span> return totalBufferUsed;<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span> }<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span><a name="line.1091"></a>
+<span class="sourceLineNo">1092</span> /**<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span> * Returns the maximum size in bytes of edits held in memory which are pending replication<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span> * across all sources inside this RegionServer.<a name="line.1094"></a>
+<span class="sourceLineNo">1095</span> */<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span> public long getTotalBufferLimit() {<a name="line.1096"></a>
+<span class="sourceLineNo">1097</span> return totalBufferLimit;<a name="line.1097"></a>
+<span class="sourceLineNo">1098</span> }<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span><a name="line.1099"></a>
+<span class="sourceLineNo">1100</span> /**<a name="line.1100"></a>
+<span class="sourceLineNo">1101</span> * Get the directory where wals are archived<a name="line.1101"></a>
+<span class="sourceLineNo">1102</span> * @return the directory where wals are archived<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span> */<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span> public Path getOldLogDir() {<a name="line.1104"></a>
+<span class="sourceLineNo">1105</span> return this.oldLogDir;<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span> }<a name="line.1106"></a>
+<span class="sourceLineNo">1107</span><a name="line.1107"></a>
+<span class="sourceLineNo">1108</span> /**<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span> * Get the directory where wals are stored by their RSs<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span> * @return the directory where wals are stored by their RSs<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span> */<a name="line.1111"></a>
+<span class="sourceLineNo">1112</span> public Path getLogDir() {<a name="line.1112"></a>
+<span class="sourceLineNo">1113</span> return this.logDir;<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span> }<a name="line.1114"></a>
+<span class="sourceLineNo">1115</span><a name="line.1115"></a>
+<span class="sourceLineNo">1116</span> /**<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span> * Get the handle on the local file system<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span> * @return Handle on the local file system<a name="line.1118"></a>
+<span class="sourceLineNo">1119</span> */<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span> public FileSystem getFs() {<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span> return this.fs;<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span> }<a name="line.1122"></a>
+<span class="sourceLineNo">1123</span><a name="line.1123"></a>
+<span class="sourceLineNo">1124</span> /**<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span> * Get the ReplicationPeers used by this ReplicationSourceManager<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span> * @return the ReplicationPeers used by this ReplicationSourceManager<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span> */<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span> public ReplicationPeers getReplicationPeers() {<a name="line.1128"></a>
+<span class="sourceLineNo">1129</span> return this.replicationPeers;<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span> }<a name="line.1130"></a>
+<span class="sourceLineNo">1131</span><a name="line.1131"></a>
+<span class="sourceLineNo">1132</span> /**<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span> * Get a string representation of all the sources' metrics<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span> */<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span> public String getStats() {<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span> StringBuilder stats = new StringBuilder();<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span> // Print stats that apply across all Replication Sources<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span> stats.append("Global stats: ");<a name="line.1138"></a>
+<span class="sourceLineNo">1139</span> stats.append("WAL Edits Buffer Used=").append(getTotalBufferUsed().get()).append("B, Limit=")<a name="line.1139"></a>
+<span class="sourceLineNo">1140</span> .append(getTotalBufferLimit()).append("B\n");<a name="line.1140"></a>
+<span class="sourceLineNo">1141</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span> stats.append("Normal source for cluster " + source.getPeerId() + ": ");<a name="line.1142"></a>
+<span class="sourceLineNo">1143</span> stats.append(source.getStats() + "\n");<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span> }<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span> for (ReplicationSourceInterface oldSource : oldsources) {<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span> stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerId() + ": ");<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span> stats.append(oldSource.getStats() + "\n");<a name="line.1147"></a>
+<span class="sourceLineNo">1148</span> }<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span> return stats.toString();<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span> }<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span><a name="line.1151"></a>
+<span class="sourceLineNo">1152</span> public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span> throws IOException {<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span> for (ReplicationSourceInterface source : this.sources.values()) {<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span> throwIOExceptionWhenFail(() -> source.addHFileRefs(tableName, family, pairs));<a name="line.1155"></a>
+<span class="sourceLineNo">1156</span> }<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span> }<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span><a name="line.1158"></a>
+<span class="sourceLineNo">1159</span> public void cleanUpHFileRefs(String peerId, List<String> files) {<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span> interruptOrAbortWhenFail(() -> this.queueStorage.removeHFileRefs(peerId, files));<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span> }<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span><a name="line.1162"></a>
+<span class="sourceLineNo">1163</span> int activeFailoverTaskCount() {<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span> return executor.getActiveCount();<a name="line.1164"></a>
+<span class="sourceLineNo">1165</span> }<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span><a name="line.1166"></a>
+<span class="sourceLineNo">1167</span> MetricsReplicationGlobalSourceSource getGlobalMetrics() {<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span> return this.globalMetrics;<a name="line.1168"></a>
+<span class="sourceLineNo">1169</span> }<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span><a name="line.1170"></a>
+<span class="sourceLineNo">1171</span> /**<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span> * Add an hbase:meta Catalog replication source. Called on open of an hbase:meta Region.<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span> * Create it once only. If exists already, use the existing one.<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span> * @see #removeCatalogReplicationSource(RegionInfo)<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span> * @see #addSource(String) This is specialization on the addSource method.<a name="line.1175"></a>
+<span class="sourceLineNo">1176</span> */<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span> public ReplicationSourceInterface addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span> throws IOException {<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span> // Poor-man's putIfAbsent<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span> synchronized (this.catalogReplicationSource) {<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span> ReplicationSourceInterface rs = this.catalogReplicationSource.get();<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span> return rs != null ? rs :<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span> this.catalogReplicationSource.getAndSet(createCatalogReplicationSource(regionInfo));<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span> }<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span> }<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span><a name="line.1186"></a>
+<span class="sourceLineNo">1187</span> /**<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span> * Remove the hbase:meta Catalog replication source.<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span> * Called when we close hbase:meta.<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span> * @see #addCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span> */<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span> public void removeCatalogReplicationSource(RegionInfo regionInfo) {<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span> // Nothing to do. Leave any CatalogReplicationSource in place in case an hbase:meta Region<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span> // comes back to this server.<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span> }<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span><a name="line.1196"></a>
+<span class="sourceLineNo">1197</span> /**<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span> * Create, initialize, and start the Catalog ReplicationSource.<a name="line.1198"></a>
+<span class="sourceLineNo">1199</span> * Presumes called one-time only (caller must ensure one-time only call).<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span> * This ReplicationSource is NOT created via {@link ReplicationSourceFactory}.<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span> * @see #addSource(String) This is a specialization of the addSource call.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span> * @see #catalogReplicationSource for a note on this ReplicationSource's lifecycle (and more on<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span> * why the special handling).<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span> */<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span> private ReplicationSourceInterface createCatalogReplicationSource(RegionInfo regionInfo)<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span> throws IOException {<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span> // Instantiate meta walProvider. Instantiated here or over in the #warmupRegion call made by the<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span> // Master on a 'move' operation. Need to do extra work if we did NOT instantiate the provider.<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span> WALProvider walProvider = this.walFactory.getMetaWALProvider();<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span> boolean instantiate = walProvider == null;<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span> if (instantiate) {<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span> walProvider = this.walFactory.getMetaProvider();<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span> }<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span> // Here we do a specialization on what {@link ReplicationSourceFactory} does. There is no need<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span> // for persisting offset into WALs up in zookeeper (via ReplicationQueueInfo) as the catalog<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span> // read replicas feature that makes use of the source does a reset on a crash of the WAL<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span> // source process. See "4.1 Skip maintaining zookeeper replication queue (offsets/WALs)" in the<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span> // design doc attached to HBASE-18070 'Enable memstore replication for meta replica' for detail.<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span> CatalogReplicationSourcePeer peer = new CatalogReplicationSourcePeer(this.conf,<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span> this.clusterId.toString(), "meta_" + ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER);<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span> final ReplicationSourceInterface crs = new CatalogReplicationSource();<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span> crs.init(conf, fs, this, new NoopReplicationQueueStorage(), peer, server, peer.getId(),<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span> clusterId, walProvider.getWALFileLengthProvider(), new MetricsSource(peer.getId()));<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span> // Add listener on the provider so we can pick up the WAL to replicate on roll.<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span> WALActionsListener listener = new WALActionsListener() {<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span> @Override public void postLogRoll(Path oldPath, Path newPath) throws IOException {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span> crs.enqueueLog(newPath);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span> }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span> };<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span> walProvider.addWALActionsListener(listener);<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span> if (!instantiate) {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span> // If we did not instantiate provider, need to add our listener on already-created WAL<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span> // instance too (listeners are passed by provider to WAL instance on creation but if provider<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span> // created already, our listener add above is missed). And add the current WAL file to the<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span> // Replication Source so it can start replicating it.<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span> WAL wal = walProvider.getWAL(regionInfo);<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span> wal.registerWALActionsListener(listener);<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span> crs.enqueueLog(((AbstractFSWAL)wal).getCurrentFileName());<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span> }<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span> return crs.startup();<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span> }<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span>}<a name="line.1242"></a>
diff --git a/downloads.html b/downloads.html
index ae36e73..4a145e9 100644
--- a/downloads.html
+++ b/downloads.html
@@ -462,7 +462,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/export_control.html b/export_control.html
index 0e52074..772c16f 100644
--- a/export_control.html
+++ b/export_control.html
@@ -197,7 +197,7 @@ for more details.</p>
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/index.html b/index.html
index 60039ce..9d6b8f7 100644
--- a/index.html
+++ b/index.html
@@ -275,7 +275,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/issue-tracking.html b/issue-tracking.html
index a6cef09..75144af 100644
--- a/issue-tracking.html
+++ b/issue-tracking.html
@@ -169,7 +169,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/mail-lists.html b/mail-lists.html
index 35b48ee..71dcf4a 100644
--- a/mail-lists.html
+++ b/mail-lists.html
@@ -229,7 +229,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/metrics.html b/metrics.html
index dbfd66d..c691e9d 100644
--- a/metrics.html
+++ b/metrics.html
@@ -325,7 +325,7 @@ export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxrem
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/old_news.html b/old_news.html
index 29e4cc7..f42d33b 100644
--- a/old_news.html
+++ b/old_news.html
@@ -316,7 +316,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/plugin-management.html b/plugin-management.html
index 7d09955..9c29bb5 100644
--- a/plugin-management.html
+++ b/plugin-management.html
@@ -321,7 +321,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/plugins.html b/plugins.html
index 692448b..e4c68d7 100644
--- a/plugins.html
+++ b/plugins.html
@@ -248,7 +248,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/poweredbyhbase.html b/poweredbyhbase.html
index 2bf5caa..f2904f7 100644
--- a/poweredbyhbase.html
+++ b/poweredbyhbase.html
@@ -650,7 +650,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/project-info.html b/project-info.html
index c9e8e0c..b3506b4 100644
--- a/project-info.html
+++ b/project-info.html
@@ -210,7 +210,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/project-reports.html b/project-reports.html
index 2fab532..7cce3bb 100644
--- a/project-reports.html
+++ b/project-reports.html
@@ -186,7 +186,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/project-summary.html b/project-summary.html
index 78adc09..80607d6 100644
--- a/project-summary.html
+++ b/project-summary.html
@@ -212,7 +212,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/pseudo-distributed.html b/pseudo-distributed.html
index 17f81c8..993d4fe 100644
--- a/pseudo-distributed.html
+++ b/pseudo-distributed.html
@@ -174,7 +174,7 @@ Running Apache HBase (TM) in pseudo-distributed mode
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/replication.html b/replication.html
index c6976f7..337206f 100644
--- a/replication.html
+++ b/replication.html
@@ -169,7 +169,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/resources.html b/resources.html
index e94c2d2..f48126c 100644
--- a/resources.html
+++ b/resources.html
@@ -197,7 +197,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/source-repository.html b/source-repository.html
index 0d1fc0b..a22af72 100644
--- a/source-repository.html
+++ b/source-repository.html
@@ -180,7 +180,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/sponsors.html b/sponsors.html
index 4721727..51a34c3 100644
--- a/sponsors.html
+++ b/sponsors.html
@@ -199,7 +199,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/supportingprojects.html b/supportingprojects.html
index 2e028af..9e9fef6 100644
--- a/supportingprojects.html
+++ b/supportingprojects.html
@@ -390,7 +390,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/team-list.html b/team-list.html
index 242bb37..8106040 100644
--- a/team-list.html
+++ b/team-list.html
@@ -719,7 +719,7 @@
<div class="row">
<p>Copyright ©2007–2021
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-05</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2021-02-07</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.DummyReplicationListener.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.DummyReplicationListener.html
index c87c8d4..db4c822 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.DummyReplicationListener.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.DummyReplicationListener.html
@@ -123,26 +123,26 @@
<span class="sourceLineNo">115</span> assertEquals(0, rt.getListOfRegionServers().size());<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // 1 region server<a name="line.117"></a>
-<span class="sourceLineNo">118</span> ZKUtil.createWithParents(zkw,<a name="line.118"></a>
-<span class="sourceLineNo">119</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname1.example.org:1234"));<a name="line.119"></a>
-<span class="sourceLineNo">120</span> List<String> rss = rt.getListOfRegionServers();<a name="line.120"></a>
+<span class="sourceLineNo">118</span> ZKUtil.createWithParents(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.118"></a>
+<span class="sourceLineNo">119</span> "hostname1.example.org,1234,1611218678009"));<a name="line.119"></a>
+<span class="sourceLineNo">120</span> List<ServerName> rss = rt.getListOfRegionServers();<a name="line.120"></a>
<span class="sourceLineNo">121</span> assertEquals(rss.toString(), 1, rss.size());<a name="line.121"></a>
<span class="sourceLineNo">122</span><a name="line.122"></a>
<span class="sourceLineNo">123</span> // 2 region servers<a name="line.123"></a>
-<span class="sourceLineNo">124</span> ZKUtil.createWithParents(zkw,<a name="line.124"></a>
-<span class="sourceLineNo">125</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname2.example.org:1234"));<a name="line.125"></a>
+<span class="sourceLineNo">124</span> ZKUtil.createWithParents(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.124"></a>
+<span class="sourceLineNo">125</span> "hostname2.example.org,1234,1611218678009"));<a name="line.125"></a>
<span class="sourceLineNo">126</span> rss = rt.getListOfRegionServers();<a name="line.126"></a>
<span class="sourceLineNo">127</span> assertEquals(rss.toString(), 2, rss.size());<a name="line.127"></a>
<span class="sourceLineNo">128</span><a name="line.128"></a>
<span class="sourceLineNo">129</span> // 1 region server<a name="line.129"></a>
-<span class="sourceLineNo">130</span> ZKUtil.deleteNode(zkw,<a name="line.130"></a>
-<span class="sourceLineNo">131</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname2.example.org:1234"));<a name="line.131"></a>
+<span class="sourceLineNo">130</span> ZKUtil.deleteNode(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.130"></a>
+<span class="sourceLineNo">131</span> "hostname2.example.org,1234,1611218678009"));<a name="line.131"></a>
<span class="sourceLineNo">132</span> rss = rt.getListOfRegionServers();<a name="line.132"></a>
<span class="sourceLineNo">133</span> assertEquals(1, rss.size());<a name="line.133"></a>
<span class="sourceLineNo">134</span><a name="line.134"></a>
<span class="sourceLineNo">135</span> // 0 region server<a name="line.135"></a>
-<span class="sourceLineNo">136</span> ZKUtil.deleteNode(zkw,<a name="line.136"></a>
-<span class="sourceLineNo">137</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname1.example.org:1234"));<a name="line.137"></a>
+<span class="sourceLineNo">136</span> ZKUtil.deleteNode(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.136"></a>
+<span class="sourceLineNo">137</span> "hostname1.example.org,1234,1611218678009"));<a name="line.137"></a>
<span class="sourceLineNo">138</span> rss = rt.getListOfRegionServers();<a name="line.138"></a>
<span class="sourceLineNo">139</span> assertEquals(rss.toString(), 0, rss.size());<a name="line.139"></a>
<span class="sourceLineNo">140</span> }<a name="line.140"></a>
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.DummyServer.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.DummyServer.html
index c87c8d4..db4c822 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.DummyServer.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.DummyServer.html
@@ -123,26 +123,26 @@
<span class="sourceLineNo">115</span> assertEquals(0, rt.getListOfRegionServers().size());<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // 1 region server<a name="line.117"></a>
-<span class="sourceLineNo">118</span> ZKUtil.createWithParents(zkw,<a name="line.118"></a>
-<span class="sourceLineNo">119</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname1.example.org:1234"));<a name="line.119"></a>
-<span class="sourceLineNo">120</span> List<String> rss = rt.getListOfRegionServers();<a name="line.120"></a>
+<span class="sourceLineNo">118</span> ZKUtil.createWithParents(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.118"></a>
+<span class="sourceLineNo">119</span> "hostname1.example.org,1234,1611218678009"));<a name="line.119"></a>
+<span class="sourceLineNo">120</span> List<ServerName> rss = rt.getListOfRegionServers();<a name="line.120"></a>
<span class="sourceLineNo">121</span> assertEquals(rss.toString(), 1, rss.size());<a name="line.121"></a>
<span class="sourceLineNo">122</span><a name="line.122"></a>
<span class="sourceLineNo">123</span> // 2 region servers<a name="line.123"></a>
-<span class="sourceLineNo">124</span> ZKUtil.createWithParents(zkw,<a name="line.124"></a>
-<span class="sourceLineNo">125</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname2.example.org:1234"));<a name="line.125"></a>
+<span class="sourceLineNo">124</span> ZKUtil.createWithParents(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.124"></a>
+<span class="sourceLineNo">125</span> "hostname2.example.org,1234,1611218678009"));<a name="line.125"></a>
<span class="sourceLineNo">126</span> rss = rt.getListOfRegionServers();<a name="line.126"></a>
<span class="sourceLineNo">127</span> assertEquals(rss.toString(), 2, rss.size());<a name="line.127"></a>
<span class="sourceLineNo">128</span><a name="line.128"></a>
<span class="sourceLineNo">129</span> // 1 region server<a name="line.129"></a>
-<span class="sourceLineNo">130</span> ZKUtil.deleteNode(zkw,<a name="line.130"></a>
-<span class="sourceLineNo">131</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname2.example.org:1234"));<a name="line.131"></a>
+<span class="sourceLineNo">130</span> ZKUtil.deleteNode(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.130"></a>
+<span class="sourceLineNo">131</span> "hostname2.example.org,1234,1611218678009"));<a name="line.131"></a>
<span class="sourceLineNo">132</span> rss = rt.getListOfRegionServers();<a name="line.132"></a>
<span class="sourceLineNo">133</span> assertEquals(1, rss.size());<a name="line.133"></a>
<span class="sourceLineNo">134</span><a name="line.134"></a>
<span class="sourceLineNo">135</span> // 0 region server<a name="line.135"></a>
-<span class="sourceLineNo">136</span> ZKUtil.deleteNode(zkw,<a name="line.136"></a>
-<span class="sourceLineNo">137</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname1.example.org:1234"));<a name="line.137"></a>
+<span class="sourceLineNo">136</span> ZKUtil.deleteNode(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.136"></a>
+<span class="sourceLineNo">137</span> "hostname1.example.org,1234,1611218678009"));<a name="line.137"></a>
<span class="sourceLineNo">138</span> rss = rt.getListOfRegionServers();<a name="line.138"></a>
<span class="sourceLineNo">139</span> assertEquals(rss.toString(), 0, rss.size());<a name="line.139"></a>
<span class="sourceLineNo">140</span> }<a name="line.140"></a>
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.html
index c87c8d4..db4c822 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.html
@@ -123,26 +123,26 @@
<span class="sourceLineNo">115</span> assertEquals(0, rt.getListOfRegionServers().size());<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // 1 region server<a name="line.117"></a>
-<span class="sourceLineNo">118</span> ZKUtil.createWithParents(zkw,<a name="line.118"></a>
-<span class="sourceLineNo">119</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname1.example.org:1234"));<a name="line.119"></a>
-<span class="sourceLineNo">120</span> List<String> rss = rt.getListOfRegionServers();<a name="line.120"></a>
+<span class="sourceLineNo">118</span> ZKUtil.createWithParents(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.118"></a>
+<span class="sourceLineNo">119</span> "hostname1.example.org,1234,1611218678009"));<a name="line.119"></a>
+<span class="sourceLineNo">120</span> List<ServerName> rss = rt.getListOfRegionServers();<a name="line.120"></a>
<span class="sourceLineNo">121</span> assertEquals(rss.toString(), 1, rss.size());<a name="line.121"></a>
<span class="sourceLineNo">122</span><a name="line.122"></a>
<span class="sourceLineNo">123</span> // 2 region servers<a name="line.123"></a>
-<span class="sourceLineNo">124</span> ZKUtil.createWithParents(zkw,<a name="line.124"></a>
-<span class="sourceLineNo">125</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname2.example.org:1234"));<a name="line.125"></a>
+<span class="sourceLineNo">124</span> ZKUtil.createWithParents(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.124"></a>
+<span class="sourceLineNo">125</span> "hostname2.example.org,1234,1611218678009"));<a name="line.125"></a>
<span class="sourceLineNo">126</span> rss = rt.getListOfRegionServers();<a name="line.126"></a>
<span class="sourceLineNo">127</span> assertEquals(rss.toString(), 2, rss.size());<a name="line.127"></a>
<span class="sourceLineNo">128</span><a name="line.128"></a>
<span class="sourceLineNo">129</span> // 1 region server<a name="line.129"></a>
-<span class="sourceLineNo">130</span> ZKUtil.deleteNode(zkw,<a name="line.130"></a>
-<span class="sourceLineNo">131</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname2.example.org:1234"));<a name="line.131"></a>
+<span class="sourceLineNo">130</span> ZKUtil.deleteNode(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.130"></a>
+<span class="sourceLineNo">131</span> "hostname2.example.org,1234,1611218678009"));<a name="line.131"></a>
<span class="sourceLineNo">132</span> rss = rt.getListOfRegionServers();<a name="line.132"></a>
<span class="sourceLineNo">133</span> assertEquals(1, rss.size());<a name="line.133"></a>
<span class="sourceLineNo">134</span><a name="line.134"></a>
<span class="sourceLineNo">135</span> // 0 region server<a name="line.135"></a>
-<span class="sourceLineNo">136</span> ZKUtil.deleteNode(zkw,<a name="line.136"></a>
-<span class="sourceLineNo">137</span> ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, "hostname1.example.org:1234"));<a name="line.137"></a>
+<span class="sourceLineNo">136</span> ZKUtil.deleteNode(zkw, ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode,<a name="line.136"></a>
+<span class="sourceLineNo">137</span> "hostname1.example.org,1234,1611218678009"));<a name="line.137"></a>
<span class="sourceLineNo">138</span> rss = rt.getListOfRegionServers();<a name="line.138"></a>
<span class="sourceLineNo">139</span> assertEquals(rss.toString(), 0, rss.size());<a name="line.139"></a>
<span class="sourceLineNo">140</span> }<a name="line.140"></a>