You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2019/01/14 22:25:49 UTC
[hbase] 01/02: HBASE-16499 slow replication for small HBase clusters
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit feb8a57a9a576a7e311a7447480b00bf23ab46b1
Author: Ashish Singhi <as...@apache.org>
AuthorDate: Wed Apr 4 09:59:50 2018 +0530
HBASE-16499 slow replication for small HBase clusters
Signed-off-by: Ashish Singhi <as...@apache.org>
---
.../regionserver/ReplicationSinkManager.java | 2 +-
.../regionserver/TestReplicationSinkManager.java | 36 ++++++++++++----------
2 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
index 0469f9b..c5977a3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
@@ -54,7 +54,7 @@ public class ReplicationSinkManager {
* Default ratio of the total number of peer cluster region servers to consider
* replicating to.
*/
- static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.1f;
+ static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.5f;
private final HConnection conn;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java
index c0b7d0c..efee9af 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.regionserver.ReplicationSinkManager.SinkPeer;
import org.junit.Before;
import org.junit.Test;
@@ -42,13 +41,11 @@ public class TestReplicationSinkManager {
private static final String PEER_CLUSTER_ID = "PEER_CLUSTER_ID";
- private ReplicationPeers replicationPeers;
private HBaseReplicationEndpoint replicationEndpoint;
private ReplicationSinkManager sinkManager;
@Before
public void setUp() {
- replicationPeers = mock(ReplicationPeers.class);
replicationEndpoint = mock(HBaseReplicationEndpoint.class);
sinkManager = new ReplicationSinkManager(mock(HConnection.class),
PEER_CLUSTER_ID, replicationEndpoint, new Configuration());
@@ -57,7 +54,8 @@ public class TestReplicationSinkManager {
@Test
public void testChooseSinks() {
List<ServerName> serverNames = Lists.newArrayList();
- for (int i = 0; i < 20; i++) {
+ int totalServers = 20;
+ for (int i = 0; i < totalServers; i++) {
serverNames.add(mock(ServerName.class));
}
@@ -66,7 +64,8 @@ public class TestReplicationSinkManager {
sinkManager.chooseSinks();
- assertEquals(2, sinkManager.getNumSinks());
+ int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO);
+ assertEquals(expected, sinkManager.getNumSinks());
}
@@ -110,7 +109,8 @@ public class TestReplicationSinkManager {
@Test
public void testReportBadSink_PastThreshold() {
List<ServerName> serverNames = Lists.newArrayList();
- for (int i = 0; i < 30; i++) {
+ int totalServers = 30;
+ for (int i = 0; i < totalServers; i++) {
serverNames.add(mock(ServerName.class));
}
when(replicationEndpoint.getRegionServers())
@@ -119,7 +119,8 @@ public class TestReplicationSinkManager {
sinkManager.chooseSinks();
// Sanity check
- assertEquals(3, sinkManager.getNumSinks());
+ int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO);
+ assertEquals(expected, sinkManager.getNumSinks());
ServerName serverName = sinkManager.getSinksForTesting().get(0);
@@ -132,7 +133,7 @@ public class TestReplicationSinkManager {
// Reporting a bad sink more than the threshold count should remove it
// from the list of potential sinks
- assertEquals(2, sinkManager.getNumSinks());
+ assertEquals(expected - 1, sinkManager.getNumSinks());
//
// now try a sink that has some successes
@@ -147,23 +148,24 @@ public class TestReplicationSinkManager {
sinkManager.reportBadSink(sinkPeer);
// did not remove the sink, since we had one successful try
- assertEquals(2, sinkManager.getNumSinks());
+ assertEquals(expected - 1, sinkManager.getNumSinks());
for (int i = 0; i <= ReplicationSinkManager.DEFAULT_BAD_SINK_THRESHOLD-2; i++) {
sinkManager.reportBadSink(sinkPeer);
}
// still not remove, since the success reset the counter
- assertEquals(2, sinkManager.getNumSinks());
+ assertEquals(expected - 1, sinkManager.getNumSinks());
sinkManager.reportBadSink(sinkPeer);
// but we exhausted the tries
- assertEquals(1, sinkManager.getNumSinks());
+ assertEquals(expected - 2, sinkManager.getNumSinks());
}
@Test
public void testReportBadSink_DownToZeroSinks() {
List<ServerName> serverNames = Lists.newArrayList();
- for (int i = 0; i < 20; i++) {
+ int totalServers = 4;
+ for (int i = 0; i < totalServers; i++) {
serverNames.add(mock(ServerName.class));
}
when(replicationEndpoint.getRegionServers())
@@ -172,9 +174,9 @@ public class TestReplicationSinkManager {
sinkManager.chooseSinks();
// Sanity check
-
List<ServerName> sinkList = sinkManager.getSinksForTesting();
- assertEquals(2, sinkList.size());
+ int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO);
+ assertEquals(expected, sinkList.size());
ServerName serverNameA = sinkList.get(0);
ServerName serverNameB = sinkList.get(1);
@@ -188,8 +190,10 @@ public class TestReplicationSinkManager {
}
// We've gone down to 0 good sinks, so the replication sinks
- // should have been refreshed now
- assertEquals(2, sinkManager.getNumSinks());
+ // should have been refreshed now, so out of 4 servers, 2 are not considered as they are
+ // reported as bad.
+ expected = (int) ((totalServers - 2) * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO);
+ assertEquals(expected, sinkManager.getNumSinks());
}
}