You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2019/01/14 22:25:48 UTC
[hbase] branch branch-1 updated (a3e274c -> 1889146)
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a change to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from a3e274c HBASE-21196 HTableMultiplexer clears the meta cache after every put operation
new feb8a57 HBASE-16499 slow replication for small HBase clusters
new 1889146 HBASE-20986 Separate the config of block size when we do log splitting and write Hlog
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/hbase/regionserver/wal/FSHLog.java | 4 +--
.../hbase/regionserver/wal/ProtobufLogWriter.java | 3 +-
.../hadoop/hbase/regionserver/wal/WALUtil.java | 29 +++++++++++++++++
.../regionserver/ReplicationSinkManager.java | 2 +-
.../regionserver/TestReplicationSinkManager.java | 36 ++++++++++++----------
5 files changed, 52 insertions(+), 22 deletions(-)
[hbase] 02/02: HBASE-20986 Separate the config of block size when
we do log splitting and write Hlog
Posted by ap...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 18891467faf8f5220b32e4e2d5164e94b0a8b6ce
Author: jingyuntian <ti...@gmail.com>
AuthorDate: Fri Aug 3 16:10:56 2018 +0800
HBASE-20986 Separate the config of block size when we do log splitting and write Hlog
Signed-off-by: Guanghao Zhang <zg...@apache.org>
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
Amending-Author: Andrew Purtell <ap...@apache.org>
---
.../hadoop/hbase/regionserver/wal/FSHLog.java | 4 +--
.../hbase/regionserver/wal/ProtobufLogWriter.java | 3 +--
.../hadoop/hbase/regionserver/wal/WALUtil.java | 29 ++++++++++++++++++++++
3 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 423ceb2..b46ad0f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -34,7 +34,6 @@ import java.util.Map;
import java.util.NavigableMap;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
@@ -511,8 +510,7 @@ public class FSHLog implements WAL {
// Get size to roll log at. Roll at 95% of HDFS block size so we avoid crossing HDFS blocks
// (it costs a little x'ing bocks)
- final long blocksize = this.conf.getLong("hbase.regionserver.hlog.blocksize",
- FSUtils.getDefaultBlockSize(this.fs, this.fullPathLogDir));
+ final long blocksize = WALUtil.getWALBlockSize(conf, fs, fullPathLogDir, false);
this.logrollsize =
(long)(blocksize * conf.getFloat("hbase.regionserver.logroll.multiplier", 0.95f));
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
index 2e4226f..2bfc6a8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogWriter.java
@@ -85,8 +85,7 @@ public class ProtobufLogWriter extends WriterBase {
int bufferSize = FSUtils.getDefaultBufferSize(fs);
short replication = (short)conf.getInt(
"hbase.regionserver.hlog.replication", FSUtils.getDefaultReplication(fs, path));
- long blockSize = conf.getLong("hbase.regionserver.hlog.blocksize",
- FSUtils.getDefaultBlockSize(fs, path));
+ long blockSize = WALUtil.getWALBlockSize(conf, fs, path);
output = fs.createNonRecursive(path, overwritable, bufferSize, replication, blockSize, null);
output.write(ProtobufLogReader.PB_WAL_MAGIC);
boolean doTagCompress = doCompress
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java
index c89a466..f47d3ef 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java
@@ -23,6 +23,9 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
@@ -31,6 +34,7 @@ import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALKey;
@@ -133,4 +137,29 @@ public class WALUtil {
}
return trx;
}
+
+ /**
+ * Blocksize returned here is 2x the default HDFS blocksize unless explicitly set in
+ * Configuration. Works in tandem with hbase.regionserver.logroll.multiplier. See comment in
+ * AbstractFSWAL in Constructor where we set blocksize and logrollsize for why.
+ * @return Blocksize to use writing WALs.
+ */
+ public static long getWALBlockSize(Configuration conf, FileSystem fs, Path dir)
+ throws IOException {
+ return getWALBlockSize(conf, fs, dir, false);
+ }
+
+ /**
+ * Public because of FSHLog. Should be package-private
+ * @param isRecoverEdits the created writer is for recovered edits or WAL. For recovered edits, it
+ * is true and for WAL it is false.
+ */
+ public static long getWALBlockSize(Configuration conf, FileSystem fs, Path dir,
+ boolean isRecoverEdits) throws IOException {
+ long defaultBlockSize = FSUtils.getDefaultBlockSize(fs, dir) * 2;
+ if (isRecoverEdits) {
+ return conf.getLong("hbase.regionserver.recoverededits.blocksize", defaultBlockSize);
+ }
+ return conf.getLong("hbase.regionserver.hlog.blocksize", defaultBlockSize);
+ }
}
\ No newline at end of file
[hbase] 01/02: HBASE-16499 slow replication for small HBase clusters
Posted by ap...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit feb8a57a9a576a7e311a7447480b00bf23ab46b1
Author: Ashish Singhi <as...@apache.org>
AuthorDate: Wed Apr 4 09:59:50 2018 +0530
HBASE-16499 slow replication for small HBase clusters
Signed-off-by: Ashish Singhi <as...@apache.org>
---
.../regionserver/ReplicationSinkManager.java | 2 +-
.../regionserver/TestReplicationSinkManager.java | 36 ++++++++++++----------
2 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
index 0469f9b..c5977a3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java
@@ -54,7 +54,7 @@ public class ReplicationSinkManager {
* Default ratio of the total number of peer cluster region servers to consider
* replicating to.
*/
- static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.1f;
+ static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.5f;
private final HConnection conn;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java
index c0b7d0c..efee9af 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.regionserver.ReplicationSinkManager.SinkPeer;
import org.junit.Before;
import org.junit.Test;
@@ -42,13 +41,11 @@ public class TestReplicationSinkManager {
private static final String PEER_CLUSTER_ID = "PEER_CLUSTER_ID";
- private ReplicationPeers replicationPeers;
private HBaseReplicationEndpoint replicationEndpoint;
private ReplicationSinkManager sinkManager;
@Before
public void setUp() {
- replicationPeers = mock(ReplicationPeers.class);
replicationEndpoint = mock(HBaseReplicationEndpoint.class);
sinkManager = new ReplicationSinkManager(mock(HConnection.class),
PEER_CLUSTER_ID, replicationEndpoint, new Configuration());
@@ -57,7 +54,8 @@ public class TestReplicationSinkManager {
@Test
public void testChooseSinks() {
List<ServerName> serverNames = Lists.newArrayList();
- for (int i = 0; i < 20; i++) {
+ int totalServers = 20;
+ for (int i = 0; i < totalServers; i++) {
serverNames.add(mock(ServerName.class));
}
@@ -66,7 +64,8 @@ public class TestReplicationSinkManager {
sinkManager.chooseSinks();
- assertEquals(2, sinkManager.getNumSinks());
+ int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO);
+ assertEquals(expected, sinkManager.getNumSinks());
}
@@ -110,7 +109,8 @@ public class TestReplicationSinkManager {
@Test
public void testReportBadSink_PastThreshold() {
List<ServerName> serverNames = Lists.newArrayList();
- for (int i = 0; i < 30; i++) {
+ int totalServers = 30;
+ for (int i = 0; i < totalServers; i++) {
serverNames.add(mock(ServerName.class));
}
when(replicationEndpoint.getRegionServers())
@@ -119,7 +119,8 @@ public class TestReplicationSinkManager {
sinkManager.chooseSinks();
// Sanity check
- assertEquals(3, sinkManager.getNumSinks());
+ int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO);
+ assertEquals(expected, sinkManager.getNumSinks());
ServerName serverName = sinkManager.getSinksForTesting().get(0);
@@ -132,7 +133,7 @@ public class TestReplicationSinkManager {
// Reporting a bad sink more than the threshold count should remove it
// from the list of potential sinks
- assertEquals(2, sinkManager.getNumSinks());
+ assertEquals(expected - 1, sinkManager.getNumSinks());
//
// now try a sink that has some successes
@@ -147,23 +148,24 @@ public class TestReplicationSinkManager {
sinkManager.reportBadSink(sinkPeer);
// did not remove the sink, since we had one successful try
- assertEquals(2, sinkManager.getNumSinks());
+ assertEquals(expected - 1, sinkManager.getNumSinks());
for (int i = 0; i <= ReplicationSinkManager.DEFAULT_BAD_SINK_THRESHOLD-2; i++) {
sinkManager.reportBadSink(sinkPeer);
}
// still not remove, since the success reset the counter
- assertEquals(2, sinkManager.getNumSinks());
+ assertEquals(expected - 1, sinkManager.getNumSinks());
sinkManager.reportBadSink(sinkPeer);
// but we exhausted the tries
- assertEquals(1, sinkManager.getNumSinks());
+ assertEquals(expected - 2, sinkManager.getNumSinks());
}
@Test
public void testReportBadSink_DownToZeroSinks() {
List<ServerName> serverNames = Lists.newArrayList();
- for (int i = 0; i < 20; i++) {
+ int totalServers = 4;
+ for (int i = 0; i < totalServers; i++) {
serverNames.add(mock(ServerName.class));
}
when(replicationEndpoint.getRegionServers())
@@ -172,9 +174,9 @@ public class TestReplicationSinkManager {
sinkManager.chooseSinks();
// Sanity check
-
List<ServerName> sinkList = sinkManager.getSinksForTesting();
- assertEquals(2, sinkList.size());
+ int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO);
+ assertEquals(expected, sinkList.size());
ServerName serverNameA = sinkList.get(0);
ServerName serverNameB = sinkList.get(1);
@@ -188,8 +190,10 @@ public class TestReplicationSinkManager {
}
// We've gone down to 0 good sinks, so the replication sinks
- // should have been refreshed now
- assertEquals(2, sinkManager.getNumSinks());
+ // should have been refreshed now, so out of 4 servers, 2 are not considered as they are
+ // reported as bad.
+ expected = (int) ((totalServers - 2) * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO);
+ assertEquals(expected, sinkManager.getNumSinks());
}
}