You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2020/04/15 00:29:36 UTC
[hbase] 01/03: HBASE-23956 Use less resources running tests (#1266)
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit fe72f5687fa5fcfa4ad4a19bc06c4c67f4e711f9
Author: Michael Stack <sa...@users.noreply.github.com>
AuthorDate: Wed Mar 11 10:25:11 2020 -0700
HBASE-23956 Use less resources running tests (#1266)
Add being able to configure netty thread counts. Enable socket reuse
(should not have any impact).
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
Rename the threads we create in here so they are NOT named same was
threads created by Hadoop RPC.
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
Allow configuring eventloopgroup thread count (so can override for
tests)
hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
Enable socket resuse.
hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
Enable socket resuse and config for how many threads to use.
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
Thread name edit; drop the redundant 'Thread' suffix.
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
Make closeable and shutdown executor when called.
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
Call close on HFileReplicator
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
HDFS creates lots of threads. Use less of it so less threads overall.
hbase-server/src/test/resources/hbase-site.xml
hbase-server/src/test/resources/hdfs-site.xml
Constrain resources when running in test context.
hbase-server/src/test/resources/log4j.properties
Enable debug on netty to see netty configs in our log
pom.xml
Add system properties when we launch JVMs to constrain thread counts in
tests
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java | 6 ++++--
pom.xml | 9 +++++++++
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
index a8c9937..4a7bc66 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
@@ -23,7 +23,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
-
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.io.netty.channel.Channel;
import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
@@ -107,7 +106,10 @@ public final class NettyRpcClientConfigHelper {
static Pair<EventLoopGroup, Class<? extends Channel>> getEventLoopConfig(Configuration conf) {
String name = conf.get(EVENT_LOOP_CONFIG);
if (name == null) {
- return getDefaultEventLoopConfig(conf);
+ int threadCount = conf.getInt(HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0);
+ return new Pair<>(new NioEventLoopGroup(threadCount,
+ new DefaultThreadFactory("RPCClient-NioEventLoopGroup", true,
+ Thread.NORM_PRIORITY)), NioSocketChannel.class);
}
if (StringUtils.isBlank(name)) {
return null;
diff --git a/pom.xml b/pom.xml
index 36e9557..0768ce2 100755
--- a/pom.xml
+++ b/pom.xml
@@ -584,8 +584,17 @@
<testFailureIgnore>${surefire.testFailureIgnore}</testFailureIgnore>
<forkedProcessTimeoutInSeconds>${surefire.timeout}</forkedProcessTimeoutInSeconds>
<redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
+ <!-- These are incontext system properties set on mvn. For forked
+ jvms, see hbase-surefire.argLine... So, we might be dup'ing
+ configs here and over on what we pass to the forked jvm.
+ -->
<systemPropertyVariables>
<test.build.classes>${test.build.classes}</test.build.classes>
+ <!--And for eventloops that have no explicit configuration, netty sets
+ nioeventloopgroup thread count to CPU count * 2. Thats too much
+ for mini clusters/tests.
+ -->
+ <io.netty.eventLoopThreads>3</io.netty.eventLoopThreads>
</systemPropertyVariables>
<excludes>
<!-- users can add -D option to skip particular test classes