You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2020/03/11 17:40:03 UTC
[hbase] branch master updated: HBASE-23956 Use less resources
running tests (#1266)
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new aaf9a14 HBASE-23956 Use less resources running tests (#1266)
aaf9a14 is described below
commit aaf9a148a061f139776188665d7a4d09cec9e8b8
Author: Michael Stack <sa...@users.noreply.github.com>
AuthorDate: Wed Mar 11 10:25:11 2020 -0700
HBASE-23956 Use less resources running tests (#1266)
Add being able to configure netty thread counts. Enable socket reuse
(should not have any impact).
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
Rename the threads we create in here so they are NOT named same was
threads created by Hadoop RPC.
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
Allow configuring eventloopgroup thread count (so can override for
tests)
hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
Enable socket resuse.
hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
Enable socket resuse and config for how many threads to use.
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
Thread name edit; drop the redundant 'Thread' suffix.
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
Make closeable and shutdown executor when called.
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
Call close on HFileReplicator
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
HDFS creates lots of threads. Use less of it so less threads overall.
hbase-server/src/test/resources/hbase-site.xml
hbase-server/src/test/resources/hdfs-site.xml
Constrain resources when running in test context.
hbase-server/src/test/resources/log4j.properties
Enable debug on netty to see netty configs in our log
pom.xml
Add system properties when we launch JVMs to constrain thread counts in
tests
Signed-off-by: Duo Zhang <zh...@apache.org>
---
hbase-backup/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../hadoop/hbase/ipc/BlockingRpcConnection.java | 2 +-
.../hbase/ipc/DefaultNettyEventLoopConfig.java | 40 -------
.../apache/hadoop/hbase/ipc/NettyRpcClient.java | 13 +-
.../hbase/ipc/NettyRpcClientConfigHelper.java | 36 ++++--
hbase-client/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
hbase-endpoint/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
.../hbase/client/example/HttpProxyExample.java | 1 +
hbase-examples/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
hbase-mapreduce/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
hbase-mapreduce/src/test/resources/hdfs-site.xml | 26 +++-
hbase-procedure/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
hbase-rest/src/test/resources/hdfs-site.xml | 26 +++-
.../src/test/resources/hdfs-site.xml | 26 +++-
.../apache/hadoop/hbase/ipc/NettyRpcServer.java | 61 ++++++----
.../apache/hadoop/hbase/regionserver/HRegion.java | 2 +-
.../apache/hadoop/hbase/regionserver/HStore.java | 4 +-
.../hadoop/hbase/regionserver/wal/AsyncFSWAL.java | 6 +-
.../replication/regionserver/HFileReplicator.java | 27 +++--
.../replication/regionserver/ReplicationSink.java | 35 ++----
.../hadoop/hbase/util/ModifyRegionUtils.java | 6 +-
.../hbase/util/NettyEventLoopGroupConfig.java | 3 -
.../hbase/replication/TestReplicationBase.java | 4 +-
hbase-server/src/test/resources/hbase-site.xml | 126 +++++++++++++++++++-
hbase-server/src/test/resources/hdfs-site.xml | 26 +++-
hbase-server/src/test/resources/log4j.properties | 1 +
hbase-shell/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
hbase-thrift/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
pom.xml | 14 +++
35 files changed, 1545 insertions(+), 144 deletions(-)
diff --git a/hbase-backup/src/test/resources/hbase-site.xml b/hbase-backup/src/test/resources/hbase-site.xml
index 99d2ab8..3a2b052 100644
--- a/hbase-backup/src/test/resources/hbase-site.xml
+++ b/hbase-backup/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
index 99708e3..46759a1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
@@ -233,7 +233,7 @@ class BlockingRpcConnection extends RpcConnection implements Runnable {
this.connectionHeaderWithLength = baos.getBuffer();
UserGroupInformation ticket = remoteId.ticket.getUGI();
- this.threadName = "IPC Client (" + this.rpcClient.socketFactory.hashCode() + ") connection to "
+ this.threadName = "BRPC Connection (" + this.rpcClient.socketFactory.hashCode() + ") to "
+ remoteId.getAddress().toString()
+ ((ticket == null) ? " from an unknown user" : (" from " + ticket.getUserName()));
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java
deleted file mode 100644
index 87e5540..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.ipc;
-
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
-import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
-
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * The default netty event loop config
- */
-@InterfaceAudience.Private
-class DefaultNettyEventLoopConfig {
-
- public static final Pair<EventLoopGroup, Class<? extends Channel>> GROUP_AND_CHANNEL_CLASS = Pair
- .<EventLoopGroup, Class<? extends Channel>> newPair(
- new NioEventLoopGroup(0,
- new DefaultThreadFactory("Default-IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY)),
- NioSocketChannel.class);
-}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
index c4f70b0..f7a65e4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -54,12 +54,15 @@ public class NettyRpcClient extends AbstractRpcClient<NettyRpcConnection> {
public NettyRpcClient(Configuration configuration, String clusterId, SocketAddress localAddress,
MetricsConnection metrics) {
super(configuration, clusterId, localAddress, metrics);
- Pair<EventLoopGroup, Class<? extends Channel>> groupAndChannelClass = NettyRpcClientConfigHelper
- .getEventLoopConfig(conf);
+ Pair<EventLoopGroup, Class<? extends Channel>> groupAndChannelClass =
+ NettyRpcClientConfigHelper.getEventLoopConfig(conf);
if (groupAndChannelClass == null) {
// Use our own EventLoopGroup.
- this.group = new NioEventLoopGroup(0,
- new DefaultThreadFactory("IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY));
+ int threadCount = conf.getInt(
+ NettyRpcClientConfigHelper.HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0);
+ this.group = new NioEventLoopGroup(threadCount,
+ new DefaultThreadFactory("RPCClient(own)-NioEventLoopGroup", true,
+ Thread.NORM_PRIORITY));
this.channelClass = NioSocketChannel.class;
this.shutdownGroupWhenClose = true;
} else {
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
index e779339..6107183 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,18 +17,18 @@
*/
package org.apache.hadoop.hbase.ipc;
-import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
-
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-
import java.util.HashMap;
import java.util.Map;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
+import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
/**
* Helper class for passing config to {@link NettyRpcClient}.
@@ -39,16 +39,28 @@ import org.apache.hadoop.hbase.util.Pair;
* @since 2.0.0
*/
@InterfaceAudience.Public
-public class NettyRpcClientConfigHelper {
+public final class NettyRpcClientConfigHelper {
public static final String EVENT_LOOP_CONFIG = "hbase.rpc.client.event-loop.config";
+ /**
+ * Name of property to change netty rpc client eventloop thread count. Default is 0.
+ * Tests may set this down from unlimited.
+ */
+ public static final String HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY =
+ "hbase.netty.eventloop.rpcclient.thread.count";
+
private static final String CONFIG_NAME = "global-event-loop";
private static final Map<String, Pair<EventLoopGroup, Class<? extends Channel>>>
EVENT_LOOP_CONFIG_MAP = new HashMap<>();
/**
+ * Shutdown constructor.
+ */
+ private NettyRpcClientConfigHelper() {}
+
+ /**
* Set the EventLoopGroup and channel class for {@code AsyncRpcClient}.
*/
public static void setEventLoopConfig(Configuration conf, EventLoopGroup group,
@@ -71,12 +83,14 @@ public class NettyRpcClientConfigHelper {
static Pair<EventLoopGroup, Class<? extends Channel>> getEventLoopConfig(Configuration conf) {
String name = conf.get(EVENT_LOOP_CONFIG);
if (name == null) {
- return DefaultNettyEventLoopConfig.GROUP_AND_CHANNEL_CLASS;
+ int threadCount = conf.getInt(HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0);
+ return new Pair<>(new NioEventLoopGroup(threadCount,
+ new DefaultThreadFactory("RPCClient-NioEventLoopGroup", true,
+ Thread.NORM_PRIORITY)), NioSocketChannel.class);
}
if (StringUtils.isBlank(name)) {
return null;
}
return EVENT_LOOP_CONFIG_MAP.get(name);
}
-
}
diff --git a/hbase-client/src/test/resources/hbase-site.xml b/hbase-client/src/test/resources/hbase-site.xml
index 99d2ab8..3a2b052 100644
--- a/hbase-client/src/test/resources/hbase-site.xml
+++ b/hbase-client/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-client/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-client/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-client/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-endpoint/src/test/resources/hbase-site.xml b/hbase-endpoint/src/test/resources/hbase-site.xml
index 99d2ab8..3a2b052 100644
--- a/hbase-endpoint/src/test/resources/hbase-site.xml
+++ b/hbase-endpoint/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-endpoint/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-endpoint/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-endpoint/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
index 4ada599..00d01a8 100644
--- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
+++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
@@ -229,6 +229,7 @@ public class HttpProxyExample {
channelGroup = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE);
serverChannel = new ServerBootstrap().group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class).childOption(ChannelOption.TCP_NODELAY, true)
+ .childOption(ChannelOption.SO_REUSEADDR, true)
.childHandler(new ChannelInitializer<Channel>() {
@Override
diff --git a/hbase-examples/src/test/resources/hbase-site.xml b/hbase-examples/src/test/resources/hbase-site.xml
index ab4d1cd..6d6f4bf 100644
--- a/hbase-examples/src/test/resources/hbase-site.xml
+++ b/hbase-examples/src/test/resources/hbase-site.xml
@@ -22,7 +22,138 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-examples/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-examples/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-examples/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-mapreduce/src/test/resources/hbase-site.xml b/hbase-mapreduce/src/test/resources/hbase-site.xml
index 64a1964..a2aee0c 100644
--- a/hbase-mapreduce/src/test/resources/hbase-site.xml
+++ b/hbase-mapreduce/src/test/resources/hbase-site.xml
@@ -23,6 +23,22 @@
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.msginterval</name>
<value>1000</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
@@ -158,4 +174,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-mapreduce/src/test/resources/hdfs-site.xml b/hbase-mapreduce/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-mapreduce/src/test/resources/hdfs-site.xml
+++ b/hbase-mapreduce/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-procedure/src/test/resources/hbase-site.xml b/hbase-procedure/src/test/resources/hbase-site.xml
index 3709a71..c938f28 100644
--- a/hbase-procedure/src/test/resources/hbase-site.xml
+++ b/hbase-procedure/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -45,4 +61,119 @@
WARNING: Doing so may expose you to additional risk of data loss!
</description>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-procedure/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-procedure/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-procedure/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-rest/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-rest/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-rsgroup/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-rsgroup/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-rsgroup/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
index bba1bed..eab2a0e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,44 +17,42 @@
*/
package org.apache.hadoop.hbase.ipc;
-import org.apache.hbase.thirdparty.io.netty.bootstrap.ServerBootstrap;
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelInitializer;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelOption;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelPipeline;
-import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.ServerChannel;
-import org.apache.hbase.thirdparty.io.netty.channel.group.ChannelGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel;
-import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecoder;
-import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
-import org.apache.hbase.thirdparty.io.netty.util.concurrent.GlobalEventExecutor;
-
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.CountDownLatch;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.Server;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.security.HBasePolicyProvider;
+import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.protobuf.BlockingService;
import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor;
import org.apache.hbase.thirdparty.com.google.protobuf.Message;
-import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
+import org.apache.hbase.thirdparty.io.netty.bootstrap.ServerBootstrap;
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelInitializer;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelOption;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelPipeline;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.ServerChannel;
+import org.apache.hbase.thirdparty.io.netty.channel.group.ChannelGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel;
+import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecoder;
+import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
+import org.apache.hbase.thirdparty.io.netty.util.concurrent.GlobalEventExecutor;
/**
* An RPC server with Netty4 implementation.
@@ -62,9 +60,16 @@ import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
*/
@InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.CONFIG})
public class NettyRpcServer extends RpcServer {
-
public static final Logger LOG = LoggerFactory.getLogger(NettyRpcServer.class);
+ /**
+ * Name of property to change netty rpc server eventloop thread count. Default is 0.
+ * Tests may set this down from unlimited.
+ */
+ public static final String HBASE_NETTY_EVENTLOOP_RPCSERVER_THREADCOUNT_KEY =
+ "hbase.netty.eventloop.rpcserver.thread.count";
+ private static final int EVENTLOOP_THREADCOUNT_DEFAULT = 0;
+
private final InetSocketAddress bindAddress;
private final CountDownLatch closed = new CountDownLatch(1);
@@ -84,13 +89,17 @@ public class NettyRpcServer extends RpcServer {
eventLoopGroup = config.group();
channelClass = config.serverChannelClass();
} else {
- eventLoopGroup = new NioEventLoopGroup(0,
- new DefaultThreadFactory("NettyRpcServer", true, Thread.MAX_PRIORITY));
+ int threadCount = server == null? EVENTLOOP_THREADCOUNT_DEFAULT:
+ server.getConfiguration().getInt(HBASE_NETTY_EVENTLOOP_RPCSERVER_THREADCOUNT_KEY,
+ EVENTLOOP_THREADCOUNT_DEFAULT);
+ eventLoopGroup = new NioEventLoopGroup(threadCount,
+ new DefaultThreadFactory("NettyRpcServer", true, Thread.MAX_PRIORITY));
channelClass = NioServerSocketChannel.class;
}
ServerBootstrap bootstrap = new ServerBootstrap().group(eventLoopGroup).channel(channelClass)
.childOption(ChannelOption.TCP_NODELAY, tcpNoDelay)
.childOption(ChannelOption.SO_KEEPALIVE, tcpKeepAlive)
+ .childOption(ChannelOption.SO_REUSEADDR, true)
.childHandler(new ChannelInitializer<Channel>() {
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c0ad87c..c382606 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1691,7 +1691,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
if (!stores.isEmpty()) {
// initialize the thread pool for closing stores in parallel.
ThreadPoolExecutor storeCloserThreadPool =
- getStoreOpenAndCloseThreadPool("StoreCloserThread-" +
+ getStoreOpenAndCloseThreadPool("StoreCloser-" +
getRegionInfo().getRegionNameAsString());
CompletionService<Pair<byte[], Collection<HStoreFile>>> completionService =
new ExecutorCompletionService<>(storeCloserThreadPool);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index affd43f..1c376f9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -569,7 +569,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
}
// initialize the thread pool for opening store files in parallel..
ThreadPoolExecutor storeFileOpenerThreadPool =
- this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-"
+ this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpener-"
+ this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName());
CompletionService<HStoreFile> completionService =
new ExecutorCompletionService<>(storeFileOpenerThreadPool);
@@ -969,7 +969,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
if (!result.isEmpty()) {
// initialize the thread pool for closing store files in parallel.
ThreadPoolExecutor storeFileCloserThreadPool = this.region
- .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"
+ .getStoreFileOpenAndCloseThreadPool("StoreFileCloser-"
+ this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName());
// close each store file in parallel
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index 3d8105d..a40e503 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -228,8 +228,10 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
}
} else {
ThreadPoolExecutor threadPool =
- new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
- new ThreadFactoryBuilder().setNameFormat("AsyncFSWAL-%d").setDaemon(true).build());
+ new ThreadPoolExecutor(1, 1, 0L,
+ TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
+ new ThreadFactoryBuilder().setNameFormat("AsyncFSWAL-%d-" + rootDir.toString()).
+ setDaemon(true).build());
hasConsumerTask = () -> threadPool.getQueue().peek() == consumer;
this.consumeExecutor = threadPool;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
index 169f747..92da81b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
+import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
@@ -55,16 +56,16 @@ import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* It is used for replicating HFile entries. It will first copy parallely all the hfiles to a local
* staging directory and then it will use ({@link BulkLoadHFiles} to prepare a collection of
* {@link LoadQueueItem} which will finally be loaded(replicated) into the table of this cluster.
+ * Call {@link #close()} when done.
*/
@InterfaceAudience.Private
-public class HFileReplicator {
+public class HFileReplicator implements Closeable {
/** Maximum number of threads to allow in pool to copy hfiles during replication */
public static final String REPLICATION_BULKLOAD_COPY_MAXTHREADS_KEY =
"hbase.replication.bulkload.copy.maxthreads";
@@ -113,7 +114,8 @@ public class HFileReplicator {
REPLICATION_BULKLOAD_COPY_MAXTHREADS_DEFAULT);
this.exec = Threads.getBoundedCachedThreadPool(maxCopyThreads, 60, TimeUnit.SECONDS,
new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("HFileReplicationCallable-%1$d").build());
+ .setNameFormat("HFileReplicationCopier-%1$d-" + this.sourceBaseNamespaceDirPath).
+ build());
this.copiesPerThread =
conf.getInt(REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_KEY,
REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_DEFAULT);
@@ -121,6 +123,13 @@ public class HFileReplicator {
sinkFs = FileSystem.get(conf);
}
+ @Override
+ public void close() throws IOException {
+ if (this.exec != null) {
+ this.exec.shutdown();
+ }
+ }
+
public Void replicate() throws IOException {
// Copy all the hfiles to the local file system
Map<String, Path> tableStagingDirsMap = copyHFilesToStagingDir();
@@ -130,7 +139,6 @@ public class HFileReplicator {
for (Entry<String, Path> tableStagingDir : tableStagingDirsMap.entrySet()) {
String tableNameString = tableStagingDir.getKey();
Path stagingDir = tableStagingDir.getValue();
-
TableName tableName = TableName.valueOf(tableNameString);
// Prepare collection of queue of hfiles to be loaded(replicated)
@@ -139,8 +147,7 @@ public class HFileReplicator {
false);
if (queue.isEmpty()) {
- LOG.warn("Replication process did not find any files to replicate in directory "
- + stagingDir.toUri());
+ LOG.warn("Did not find any files to replicate in directory {}", stagingDir.toUri());
return null;
}
fsDelegationToken.acquireDelegationToken(sinkFs);
@@ -162,13 +169,11 @@ public class HFileReplicator {
loader.setClusterIds(sourceClusterIds);
for (int count = 0; !queue.isEmpty(); count++) {
if (count != 0) {
- LOG.warn("Error occurred while replicating HFiles, retry attempt " + count + " with " +
- queue.size() + " files still remaining to replicate.");
+ LOG.warn("Error replicating HFiles; retry={} with {} remaining.", count, queue.size());
}
if (maxRetries != 0 && count >= maxRetries) {
- throw new IOException(
- "Retry attempted " + count + " times without completing, bailing out.");
+ throw new IOException("Retry attempted " + count + " times without completing, bailing.");
}
// Try bulk load
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index 55a90a0..18926b7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
@@ -151,7 +150,7 @@ public class ReplicationSink {
if (this.conf.get(HConstants.CLIENT_ZOOKEEPER_QUORUM) != null) {
this.conf.unset(HConstants.CLIENT_ZOOKEEPER_QUORUM);
}
- }
+ }
/**
* Replicate this array of entries directly into the local cluster using the native client. Only
@@ -166,7 +165,9 @@ public class ReplicationSink {
public void replicateEntries(List<WALEntry> entries, final CellScanner cells,
String replicationClusterId, String sourceBaseNamespaceDirPath,
String sourceHFileArchiveDirPath) throws IOException {
- if (entries.isEmpty()) return;
+ if (entries.isEmpty()) {
+ return;
+ }
// Very simple optimization where we batch sequences of rows going
// to the same table.
try {
@@ -259,18 +260,13 @@ public class ReplicationSink {
bulkLoadsPerClusters.entrySet()) {
Map<String, List<Pair<byte[], List<String>>>> bulkLoadHFileMap = entry.getValue();
if (bulkLoadHFileMap != null && !bulkLoadHFileMap.isEmpty()) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Started replicating bulk loaded data from cluster ids: {}.",
- entry.getKey().toString());
- }
- HFileReplicator hFileReplicator =
- new HFileReplicator(this.provider.getConf(this.conf, replicationClusterId),
+ LOG.debug("Replicating {} bulk loaded data", entry.getKey().toString());
+ Configuration providerConf = this.provider.getConf(this.conf, replicationClusterId);
+ try (HFileReplicator hFileReplicator = new HFileReplicator(providerConf,
sourceBaseNamespaceDirPath, sourceHFileArchiveDirPath, bulkLoadHFileMap, conf,
- getConnection(), entry.getKey());
- hFileReplicator.replicate();
- if(LOG.isDebugEnabled()) {
- LOG.debug("Finished replicating bulk loaded data from cluster id: {}",
- entry.getKey().toString());
+ getConnection(), entry.getKey())) {
+ hFileReplicator.replicate();
+ LOG.debug("Finished replicating {} bulk loaded data", entry.getKey().toString());
}
}
}
@@ -352,8 +348,6 @@ public class ReplicationSink {
}
/**
- * @param previousCell
- * @param cell
* @return True if we have crossed over onto a new row or type
*/
private boolean isNewRowOrType(final Cell previousCell, final Cell cell) {
@@ -368,13 +362,10 @@ public class ReplicationSink {
/**
* Simple helper to a map from key to (a list of) values
* TODO: Make a general utility method
- * @param map
- * @param key1
- * @param key2
- * @param value
* @return the list of values corresponding to key1 and key2
*/
- private <K1, K2, V> List<V> addToHashMultiMap(Map<K1, Map<K2,List<V>>> map, K1 key1, K2 key2, V value) {
+ private <K1, K2, V> List<V> addToHashMultiMap(Map<K1, Map<K2,List<V>>> map, K1 key1,
+ K2 key2, V value) {
Map<K2,List<V>> innerMap = map.get(key1);
if (innerMap == null) {
innerMap = new HashMap<>();
@@ -450,7 +441,7 @@ public class ReplicationSink {
/**
* Get a string representation of this sink's metrics
* @return string with the total replicated edits count and the date
- * of the last edit that was applied
+ * of the last edit that was applied
*/
public String getStats() {
long total = this.totalReplicatedEdits.get();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
index 683d175..c0189c8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -110,7 +110,7 @@ public abstract class ModifyRegionUtils {
if (newRegions == null) return null;
int regionNumber = newRegions.length;
ThreadPoolExecutor exec = getRegionOpenAndInitThreadPool(conf,
- "RegionOpenAndInitThread-" + tableDescriptor.getTableName(), regionNumber);
+ "RegionOpenAndInit-" + tableDescriptor.getTableName(), regionNumber);
try {
return createRegions(exec, conf, rootDir, tableDescriptor, newRegions, task);
} finally {
@@ -230,8 +230,8 @@ public abstract class ModifyRegionUtils {
final String threadNamePrefix, int regionNumber) {
int maxThreads = Math.min(regionNumber, conf.getInt(
"hbase.hregion.open.and.init.threads.max", 16));
- ThreadPoolExecutor regionOpenAndInitThreadPool = Threads
- .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
+ ThreadPoolExecutor regionOpenAndInitThreadPool = Threads.
+ getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
Threads.newDaemonThreadFactory(threadNamePrefix));
return regionOpenAndInitThreadPool;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java
index 3e7b488..3e247f3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java
@@ -27,9 +27,7 @@ import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel;
import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
-
import java.util.concurrent.ThreadFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.yetus.audience.InterfaceAudience;
@@ -38,7 +36,6 @@ import org.apache.yetus.audience.InterfaceAudience;
*/
@InterfaceAudience.Private
public class NettyEventLoopGroupConfig {
-
private final EventLoopGroup group;
private final Class<? extends ServerChannel> serverChannelClass;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
index a38fcde..786bb64 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -79,8 +79,8 @@ public class TestReplicationBase {
protected static Configuration CONF1 = UTIL1.getConfiguration();
protected static Configuration CONF2 = UTIL2.getConfiguration();
- protected static final int NUM_SLAVES1 = 2;
- protected static final int NUM_SLAVES2 = 4;
+ protected static final int NUM_SLAVES1 = 1;
+ protected static final int NUM_SLAVES2 = 1;
protected static final int NB_ROWS_IN_BATCH = 100;
protected static final int NB_ROWS_IN_BIG_BATCH =
NB_ROWS_IN_BATCH * 10;
diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml
index 64a1964..a317376 100644
--- a/hbase-server/src/test/resources/hbase-site.xml
+++ b/hbase-server/src/test/resources/hbase-site.xml
@@ -23,7 +23,7 @@
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
- <value>1000</value>
+ <value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
@@ -35,7 +35,7 @@
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
- <value>1000</value>
+ <value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
@@ -48,11 +48,13 @@
</property>
<property>
<name>hbase.regionserver.handler.count</name>
- <value>5</value>
+ <value>3</value>
+ <description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
- <value>6</value>
+ <value>3</value>
+ <description>Default is 20</description>
</property>
<property>
<name>hbase.ipc.server.read.threadpool.size</name>
@@ -83,7 +85,6 @@
<name>hbase.ipc.client.fallback-to-simple-auth-allowed</name>
<value>true</value>
</property>
-
<property>
<name>hbase.regionserver.info.port</name>
<value>-1</value>
@@ -158,4 +159,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.netty.eventloop.rpclient.thread.count</name>
+ <value>3</value>
+ <description>Default is unbounded</description>
+ </property>
+ <property>
+ <name>hbase.netty.eventloop.rpcserver.thread.count</name>
+ <value>3</value>
+ <description>Default is unbounded</description>
+ </property>
</configuration>
diff --git a/hbase-server/src/test/resources/hdfs-site.xml b/hbase-server/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-server/src/test/resources/hdfs-site.xml
+++ b/hbase-server/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-server/src/test/resources/log4j.properties b/hbase-server/src/test/resources/log4j.properties
index 785371d..8e590b1 100644
--- a/hbase-server/src/test/resources/log4j.properties
+++ b/hbase-server/src/test/resources/log4j.properties
@@ -66,6 +66,7 @@ log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN
log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN
+log4j.logger.io.netty.channel=DEBUG
# Enable this to get detailed connection error/retry logging.
# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE
log4j.logger.org.apache.directory=WARN
diff --git a/hbase-shell/src/test/resources/hbase-site.xml b/hbase-shell/src/test/resources/hbase-site.xml
index 99d2ab8..3a2b052 100644
--- a/hbase-shell/src/test/resources/hbase-site.xml
+++ b/hbase-shell/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-shell/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-shell/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-shell/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-thrift/src/test/resources/hbase-site.xml b/hbase-thrift/src/test/resources/hbase-site.xml
index b3fb0d9..a1e89ce 100644
--- a/hbase-thrift/src/test/resources/hbase-site.xml
+++ b/hbase-thrift/src/test/resources/hbase-site.xml
@@ -23,6 +23,22 @@
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.msginterval</name>
<value>1000</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
@@ -154,4 +170,119 @@
Enable replay sanity checks on procedure tests.
</description>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-thrift/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-thrift/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-thrift/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/pom.xml b/pom.xml
index fe07edf..ae88fa1 100755
--- a/pom.xml
+++ b/pom.xml
@@ -746,8 +746,17 @@
<testFailureIgnore>${surefire.testFailureIgnore}</testFailureIgnore>
<forkedProcessTimeoutInSeconds>${surefire.timeout}</forkedProcessTimeoutInSeconds>
<redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
+ <!-- These are incontext system properties set on mvn. For forked
+ jvms, see hbase-surefire.argLine... So, we might be dup'ing
+ configs here and over on what we pass to the forked jvm.
+ -->
<systemPropertyVariables>
<test.build.classes>${test.build.classes}</test.build.classes>
+ <!--And for eventloops that have no explicit configuration, netty sets
+ nioeventloopgroup thread count to CPU count * 2. Thats too much
+ for mini clusters/tests.
+ -->
+ <io.netty.eventLoopThreads>3</io.netty.eventLoopThreads>
</systemPropertyVariables>
<excludes>
<!-- users can add -D option to skip particular test classes
@@ -1596,11 +1605,16 @@
<surefire.Xmx>2800m</surefire.Xmx>
<surefire.cygwinXmx>2800m</surefire.cygwinXmx>
<!--Mark our test runs with '-Dhbase.build.id' so we can identify a surefire test as ours in a process listing
+
+ And for netty eventloops that have no explicit configuration, netty sets
+ nioeventloopgroup thread count to CPU count * 2. Thats too much for mini
+ clusters/tests.
-->
<hbase-surefire.argLine>-enableassertions -Dhbase.build.id=${build.id} -Xmx${surefire.Xmx}
-Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true
-Djava.awt.headless=true -Djdk.net.URLClassPath.disableClassPathURLCheck=true
-Dorg.apache.hbase.thirdparty.io.netty.leakDetection.level=advanced
+ -Dio.netty.eventLoopThreads=3
</hbase-surefire.argLine>
<hbase-surefire.cygwin-argLine>-enableassertions -Xmx${surefire.cygwinXmx}
-Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true