You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2020/03/11 17:25:21 UTC
[hbase] branch branch-2 updated: HBASE-23956 Use less resources
running tests (#1266)
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 2655f96 HBASE-23956 Use less resources running tests (#1266)
2655f96 is described below
commit 2655f9647e492901ac9aebf7974ff8dc6b585f40
Author: Michael Stack <sa...@users.noreply.github.com>
AuthorDate: Wed Mar 11 10:25:11 2020 -0700
HBASE-23956 Use less resources running tests (#1266)
Add being able to configure netty thread counts. Enable socket reuse
(should not have any impact).
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
Rename the threads we create in here so they are NOT named same was
threads created by Hadoop RPC.
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
Allow configuring eventloopgroup thread count (so can override for
tests)
hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
Enable socket resuse.
hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
Enable socket resuse and config for how many threads to use.
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
Thread name edit; drop the redundant 'Thread' suffix.
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
Make closeable and shutdown executor when called.
hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
Call close on HFileReplicator
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
HDFS creates lots of threads. Use less of it so less threads overall.
hbase-server/src/test/resources/hbase-site.xml
hbase-server/src/test/resources/hdfs-site.xml
Constrain resources when running in test context.
hbase-server/src/test/resources/log4j.properties
Enable debug on netty to see netty configs in our log
pom.xml
Add system properties when we launch JVMs to constrain thread counts in
tests
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../hadoop/hbase/ipc/BlockingRpcConnection.java | 2 +-
.../hbase/ipc/DefaultNettyEventLoopConfig.java | 40 -------
.../apache/hadoop/hbase/ipc/NettyRpcClient.java | 13 +-
.../hbase/ipc/NettyRpcClientConfigHelper.java | 36 ++++--
hbase-client/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
hbase-endpoint/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
.../hbase/client/example/HttpProxyExample.java | 1 +
hbase-examples/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
hbase-mapreduce/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
hbase-mapreduce/src/test/resources/hdfs-site.xml | 26 +++-
hbase-procedure/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
hbase-rest/src/test/resources/hdfs-site.xml | 26 +++-
hbase-rsgroup/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
.../apache/hadoop/hbase/ipc/NettyRpcServer.java | 61 ++++++----
.../apache/hadoop/hbase/regionserver/HRegion.java | 2 +-
.../apache/hadoop/hbase/regionserver/HStore.java | 4 +-
.../hadoop/hbase/regionserver/wal/AsyncFSWAL.java | 6 +-
.../replication/regionserver/HFileReplicator.java | 35 +++---
.../replication/regionserver/ReplicationSink.java | 18 +--
.../hadoop/hbase/util/ModifyRegionUtils.java | 6 +-
.../hbase/util/NettyEventLoopGroupConfig.java | 3 -
.../hbase/replication/TestReplicationBase.java | 4 +-
hbase-server/src/test/resources/hbase-site.xml | 126 +++++++++++++++++++-
hbase-server/src/test/resources/hdfs-site.xml | 26 +++-
hbase-server/src/test/resources/log4j.properties | 1 +
hbase-shell/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
hbase-thrift/src/test/resources/hbase-site.xml | 131 +++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 26 +++-
pom.xml | 14 +++
35 files changed, 1540 insertions(+), 140 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
index 99708e3..46759a1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
@@ -233,7 +233,7 @@ class BlockingRpcConnection extends RpcConnection implements Runnable {
this.connectionHeaderWithLength = baos.getBuffer();
UserGroupInformation ticket = remoteId.ticket.getUGI();
- this.threadName = "IPC Client (" + this.rpcClient.socketFactory.hashCode() + ") connection to "
+ this.threadName = "BRPC Connection (" + this.rpcClient.socketFactory.hashCode() + ") to "
+ remoteId.getAddress().toString()
+ ((ticket == null) ? " from an unknown user" : (" from " + ticket.getUserName()));
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java
deleted file mode 100644
index 87e5540..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.ipc;
-
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
-import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
-
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * The default netty event loop config
- */
-@InterfaceAudience.Private
-class DefaultNettyEventLoopConfig {
-
- public static final Pair<EventLoopGroup, Class<? extends Channel>> GROUP_AND_CHANNEL_CLASS = Pair
- .<EventLoopGroup, Class<? extends Channel>> newPair(
- new NioEventLoopGroup(0,
- new DefaultThreadFactory("Default-IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY)),
- NioSocketChannel.class);
-}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
index c4f70b0..f7a65e4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -54,12 +54,15 @@ public class NettyRpcClient extends AbstractRpcClient<NettyRpcConnection> {
public NettyRpcClient(Configuration configuration, String clusterId, SocketAddress localAddress,
MetricsConnection metrics) {
super(configuration, clusterId, localAddress, metrics);
- Pair<EventLoopGroup, Class<? extends Channel>> groupAndChannelClass = NettyRpcClientConfigHelper
- .getEventLoopConfig(conf);
+ Pair<EventLoopGroup, Class<? extends Channel>> groupAndChannelClass =
+ NettyRpcClientConfigHelper.getEventLoopConfig(conf);
if (groupAndChannelClass == null) {
// Use our own EventLoopGroup.
- this.group = new NioEventLoopGroup(0,
- new DefaultThreadFactory("IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY));
+ int threadCount = conf.getInt(
+ NettyRpcClientConfigHelper.HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0);
+ this.group = new NioEventLoopGroup(threadCount,
+ new DefaultThreadFactory("RPCClient(own)-NioEventLoopGroup", true,
+ Thread.NORM_PRIORITY));
this.channelClass = NioSocketChannel.class;
this.shutdownGroupWhenClose = true;
} else {
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
index e779339..6107183 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,18 +17,18 @@
*/
package org.apache.hadoop.hbase.ipc;
-import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
-
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-
import java.util.HashMap;
import java.util.Map;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
+import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
/**
* Helper class for passing config to {@link NettyRpcClient}.
@@ -39,16 +39,28 @@ import org.apache.hadoop.hbase.util.Pair;
* @since 2.0.0
*/
@InterfaceAudience.Public
-public class NettyRpcClientConfigHelper {
+public final class NettyRpcClientConfigHelper {
public static final String EVENT_LOOP_CONFIG = "hbase.rpc.client.event-loop.config";
+ /**
+ * Name of property to change netty rpc client eventloop thread count. Default is 0.
+ * Tests may set this down from unlimited.
+ */
+ public static final String HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY =
+ "hbase.netty.eventloop.rpcclient.thread.count";
+
private static final String CONFIG_NAME = "global-event-loop";
private static final Map<String, Pair<EventLoopGroup, Class<? extends Channel>>>
EVENT_LOOP_CONFIG_MAP = new HashMap<>();
/**
+ * Shutdown constructor.
+ */
+ private NettyRpcClientConfigHelper() {}
+
+ /**
* Set the EventLoopGroup and channel class for {@code AsyncRpcClient}.
*/
public static void setEventLoopConfig(Configuration conf, EventLoopGroup group,
@@ -71,12 +83,14 @@ public class NettyRpcClientConfigHelper {
static Pair<EventLoopGroup, Class<? extends Channel>> getEventLoopConfig(Configuration conf) {
String name = conf.get(EVENT_LOOP_CONFIG);
if (name == null) {
- return DefaultNettyEventLoopConfig.GROUP_AND_CHANNEL_CLASS;
+ int threadCount = conf.getInt(HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0);
+ return new Pair<>(new NioEventLoopGroup(threadCount,
+ new DefaultThreadFactory("RPCClient-NioEventLoopGroup", true,
+ Thread.NORM_PRIORITY)), NioSocketChannel.class);
}
if (StringUtils.isBlank(name)) {
return null;
}
return EVENT_LOOP_CONFIG_MAP.get(name);
}
-
}
diff --git a/hbase-client/src/test/resources/hbase-site.xml b/hbase-client/src/test/resources/hbase-site.xml
index 99d2ab8..3a2b052 100644
--- a/hbase-client/src/test/resources/hbase-site.xml
+++ b/hbase-client/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-client/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-client/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-client/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-endpoint/src/test/resources/hbase-site.xml b/hbase-endpoint/src/test/resources/hbase-site.xml
index 99d2ab8..3a2b052 100644
--- a/hbase-endpoint/src/test/resources/hbase-site.xml
+++ b/hbase-endpoint/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-endpoint/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-endpoint/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-endpoint/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
index 4ada599..00d01a8 100644
--- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
+++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java
@@ -229,6 +229,7 @@ public class HttpProxyExample {
channelGroup = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE);
serverChannel = new ServerBootstrap().group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class).childOption(ChannelOption.TCP_NODELAY, true)
+ .childOption(ChannelOption.SO_REUSEADDR, true)
.childHandler(new ChannelInitializer<Channel>() {
@Override
diff --git a/hbase-examples/src/test/resources/hbase-site.xml b/hbase-examples/src/test/resources/hbase-site.xml
index ab4d1cd..6d6f4bf 100644
--- a/hbase-examples/src/test/resources/hbase-site.xml
+++ b/hbase-examples/src/test/resources/hbase-site.xml
@@ -22,7 +22,138 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-examples/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-examples/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-examples/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-mapreduce/src/test/resources/hbase-site.xml b/hbase-mapreduce/src/test/resources/hbase-site.xml
index 64a1964..a2aee0c 100644
--- a/hbase-mapreduce/src/test/resources/hbase-site.xml
+++ b/hbase-mapreduce/src/test/resources/hbase-site.xml
@@ -23,6 +23,22 @@
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.msginterval</name>
<value>1000</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
@@ -158,4 +174,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-mapreduce/src/test/resources/hdfs-site.xml b/hbase-mapreduce/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-mapreduce/src/test/resources/hdfs-site.xml
+++ b/hbase-mapreduce/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-procedure/src/test/resources/hbase-site.xml b/hbase-procedure/src/test/resources/hbase-site.xml
index 3709a71..c938f28 100644
--- a/hbase-procedure/src/test/resources/hbase-site.xml
+++ b/hbase-procedure/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -45,4 +61,119 @@
WARNING: Doing so may expose you to additional risk of data loss!
</description>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-procedure/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-procedure/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-procedure/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-rest/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-rest/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-rsgroup/src/test/resources/hbase-site.xml b/hbase-rsgroup/src/test/resources/hbase-site.xml
index 99d2ab8..3a2b052 100644
--- a/hbase-rsgroup/src/test/resources/hbase-site.xml
+++ b/hbase-rsgroup/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-rsgroup/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-rsgroup/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-rsgroup/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
index bba1bed..eab2a0e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,44 +17,42 @@
*/
package org.apache.hadoop.hbase.ipc;
-import org.apache.hbase.thirdparty.io.netty.bootstrap.ServerBootstrap;
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelInitializer;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelOption;
-import org.apache.hbase.thirdparty.io.netty.channel.ChannelPipeline;
-import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.ServerChannel;
-import org.apache.hbase.thirdparty.io.netty.channel.group.ChannelGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel;
-import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecoder;
-import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
-import org.apache.hbase.thirdparty.io.netty.util.concurrent.GlobalEventExecutor;
-
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.CountDownLatch;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.Server;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.security.HBasePolicyProvider;
+import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.protobuf.BlockingService;
import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor;
import org.apache.hbase.thirdparty.com.google.protobuf.Message;
-import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
+import org.apache.hbase.thirdparty.io.netty.bootstrap.ServerBootstrap;
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelInitializer;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelOption;
+import org.apache.hbase.thirdparty.io.netty.channel.ChannelPipeline;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.ServerChannel;
+import org.apache.hbase.thirdparty.io.netty.channel.group.ChannelGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel;
+import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecoder;
+import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
+import org.apache.hbase.thirdparty.io.netty.util.concurrent.GlobalEventExecutor;
/**
* An RPC server with Netty4 implementation.
@@ -62,9 +60,16 @@ import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
*/
@InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.CONFIG})
public class NettyRpcServer extends RpcServer {
-
public static final Logger LOG = LoggerFactory.getLogger(NettyRpcServer.class);
+ /**
+ * Name of property to change netty rpc server eventloop thread count. Default is 0.
+ * Tests may set this down from unlimited.
+ */
+ public static final String HBASE_NETTY_EVENTLOOP_RPCSERVER_THREADCOUNT_KEY =
+ "hbase.netty.eventloop.rpcserver.thread.count";
+ private static final int EVENTLOOP_THREADCOUNT_DEFAULT = 0;
+
private final InetSocketAddress bindAddress;
private final CountDownLatch closed = new CountDownLatch(1);
@@ -84,13 +89,17 @@ public class NettyRpcServer extends RpcServer {
eventLoopGroup = config.group();
channelClass = config.serverChannelClass();
} else {
- eventLoopGroup = new NioEventLoopGroup(0,
- new DefaultThreadFactory("NettyRpcServer", true, Thread.MAX_PRIORITY));
+ int threadCount = server == null? EVENTLOOP_THREADCOUNT_DEFAULT:
+ server.getConfiguration().getInt(HBASE_NETTY_EVENTLOOP_RPCSERVER_THREADCOUNT_KEY,
+ EVENTLOOP_THREADCOUNT_DEFAULT);
+ eventLoopGroup = new NioEventLoopGroup(threadCount,
+ new DefaultThreadFactory("NettyRpcServer", true, Thread.MAX_PRIORITY));
channelClass = NioServerSocketChannel.class;
}
ServerBootstrap bootstrap = new ServerBootstrap().group(eventLoopGroup).channel(channelClass)
.childOption(ChannelOption.TCP_NODELAY, tcpNoDelay)
.childOption(ChannelOption.SO_KEEPALIVE, tcpKeepAlive)
+ .childOption(ChannelOption.SO_REUSEADDR, true)
.childHandler(new ChannelInitializer<Channel>() {
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 37adbca..72eb351 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1680,7 +1680,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
if (!stores.isEmpty()) {
// initialize the thread pool for closing stores in parallel.
ThreadPoolExecutor storeCloserThreadPool =
- getStoreOpenAndCloseThreadPool("StoreCloserThread-" +
+ getStoreOpenAndCloseThreadPool("StoreCloser-" +
getRegionInfo().getRegionNameAsString());
CompletionService<Pair<byte[], Collection<HStoreFile>>> completionService =
new ExecutorCompletionService<>(storeCloserThreadPool);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 167bb3e..7e5d88b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -569,7 +569,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
}
// initialize the thread pool for opening store files in parallel..
ThreadPoolExecutor storeFileOpenerThreadPool =
- this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-" +
+ this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpener-" +
this.getColumnFamilyName());
CompletionService<HStoreFile> completionService =
new ExecutorCompletionService<>(storeFileOpenerThreadPool);
@@ -969,7 +969,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
if (!result.isEmpty()) {
// initialize the thread pool for closing store files in parallel.
ThreadPoolExecutor storeFileCloserThreadPool = this.region
- .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"
+ .getStoreFileOpenAndCloseThreadPool("StoreFileCloser-"
+ this.getColumnFamilyName());
// close each store file in parallel
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index 61f57ae..ab709d4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -228,8 +228,10 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
}
} else {
ThreadPoolExecutor threadPool =
- new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
- new ThreadFactoryBuilder().setNameFormat("AsyncFSWAL-%d").setDaemon(true).build());
+ new ThreadPoolExecutor(1, 1, 0L,
+ TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
+ new ThreadFactoryBuilder().setNameFormat("AsyncFSWAL-%d-" + rootDir.toString()).
+ setDaemon(true).build());
hasConsumerTask = () -> threadPool.getQueue().peek() == consumer;
this.consumeExecutor = threadPool;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
index 9bbc16d..a81c576 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
@@ -10,6 +10,7 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
+import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
@@ -27,7 +28,6 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
@@ -39,11 +39,11 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;
-import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles.LoadQueueItem;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.token.FsDelegationToken;
+import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;
+import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles.LoadQueueItem;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
@@ -51,16 +51,16 @@ import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* It is used for replicating HFile entries. It will first copy parallely all the hfiles to a local
* staging directory and then it will use ({@link LoadIncrementalHFiles} to prepare a collection of
* {@link LoadQueueItem} which will finally be loaded(replicated) into the table of this cluster.
+ * Call {@link #close()} when done.
*/
@InterfaceAudience.Private
-public class HFileReplicator {
+public class HFileReplicator implements Closeable {
/** Maximum number of threads to allow in pool to copy hfiles during replication */
public static final String REPLICATION_BULKLOAD_COPY_MAXTHREADS_KEY =
"hbase.replication.bulkload.copy.maxthreads";
@@ -109,7 +109,8 @@ public class HFileReplicator {
REPLICATION_BULKLOAD_COPY_MAXTHREADS_DEFAULT);
this.exec = Threads.getBoundedCachedThreadPool(maxCopyThreads, 60, TimeUnit.SECONDS,
new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("HFileReplicationCallable-%1$d").build());
+ .setNameFormat("HFileReplicationCopier-%1$d-" + this.sourceBaseNamespaceDirPath).
+ build());
this.copiesPerThread =
conf.getInt(REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_KEY,
REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_DEFAULT);
@@ -117,6 +118,13 @@ public class HFileReplicator {
sinkFs = FileSystem.get(conf);
}
+ @Override
+ public void close() throws IOException {
+ if (this.exec != null) {
+ this.exec.shutdown();
+ }
+ }
+
public Void replicate() throws IOException {
// Copy all the hfiles to the local file system
Map<String, Path> tableStagingDirsMap = copyHFilesToStagingDir();
@@ -132,8 +140,7 @@ public class HFileReplicator {
loadHFiles = new LoadIncrementalHFiles(conf);
loadHFiles.setClusterIds(sourceClusterIds);
} catch (Exception e) {
- LOG.error("Failed to initialize LoadIncrementalHFiles for replicating bulk loaded"
- + " data.", e);
+ LOG.error("Failed initialize LoadIncrementalHFiles for replicating bulk loaded data.", e);
throw new IOException(e);
}
Configuration newConf = HBaseConfiguration.create(conf);
@@ -148,19 +155,15 @@ public class HFileReplicator {
loadHFiles.prepareHFileQueue(stagingDir, table, queue, false);
if (queue.isEmpty()) {
- LOG.warn("Replication process did not find any files to replicate in directory "
- + stagingDir.toUri());
+ LOG.warn("Did not find any files to replicate in directory {}", stagingDir.toUri());
return null;
}
try (RegionLocator locator = connection.getRegionLocator(tableName)) {
-
fsDelegationToken.acquireDelegationToken(sinkFs);
-
// Set the staging directory which will be used by LoadIncrementalHFiles for loading the
// data
loadHFiles.setBulkToken(stagingDir.toString());
-
doBulkLoad(loadHFiles, table, queue, locator, maxRetries);
} finally {
cleanup(stagingDir.toString(), table);
@@ -177,13 +180,11 @@ public class HFileReplicator {
// need to reload split keys each iteration.
startEndKeys = locator.getStartEndKeys();
if (count != 0) {
- LOG.warn("Error occurred while replicating HFiles, retry attempt " + count + " with "
- + queue.size() + " files still remaining to replicate.");
+ LOG.warn("Error replicating HFiles; retry={} with {} remaining.", count, queue.size());
}
if (maxRetries != 0 && count >= maxRetries) {
- throw new IOException("Retry attempted " + count
- + " times without completing, bailing out.");
+ throw new IOException("Retry attempted " + count + " times without completing, bailing.");
}
count++;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index ae0a732..e910e53 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -29,7 +29,6 @@ import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -257,18 +256,13 @@ public class ReplicationSink {
List<String>>>>> entry : bulkLoadsPerClusters.entrySet()) {
Map<String, List<Pair<byte[], List<String>>>> bulkLoadHFileMap = entry.getValue();
if (bulkLoadHFileMap != null && !bulkLoadHFileMap.isEmpty()) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Started replicating bulk loaded data from cluster ids: {}.",
- entry.getKey().toString());
- }
- HFileReplicator hFileReplicator =
- new HFileReplicator(this.provider.getConf(this.conf, replicationClusterId),
+ LOG.debug("Replicating {} bulk loaded data", entry.getKey().toString());
+ Configuration providerConf = this.provider.getConf(this.conf, replicationClusterId);
+ try (HFileReplicator hFileReplicator = new HFileReplicator(providerConf,
sourceBaseNamespaceDirPath, sourceHFileArchiveDirPath, bulkLoadHFileMap, conf,
- getConnection(), entry.getKey());
- hFileReplicator.replicate();
- if(LOG.isDebugEnabled()) {
- LOG.debug("Finished replicating bulk loaded data from cluster id: {}",
- entry.getKey().toString());
+ getConnection(), entry.getKey())) {
+ hFileReplicator.replicate();
+ LOG.debug("Finished replicating {} bulk loaded data", entry.getKey().toString());
}
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
index 683d175..c0189c8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
@@ -110,7 +110,7 @@ public abstract class ModifyRegionUtils {
if (newRegions == null) return null;
int regionNumber = newRegions.length;
ThreadPoolExecutor exec = getRegionOpenAndInitThreadPool(conf,
- "RegionOpenAndInitThread-" + tableDescriptor.getTableName(), regionNumber);
+ "RegionOpenAndInit-" + tableDescriptor.getTableName(), regionNumber);
try {
return createRegions(exec, conf, rootDir, tableDescriptor, newRegions, task);
} finally {
@@ -230,8 +230,8 @@ public abstract class ModifyRegionUtils {
final String threadNamePrefix, int regionNumber) {
int maxThreads = Math.min(regionNumber, conf.getInt(
"hbase.hregion.open.and.init.threads.max", 16));
- ThreadPoolExecutor regionOpenAndInitThreadPool = Threads
- .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
+ ThreadPoolExecutor regionOpenAndInitThreadPool = Threads.
+ getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
Threads.newDaemonThreadFactory(threadNamePrefix));
return regionOpenAndInitThreadPool;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java
index 3e7b488..3e247f3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java
@@ -27,9 +27,7 @@ import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel;
import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
-
import java.util.concurrent.ThreadFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.yetus.audience.InterfaceAudience;
@@ -38,7 +36,6 @@ import org.apache.yetus.audience.InterfaceAudience;
*/
@InterfaceAudience.Private
public class NettyEventLoopGroupConfig {
-
private final EventLoopGroup group;
private final Class<? extends ServerChannel> serverChannelClass;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
index a440cde..beedd5e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -77,8 +77,8 @@ public class TestReplicationBase {
protected static Configuration CONF1 = UTIL1.getConfiguration();
protected static Configuration CONF2 = UTIL2.getConfiguration();
- protected static final int NUM_SLAVES1 = 2;
- protected static final int NUM_SLAVES2 = 4;
+ protected static final int NUM_SLAVES1 = 1;
+ protected static final int NUM_SLAVES2 = 1;
protected static final int NB_ROWS_IN_BATCH = 100;
protected static final int NB_ROWS_IN_BIG_BATCH =
NB_ROWS_IN_BATCH * 10;
diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml
index 64a1964..a317376 100644
--- a/hbase-server/src/test/resources/hbase-site.xml
+++ b/hbase-server/src/test/resources/hbase-site.xml
@@ -23,7 +23,7 @@
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
- <value>1000</value>
+ <value>100</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
@@ -35,7 +35,7 @@
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
- <value>1000</value>
+ <value>100</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
@@ -48,11 +48,13 @@
</property>
<property>
<name>hbase.regionserver.handler.count</name>
- <value>5</value>
+ <value>3</value>
+ <description>Default is 30</description>
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
- <value>6</value>
+ <value>3</value>
+ <description>Default is 20</description>
</property>
<property>
<name>hbase.ipc.server.read.threadpool.size</name>
@@ -83,7 +85,6 @@
<name>hbase.ipc.client.fallback-to-simple-auth-allowed</name>
<value>true</value>
</property>
-
<property>
<name>hbase.regionserver.info.port</name>
<value>-1</value>
@@ -158,4 +159,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.netty.eventloop.rpclient.thread.count</name>
+ <value>3</value>
+ <description>Default is unbounded</description>
+ </property>
+ <property>
+ <name>hbase.netty.eventloop.rpcserver.thread.count</name>
+ <value>3</value>
+ <description>Default is unbounded</description>
+ </property>
</configuration>
diff --git a/hbase-server/src/test/resources/hdfs-site.xml b/hbase-server/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-server/src/test/resources/hdfs-site.xml
+++ b/hbase-server/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-server/src/test/resources/log4j.properties b/hbase-server/src/test/resources/log4j.properties
index 785371d..8e590b1 100644
--- a/hbase-server/src/test/resources/log4j.properties
+++ b/hbase-server/src/test/resources/log4j.properties
@@ -66,6 +66,7 @@ log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN
log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN
+log4j.logger.io.netty.channel=DEBUG
# Enable this to get detailed connection error/retry logging.
# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE
log4j.logger.org.apache.directory=WARN
diff --git a/hbase-shell/src/test/resources/hbase-site.xml b/hbase-shell/src/test/resources/hbase-site.xml
index 99d2ab8..3a2b052 100644
--- a/hbase-shell/src/test/resources/hbase-site.xml
+++ b/hbase-shell/src/test/resources/hbase-site.xml
@@ -22,6 +22,22 @@
-->
<configuration>
<property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
@@ -29,4 +45,119 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-shell/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-shell/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-shell/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/hbase-thrift/src/test/resources/hbase-site.xml b/hbase-thrift/src/test/resources/hbase-site.xml
index b3fb0d9..a1e89ce 100644
--- a/hbase-thrift/src/test/resources/hbase-site.xml
+++ b/hbase-thrift/src/test/resources/hbase-site.xml
@@ -23,6 +23,22 @@
<configuration>
<property>
<name>hbase.regionserver.msginterval</name>
+ <value>100</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <value>100</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.msginterval</name>
<value>1000</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
@@ -154,4 +170,119 @@
Enable replay sanity checks on procedure tests.
</description>
</property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>3</value>
+ <description>Default is 30</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.metahandler.count</name>
+ <value>3</value>
+ <description>Default is 20</description>
+ </property>
+ <property>
+ <name>hbase.netty.worker.count</name>
+ <value>3</value>
+ <description>Default is 0</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.threads.max</name>
+ <value>6</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.htable.threads.max</name>
+ <value>3</value>
+ <description>Default is MAX_INTEGER</description>
+ </property>
+ <property>
+ <name>hbase.region.replica.replication.threads.max</name>
+ <value>7</value>
+ <description>Default is 256</description>
+ </property>
+ <property>
+ <name>hbase.rest.threads.max</name>
+ <value>5</value>
+ <description>Default is 100</description>
+ </property>
+ <property>
+ <name>hbase.replication.bulkload.copy.maxthreads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.loadincremental.threads.max</name>
+ <value>1</value>
+ <description>Default is # of CPUs</description>
+ </property>
+ <property>
+ <name>hbase.hstore.flusher.count</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.oldwals.cleaner.thread.size</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.master.procedure.threads</name>
+ <value>5</value>
+ <description>Default is at least 16</description>
+ </property>
+ <property>
+ <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+ <value>3</value>
+ <description>Default is 128</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.closeregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+ <value>1</value>
+ <description>Default is 3</description>
+ </property>
+ <property>
+ <name>hbase.storescanner.parallel.seek.threads</name>
+ <value>3</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hfile.compaction.discharger.thread.count</name>
+ <value>1</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.executor.refresh.peer.threads</name>
+ <value>1</value>
+ <description>Default is 2</description>
+ </property>
+ <property>
+ <name>hbase.hregion.open.and.init.threads.max</name>
+ <value>3</value>
+ <description>Default is 16 or # of Regions</description>
+ </property>
+ <property>
+ <name>hbase.master.handler.count</name>
+ <value>7</value>
+ <description>Default is 25</description>
+ </property>
+ <property>
+ <name>hbase.replication.source.maxthreads</name>
+ <value></value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>hbase.hconnection.meta.lookup.threads.max</name>
+ <value>5</value>
+ <description>Default is 128</description>
+ </property>
</configuration>
diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-thrift/src/test/resources/hdfs-site.xml
similarity index 63%
copy from hbase-rest/src/test/resources/hdfs-site.xml
copy to hbase-thrift/src/test/resources/hdfs-site.xml
index 03be0c7..9230105 100644
--- a/hbase-rest/src/test/resources/hdfs-site.xml
+++ b/hbase-thrift/src/test/resources/hdfs-site.xml
@@ -29,4 +29,28 @@
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>dfs.datanode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>5</value>
+ <description>Default is 10</description>
+ </property>
+ <!--
+ Constraining this config makes tests fail.
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16</value>
+ <description>Default is 4096. If constrain this
+ too much, tests do not complete.</description>
+ </property>
+ -->
+</configuration>
diff --git a/pom.xml b/pom.xml
index 8886cc4..0559f67 100755
--- a/pom.xml
+++ b/pom.xml
@@ -584,8 +584,17 @@
<testFailureIgnore>${surefire.testFailureIgnore}</testFailureIgnore>
<forkedProcessTimeoutInSeconds>${surefire.timeout}</forkedProcessTimeoutInSeconds>
<redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
+ <!-- These are incontext system properties set on mvn. For forked
+ jvms, see hbase-surefire.argLine... So, we might be dup'ing
+ configs here and over on what we pass to the forked jvm.
+ -->
<systemPropertyVariables>
<test.build.classes>${test.build.classes}</test.build.classes>
+ <!--And for eventloops that have no explicit configuration, netty sets
+ nioeventloopgroup thread count to CPU count * 2. Thats too much
+ for mini clusters/tests.
+ -->
+ <io.netty.eventLoopThreads>3</io.netty.eventLoopThreads>
</systemPropertyVariables>
<excludes>
<!-- users can add -D option to skip particular test classes
@@ -1416,11 +1425,16 @@
<surefire.Xmx>2800m</surefire.Xmx>
<surefire.cygwinXmx>2800m</surefire.cygwinXmx>
<!--Mark our test runs with '-Dhbase.build.id' so we can identify a surefire test as ours in a process listing
+
+ And for netty eventloops that have no explicit configuration, netty sets
+ nioeventloopgroup thread count to CPU count * 2. Thats too much for mini
+ clusters/tests.
-->
<hbase-surefire.argLine>-enableassertions -Dhbase.build.id=${build.id} -Xmx${surefire.Xmx}
-Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true
-Djava.awt.headless=true -Djdk.net.URLClassPath.disableClassPathURLCheck=true
-Dorg.apache.hbase.thirdparty.io.netty.leakDetection.level=advanced
+ -Dio.netty.eventLoopThreads=3
</hbase-surefire.argLine>
<hbase-surefire.cygwin-argLine>-enableassertions -Xmx${surefire.cygwinXmx}
-Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true