You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ec...@apache.org on 2015/10/02 17:57:12 UTC
[33/52] [abbrv] hadoop git commit: HDFS-8696. Make the lower and
higher watermark in the DN Netty server configurable. Contributed by Xiaobing
Zhou.
HDFS-8696. Make the lower and higher watermark in the DN Netty server configurable. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f335e4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f335e4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f335e4f
Branch: refs/heads/HADOOP-11890
Commit: 6f335e4f0ea857ef11ff24c392bf6e549b5db406
Parents: 8703301
Author: Haohui Mai <wh...@apache.org>
Authored: Tue Sep 29 14:59:13 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Tue Sep 29 14:59:13 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 ++++++
.../hdfs/server/datanode/web/DatanodeHttpServer.java | 15 +++++++++++++++
3 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f335e4f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0bca0e4..7b62b97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -997,6 +997,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9165. Move entries in META-INF/services/o.a.h.fs.FileSystem to
hdfs-client. (Mingliang Liu via wheat9)
+ HDFS-8696. Make the lower and higher watermark in the DN Netty server
+ configurable. (Xiaobing Zhou via wheat9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f335e4f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 055d7e3..37d74e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -63,6 +63,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
HdfsClientConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT;
public static final String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT;
+ public static final String DFS_WEBHDFS_NETTY_LOW_WATERMARK =
+ "dfs.webhdfs.netty.low.watermark";
+ public static final int DFS_WEBHDFS_NETTY_LOW_WATERMARK_DEFAULT = 32768;
+ public static final String DFS_WEBHDFS_NETTY_HIGH_WATERMARK =
+ "dfs.webhdfs.netty.high.watermark";
+ public static final int DFS_WEBHDFS_NETTY_HIGH_WATERMARK_DEFAULT = 65535;
// HA related configuration
public static final String DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY = "dfs.datanode.restart.replica.expiration";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f335e4f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index 62c98e7..441d520 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -21,6 +21,7 @@ import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFactory;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
@@ -30,10 +31,12 @@ import io.netty.handler.codec.http.HttpRequestDecoder;
import io.netty.handler.codec.http.HttpResponseEncoder;
import io.netty.handler.ssl.SslHandler;
import io.netty.handler.stream.ChunkedWriteHandler;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
@@ -117,6 +120,18 @@ public class DatanodeHttpServer implements Closeable {
conf, confForCreate));
}
});
+
+ this.httpServer.childOption(
+ ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK,
+ conf.getInt(
+ DFSConfigKeys.DFS_WEBHDFS_NETTY_HIGH_WATERMARK,
+ DFSConfigKeys.DFS_WEBHDFS_NETTY_HIGH_WATERMARK_DEFAULT));
+ this.httpServer.childOption(
+ ChannelOption.WRITE_BUFFER_LOW_WATER_MARK,
+ conf.getInt(
+ DFSConfigKeys.DFS_WEBHDFS_NETTY_LOW_WATERMARK,
+ DFSConfigKeys.DFS_WEBHDFS_NETTY_LOW_WATERMARK_DEFAULT));
+
if (externalHttpChannel == null) {
httpServer.channel(NioServerSocketChannel.class);
} else {