You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ar...@apache.org on 2014/04/24 19:41:06 UTC
svn commit: r1589805 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/main/resources/
Author: arp
Date: Thu Apr 24 17:41:05 2014
New Revision: 1589805
URL: http://svn.apache.org/r1589805
Log:
HDFS-6273: Merging r1589803 from trunk to branch-2.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1589805&r1=1589804&r2=1589805&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Apr 24 17:41:05 2014
@@ -64,6 +64,9 @@ Release 2.5.0 - UNRELEASED
HDFS-5693. Few NN metrics data points were collected via JMX when NN
is under heavy load. (Ming Ma via jing9)
+ HDFS-6273. Config options to allow wildcard endpoints for namenode HTTP
+ and HTTPS servers. (Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1589805&r1=1589804&r2=1589805&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Apr 24 17:41:05 2014
@@ -126,6 +126,7 @@ public class DFSConfigKeys extends Commo
public static final int DFS_NAMENODE_HTTP_PORT_DEFAULT = 50070;
public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY = "dfs.namenode.http-address";
public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTP_PORT_DEFAULT;
+ public static final String DFS_NAMENODE_HTTP_BIND_HOST_KEY = "dfs.namenode.http-bind-host";
public static final String DFS_NAMENODE_RPC_ADDRESS_KEY = "dfs.namenode.rpc-address";
public static final String DFS_NAMENODE_RPC_BIND_HOST_KEY = "dfs.namenode.rpc-bind-host";
public static final String DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY = "dfs.namenode.servicerpc-address";
@@ -295,6 +296,7 @@ public class DFSConfigKeys extends Commo
public static final String DFS_DATANODE_DATA_DIR_KEY = "dfs.datanode.data.dir";
public static final int DFS_NAMENODE_HTTPS_PORT_DEFAULT = 50470;
public static final String DFS_NAMENODE_HTTPS_ADDRESS_KEY = "dfs.namenode.https-address";
+ public static final String DFS_NAMENODE_HTTPS_BIND_HOST_KEY = "dfs.namenode.https-bind-host";
public static final String DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTPS_PORT_DEFAULT;
public static final String DFS_NAMENODE_NAME_DIR_KEY = "dfs.namenode.name.dir";
public static final String DFS_NAMENODE_EDITS_DIR_KEY = "dfs.namenode.edits.dir";
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1589805&r1=1589804&r2=1589805&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Thu Apr 24 17:41:05 2014
@@ -174,6 +174,8 @@ public class NameNode implements NameNod
DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY,
DFS_NAMENODE_HTTP_ADDRESS_KEY,
DFS_NAMENODE_HTTPS_ADDRESS_KEY,
+ DFS_NAMENODE_HTTP_BIND_HOST_KEY,
+ DFS_NAMENODE_HTTPS_BIND_HOST_KEY,
DFS_NAMENODE_KEYTAB_FILE_KEY,
DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY,
@@ -443,6 +445,29 @@ public class NameNode implements NameNod
return getHttpAddress(conf);
}
+ /**
+ * HTTP server address for binding the endpoint. This method is
+ * for use by the NameNode and its derivatives. It may return
+ * a different address than the one that should be used by clients to
+ * connect to the NameNode. See
+ * {@link DFSConfigKeys#DFS_NAMENODE_HTTP_BIND_HOST_KEY}
+ *
+ * @param conf
+ * @return
+ */
+ protected InetSocketAddress getHttpServerBindAddress(Configuration conf) {
+ InetSocketAddress bindAddress = getHttpServerAddress(conf);
+
+ // If DFS_NAMENODE_HTTP_BIND_HOST_KEY exists then it overrides the
+ // host name portion of DFS_NAMENODE_HTTP_ADDRESS_KEY.
+ final String bindHost = conf.getTrimmed(DFS_NAMENODE_HTTP_BIND_HOST_KEY);
+ if (bindHost != null && !bindHost.isEmpty()) {
+ bindAddress = new InetSocketAddress(bindHost, bindAddress.getPort());
+ }
+
+ return bindAddress;
+ }
+
/** @return the NameNode HTTP address. */
public static InetSocketAddress getHttpAddress(Configuration conf) {
return NetUtils.createSocketAddr(
@@ -607,7 +632,7 @@ public class NameNode implements NameNod
}
private void startHttpServer(final Configuration conf) throws IOException {
- httpServer = new NameNodeHttpServer(conf, this, getHttpServerAddress(conf));
+ httpServer = new NameNodeHttpServer(conf, this, getHttpServerBindAddress(conf));
httpServer.start();
httpServer.setStartupProgress(startupProgress);
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1589805&r1=1589804&r2=1589805&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java Thu Apr 24 17:41:05 2014
@@ -108,6 +108,16 @@ public class NameNodeHttpServer {
DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT);
InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);
+ if (httpsAddr != null) {
+ // If DFS_NAMENODE_HTTPS_BIND_HOST_KEY exists then it overrides the
+ // host name portion of DFS_NAMENODE_HTTPS_ADDRESS_KEY.
+ final String bindHost =
+ conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_HTTPS_BIND_HOST_KEY);
+ if (bindHost != null && !bindHost.isEmpty()) {
+ httpsAddr = new InetSocketAddress(bindHost, httpsAddr.getPort());
+ }
+ }
+
HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
httpAddr, httpsAddr, "hdfs",
DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1589805&r1=1589804&r2=1589805&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Thu Apr 24 17:41:05 2014
@@ -365,6 +365,12 @@ class NameNodeRpcServer implements Namen
return clientRpcServer;
}
+ /** Allow access to the service RPC server for testing */
+ @VisibleForTesting
+ RPC.Server getServiceRpcServer() {
+ return serviceRpcServer;
+ }
+
/**
* Start client and service RPC servers.
*/
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1589805&r1=1589804&r2=1589805&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Thu Apr 24 17:41:05 2014
@@ -55,11 +55,11 @@
<name>dfs.namenode.rpc-bind-host</name>
<value></value>
<description>
- The actual address the server will bind to. If this optional address is
- set, the RPC server will bind to this address and the port specified in
- dfs.namenode.rpc-address for the RPC server. It can also be specified
- per name node or name service for HA/Federation. This is most useful for
- making name node listen to all interfaces by setting to 0.0.0.0.
+ The actual address the RPC server will bind to. If this optional address is
+ set, it overrides only the hostname portion of dfs.namenode.rpc-address.
+ It can also be specified per name node or name service for HA/Federation.
+ This is useful for making the name node listen on all interfaces by
+ setting it to 0.0.0.0.
</description>
</property>
@@ -80,11 +80,11 @@
<name>dfs.namenode.servicerpc-bind-host</name>
<value></value>
<description>
- The actual address the server will bind to. If this optional address is
- set, the service RPC server will bind to this address and the port
- specified in dfs.namenode.servicerpc-address. It can also be specified
- per name node or name service for HA/Federation. This is most useful for
- making name node listen to all interfaces by setting to 0.0.0.0.
+ The actual address the service RPC server will bind to. If this optional address is
+ set, it overrides only the hostname portion of dfs.namenode.servicerpc-address.
+ It can also be specified per name node or name service for HA/Federation.
+ This is useful for making the name node listen on all interfaces by
+ setting it to 0.0.0.0.
</description>
</property>
@@ -143,6 +143,18 @@
</property>
<property>
+ <name>dfs.namenode.http-bind-host</name>
+ <value></value>
+ <description>
+ The actual adress the HTTP server will bind to. If this optional address
+ is set, it overrides only the hostname portion of dfs.namenode.http-address.
+ It can also be specified per name node or name service for HA/Federation.
+ This is useful for making the name node HTTP server listen on all
+ interfaces by setting it to 0.0.0.0.
+ </description>
+</property>
+
+<property>
<name>dfs.https.enable</name>
<value>false</value>
<description>
@@ -207,6 +219,18 @@
<description>The namenode secure http server address and port.</description>
</property>
+<property>
+ <name>dfs.namenode.https-bind-host</name>
+ <value></value>
+ <description>
+ The actual adress the HTTPS server will bind to. If this optional address
+ is set, it overrides only the hostname portion of dfs.namenode.https-address.
+ It can also be specified per name node or name service for HA/Federation.
+ This is useful for making the name node HTTPS server listen on all
+ interfaces by setting it to 0.0.0.0.
+ </description>
+</property>
+
<property>
<name>dfs.datanode.dns.interface</name>
<value>default</value>