You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ll...@apache.org on 2013/08/21 12:12:14 UTC
svn commit: r1516128 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src:
main/java/org/apache/hadoop/hdfs/qjournal/server/
main/java/org/apache/hadoop/hdfs/server/datanode/
main/java/org/apache/hadoop/hdfs/server/namenode/ test/java/org/apac...
Author: llu
Date: Wed Aug 21 10:12:13 2013
New Revision: 1516128
URL: http://svn.apache.org/r1516128
Log:
HADOOP-9784. Add a builder for HttpServer. (Junping Du via llu)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java?rev=1516128&r1=1516127&r2=1516128&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java Wed Aug 21 10:12:13 2013
@@ -36,8 +36,8 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
/**
* Encapsulates the HTTP server started by the Journal Service.
@@ -69,16 +69,15 @@ public class JournalNodeHttpServer {
bindAddr.getHostName()));
int tmpInfoPort = bindAddr.getPort();
- httpServer = new HttpServer("journal", bindAddr.getHostName(),
- tmpInfoPort, tmpInfoPort == 0, conf, new AccessControlList(conf
- .get(DFS_ADMIN, " "))) {
- {
- if (UserGroupInformation.isSecurityEnabled()) {
- initSpnego(conf, DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY,
- DFSUtil.getSpnegoKeytabKey(conf, DFS_JOURNALNODE_KEYTAB_FILE_KEY));
- }
- }
- };
+ httpServer = new HttpServer.Builder().setName("journal")
+ .setBindAddress(bindAddr.getHostName()).setPort(tmpInfoPort)
+ .setFindPort(tmpInfoPort == 0).setConf(conf).setACL(
+ new AccessControlList(conf.get(DFS_ADMIN, " ")))
+ .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
+ .setUsernameConfKey(
+ DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY)
+ .setKeytabConfKey(DFSUtil.getSpnegoKeytabKey(conf,
+ DFS_JOURNALNODE_KEYTAB_FILE_KEY)).build();
httpServer.setAttribute(JN_ATTRIBUTE_KEY, localJournalNode);
httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
httpServer.addInternalServlet("getJournal", "/getJournal",
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1516128&r1=1516127&r2=1516128&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Wed Aug 21 10:12:13 2013
@@ -385,11 +385,15 @@ public class DataNode extends Configured
String infoHost = infoSocAddr.getHostName();
int tmpInfoPort = infoSocAddr.getPort();
this.infoServer = (secureResources == null)
- ? new HttpServer("datanode", infoHost, tmpInfoPort, tmpInfoPort == 0,
- conf, new AccessControlList(conf.get(DFS_ADMIN, " ")))
- : new HttpServer("datanode", infoHost, tmpInfoPort, tmpInfoPort == 0,
- conf, new AccessControlList(conf.get(DFS_ADMIN, " ")),
- secureResources.getListener());
+ ? new HttpServer.Builder().setName("datanode")
+ .setBindAddress(infoHost).setPort(tmpInfoPort)
+ .setFindPort(tmpInfoPort == 0).setConf(conf)
+ .setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))).build()
+ : new HttpServer.Builder().setName("datanode")
+ .setBindAddress(infoHost).setPort(tmpInfoPort)
+ .setFindPort(tmpInfoPort == 0).setConf(conf)
+ .setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")))
+ .setConnector(secureResources.getListener()).build();
LOG.info("Opened info server at " + infoHost + ":" + tmpInfoPort);
if (conf.getBoolean(DFS_HTTPS_ENABLE_KEY, false)) {
boolean needClientAuth = conf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY,
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1516128&r1=1516127&r2=1516128&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java Wed Aug 21 10:12:13 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
+
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.HashMap;
@@ -70,65 +71,30 @@ public class NameNodeHttpServer {
public void start() throws IOException {
final String infoHost = bindAddress.getHostName();
int infoPort = bindAddress.getPort();
-
- httpServer = new HttpServer("hdfs", infoHost, infoPort,
- infoPort == 0, conf,
- new AccessControlList(conf.get(DFS_ADMIN, " "))) {
- {
- // Add SPNEGO support to NameNode
- if (UserGroupInformation.isSecurityEnabled()) {
- initSpnego(conf,
- DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY,
- DFSUtil.getSpnegoKeytabKey(conf,
- DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
- }
- if (WebHdfsFileSystem.isEnabled(conf, LOG)) {
- //add SPNEGO authentication filter for webhdfs
- final String name = "SPNEGO";
- final String classname = AuthFilter.class.getName();
- final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*";
- Map<String, String> params = getAuthFilterParams(conf);
- defineFilter(webAppContext, name, classname, params,
- new String[]{pathSpec});
- LOG.info("Added filter '" + name + "' (class=" + classname + ")");
-
- // add webhdfs packages
- addJerseyResourcePackage(
- NamenodeWebHdfsMethods.class.getPackage().getName()
- + ";" + Param.class.getPackage().getName(), pathSpec);
- }
- }
-
- private Map<String, String> getAuthFilterParams(Configuration conf)
- throws IOException {
- Map<String, String> params = new HashMap<String, String>();
- String principalInConf = conf
- .get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
- if (principalInConf != null && !principalInConf.isEmpty()) {
- params
- .put(
- DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
- SecurityUtil.getServerPrincipal(principalInConf,
- bindAddress.getHostName()));
- } else if (UserGroupInformation.isSecurityEnabled()) {
- LOG.error("WebHDFS and security are enabled, but configuration property '" +
- DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY +
- "' is not set.");
- }
- String httpKeytab = conf.get(DFSUtil.getSpnegoKeytabKey(conf,
- DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
- if (httpKeytab != null && !httpKeytab.isEmpty()) {
- params.put(
- DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY,
- httpKeytab);
- } else if (UserGroupInformation.isSecurityEnabled()) {
- LOG.error("WebHDFS and security are enabled, but configuration property '" +
- DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY +
- "' is not set.");
- }
- return params;
+ httpServer = new HttpServer.Builder().setName("hdfs")
+ .setBindAddress(infoHost).setPort(infoPort)
+ .setFindPort(infoPort == 0).setConf(conf).setACL(
+ new AccessControlList(conf.get(DFS_ADMIN, " ")))
+ .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
+ .setUsernameConfKey(
+ DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY)
+ .setKeytabConfKey(DFSUtil.getSpnegoKeytabKey(conf,
+ DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY)).build();
+ if (WebHdfsFileSystem.isEnabled(conf, HttpServer.LOG)) {
+ //add SPNEGO authentication filter for webhdfs
+ final String name = "SPNEGO";
+ final String classname = AuthFilter.class.getName();
+ final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*";
+ Map<String, String> params = getAuthFilterParams(conf);
+ httpServer.defineFilter(httpServer.getWebAppContext(), name, classname, params,
+ new String[]{pathSpec});
+ HttpServer.LOG.info("Added filter '" + name + "' (class=" + classname + ")");
+
+ // add webhdfs packages
+ httpServer.addJerseyResourcePackage(
+ NamenodeWebHdfsMethods.class.getPackage().getName()
+ + ";" + Param.class.getPackage().getName(), pathSpec);
}
- };
boolean certSSL = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, false);
if (certSSL) {
@@ -153,6 +119,38 @@ public class NameNodeHttpServer {
httpServer.start();
httpAddress = new InetSocketAddress(bindAddress.getAddress(), httpServer.getPort());
}
+
+ private Map<String, String> getAuthFilterParams(Configuration conf)
+ throws IOException {
+ Map<String, String> params = new HashMap<String, String>();
+ String principalInConf = conf
+ .get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
+ if (principalInConf != null && !principalInConf.isEmpty()) {
+ params
+ .put(
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ SecurityUtil.getServerPrincipal(principalInConf,
+ bindAddress.getHostName()));
+ } else if (UserGroupInformation.isSecurityEnabled()) {
+ HttpServer.LOG.error(
+ "WebHDFS and security are enabled, but configuration property '" +
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY +
+ "' is not set.");
+ }
+ String httpKeytab = conf.get(DFSUtil.getSpnegoKeytabKey(conf,
+ DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
+ if (httpKeytab != null && !httpKeytab.isEmpty()) {
+ params.put(
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY,
+ httpKeytab);
+ } else if (UserGroupInformation.isSecurityEnabled()) {
+ HttpServer.LOG.error(
+ "WebHDFS and security are enabled, but configuration property '" +
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY +
+ "' is not set.");
+ }
+ return params;
+ }
public void stop() throws Exception {
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1516128&r1=1516127&r2=1516128&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Wed Aug 21 10:12:13 2013
@@ -256,19 +256,15 @@ public class SecondaryNameNode implement
// initialize the webserver for uploading files.
int tmpInfoPort = infoSocAddr.getPort();
- infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort,
- tmpInfoPort == 0, conf,
- new AccessControlList(conf.get(DFS_ADMIN, " "))) {
- {
- if (UserGroupInformation.isSecurityEnabled()) {
- initSpnego(
- conf,
- DFSConfigKeys.DFS_SECONDARY_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY,
- DFSUtil.getSpnegoKeytabKey(conf,
- DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY));
- }
- }
- };
+ infoServer = new HttpServer.Builder().setName("secondary")
+ .setBindAddress(infoBindAddress).setPort(tmpInfoPort)
+ .setFindPort(tmpInfoPort == 0).setConf(conf).setACL(
+ new AccessControlList(conf.get(DFS_ADMIN, " ")))
+ .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
+ .setUsernameConfKey(
+ DFSConfigKeys.DFS_SECONDARY_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY)
+ .setKeytabConfKey(DFSUtil.getSpnegoKeytabKey(conf,
+ DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY)).build();
infoServer.setAttribute("secondary.name.node", this);
infoServer.setAttribute("name.system.image", checkpointImage);
infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java?rev=1516128&r1=1516127&r2=1516128&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileInputStream.java Wed Aug 21 10:12:13 2013
@@ -42,7 +42,8 @@ public class TestEditLogFileInputStream
@Test
public void testReadURL() throws Exception {
// Start a simple web server which hosts the log data.
- HttpServer server = new HttpServer("test", "0.0.0.0", 0, true);
+ HttpServer server = new HttpServer.Builder().setName("test")
+ .setBindAddress("0.0.0.0").setPort(0).setFindPort(true).build();
server.start();
try {
server.addServlet("fakeLog", "/fakeLog", FakeLogServlet.class);