You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/06/18 20:24:50 UTC
[12/50] [abbrv] hadoop git commit: HDFS-8572. DN always uses
HTTP/localhost@REALM principals in SPNEGO. Contributed by Haohui Mai.
HDFS-8572. DN always uses HTTP/localhost@REALM principals in SPNEGO. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6ff0e81
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6ff0e81
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6ff0e81
Branch: refs/heads/YARN-2928
Commit: d6ff0e81d6cf8ecd256be526efcd6538e60f17bf
Parents: bc9d48a
Author: Haohui Mai <wh...@apache.org>
Authored: Thu Jun 11 18:53:29 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Thu Jun 18 11:10:05 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/server/datanode/DataNode.java | 36 ++------------
.../server/datanode/web/DatanodeHttpServer.java | 52 ++++++++++++++++++--
3 files changed, 55 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6ff0e81/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 033451e..51a0897 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -996,6 +996,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8583. Document that NFS gateway does not work with rpcbind
on SLES 11. (Arpit Agarwal)
+ HDFS-8572. DN always uses HTTP/localhost@REALM principals in SPNEGO.
+ (wheat9)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6ff0e81/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index f73eb66..ed2925b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
@@ -148,7 +148,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
-import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
@@ -163,7 +162,6 @@ import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.io.nativeio.NativeIO;
@@ -180,7 +178,6 @@ import org.apache.hadoop.security.SaslPropertiesResolver;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
-import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.tracing.SpanReceiverHost;
@@ -299,7 +296,6 @@ public class DataNode extends ReconfigurableBase
private volatile boolean heartbeatsDisabledForTests = false;
private DataStorage storage = null;
- private HttpServer2 infoServer = null;
private DatanodeHttpServer httpServer = null;
private int infoPort;
private int infoSecurePort;
@@ -761,29 +757,12 @@ public class DataNode extends ReconfigurableBase
*/
private void startInfoServer(Configuration conf)
throws IOException {
- Configuration confForInfoServer = new Configuration(conf);
- confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
- HttpServer2.Builder builder = new HttpServer2.Builder()
- .setName("datanode")
- .setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")))
- .addEndpoint(URI.create("http://localhost:0"))
- .setFindPort(true);
-
- this.infoServer = builder.build();
-
- this.infoServer.setAttribute("datanode", this);
- this.infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);
- this.infoServer.addServlet(null, "/blockScannerReport",
- BlockScanner.Servlet.class);
-
- this.infoServer.start();
- InetSocketAddress jettyAddr = infoServer.getConnectorAddress(0);
-
// SecureDataNodeStarter will bind the privileged port to the channel if
// the DN is started by JSVC, pass it along.
ServerSocketChannel httpServerChannel = secureResources != null ?
- secureResources.getHttpServerChannel() : null;
- this.httpServer = new DatanodeHttpServer(conf, jettyAddr, httpServerChannel);
+ secureResources.getHttpServerChannel() : null;
+
+ this.httpServer = new DatanodeHttpServer(conf, httpServerChannel);
httpServer.start();
if (httpServer.getHttpAddress() != null) {
infoPort = httpServer.getHttpAddress().getPort();
@@ -1715,13 +1694,6 @@ public class DataNode extends ReconfigurableBase
shutdownPeriodicScanners();
// Stop the web server
- if (infoServer != null) {
- try {
- infoServer.stop();
- } catch (Exception e) {
- LOG.warn("Exception shutting down DataNode", e);
- }
- }
if (httpServer != null) {
try {
httpServer.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6ff0e81/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index 613d3ce..e9ad92f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web;
-import io.netty.channel.ChannelFactory;
import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.ChannelFactory;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
@@ -35,9 +35,13 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.ssl.SSLFactory;
import java.io.Closeable;
@@ -46,13 +50,17 @@ import java.net.BindException;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.net.SocketException;
+import java.net.URI;
import java.nio.channels.ServerSocketChannel;
import java.security.GeneralSecurityException;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY;
public class DatanodeHttpServer implements Closeable {
+ private final HttpServer2 infoServer;
private final EventLoopGroup bossGroup;
private final EventLoopGroup workerGroup;
private final ServerSocketChannel externalHttpChannel;
@@ -66,10 +74,31 @@ public class DatanodeHttpServer implements Closeable {
static final Log LOG = LogFactory.getLog(DatanodeHttpServer.class);
- public DatanodeHttpServer(final Configuration conf, final InetSocketAddress
- jettyAddr, final ServerSocketChannel externalHttpChannel)
+ public DatanodeHttpServer(final Configuration conf,
+ final ServerSocketChannel externalHttpChannel)
throws IOException {
this.conf = conf;
+
+ Configuration confForInfoServer = new Configuration(conf);
+ confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
+ HttpServer2.Builder builder = new HttpServer2.Builder()
+ .setName("datanode")
+ .setConf(confForInfoServer)
+ .setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")))
+ .hostName(getHostnameForSpnegoPrincipal(confForInfoServer))
+ .addEndpoint(URI.create("http://localhost:0"))
+ .setFindPort(true);
+
+ this.infoServer = builder.build();
+
+ this.infoServer.setAttribute("datanode", this);
+ this.infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);
+ this.infoServer.addServlet(null, "/blockScannerReport",
+ BlockScanner.Servlet.class);
+
+ this.infoServer.start();
+ final InetSocketAddress jettyAddr = infoServer.getConnectorAddress(0);
+
this.confForCreate = new Configuration(conf);
confForCreate.set(FsPermission.UMASK_LABEL, "000");
@@ -191,5 +220,20 @@ public class DatanodeHttpServer implements Closeable {
if (externalHttpChannel != null) {
externalHttpChannel.close();
}
+ try {
+ infoServer.stop();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ private static String getHostnameForSpnegoPrincipal(Configuration conf) {
+ String addr = conf.getTrimmed(DFS_DATANODE_HTTP_ADDRESS_KEY, null);
+ if (addr == null) {
+ addr = conf.getTrimmed(DFS_DATANODE_HTTPS_ADDRESS_KEY,
+ DFS_DATANODE_HTTPS_ADDRESS_DEFAULT);
+ }
+ InetSocketAddress inetSocker = NetUtils.createSocketAddr(addr);
+ return inetSocker.getHostString();
}
-}
+}
\ No newline at end of file