You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2020/03/26 00:20:45 UTC
[hadoop] branch branch-3.2 updated (152cbc6 -> 6cd9290)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a change to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 152cbc6 HDFS-15219. DFS Client will stuck when ResponseProcessor.run throw Error (#1902). Contributed by zhengchenyu.
new eca7bc7 HDFS-14006. Refactor name node to allow different token verification implementations. Contributed by CR Hota.
new ba6b3a3 HDFS-14434. Ignore user.name query parameter in secure WebHDFS. Contributed by KWON BYUNGCHANG
new 6cd9290 HDFS-14647. NPE during secure namenode startup. Contributed by Fengnan Li.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +-
.../hadoop/hdfs/server/common/JspHelper.java | 16 +-
...pPutFailedException.java => TokenVerifier.java} | 24 ++-
.../hadoop/hdfs/server/namenode/NameNode.java | 13 +-
.../hdfs/server/namenode/NameNodeHttpServer.java | 6 +
.../hadoop/hdfs/server/common/TestJspHelper.java | 122 ++++++++----
.../apache/hadoop/hdfs/web/TestWebHdfsTokens.java | 218 +++++++++++++--------
.../org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 47 +++--
8 files changed, 303 insertions(+), 159 deletions(-)
copy hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/{HttpPutFailedException.java => TokenVerifier.java} (63%)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: HDFS-14006. Refactor name node to allow different
token verification implementations. Contributed by CR Hota.
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit eca7bc7ac4292ff08b9e1ea2e22116d6e58e8b95
Author: Giovanni Matteo Fumarola <gi...@apache.org>
AuthorDate: Fri Dec 14 11:10:54 2018 -0800
HDFS-14006. Refactor name node to allow different token verification implementations. Contributed by CR Hota.
(cherry picked from commit 00d5e631b596f8712600879366e5283829e7ee5d)
---
.../hadoop/hdfs/server/common/JspHelper.java | 8 ++---
.../hadoop/hdfs/server/common/TokenVerifier.java | 35 ++++++++++++++++++++++
.../hadoop/hdfs/server/namenode/NameNode.java | 9 +++++-
.../hdfs/server/namenode/NameNodeHttpServer.java | 6 ++++
4 files changed, 53 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
index 498a093..eb488e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
@@ -23,7 +23,6 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
import org.apache.hadoop.hdfs.web.resources.DelegationParam;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
@@ -176,10 +175,11 @@ public class JspHelper {
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.readFields(in);
if (context != null) {
- final NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context);
- if (nn != null) {
+ final TokenVerifier<DelegationTokenIdentifier> tokenVerifier =
+ NameNodeHttpServer.getTokenVerifierFromContext(context);
+ if (tokenVerifier != null) {
// Verify the token.
- nn.getNamesystem().verifyToken(id, token.getPassword());
+ tokenVerifier.verifyToken(id, token.getPassword());
}
}
UserGroupInformation ugi = id.getUser();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/TokenVerifier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/TokenVerifier.java
new file mode 100644
index 0000000..5691f0c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/TokenVerifier.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.common;
+
+import java.io.IOException;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+
+/**
+ * Interface to verify delegation tokens passed through WebHDFS.
+ * Implementations are intercepted by JspHelper that pass delegation token
+ * for verification.
+ */
+public interface TokenVerifier<T extends AbstractDelegationTokenIdentifier> {
+
+ /* Verify delegation token passed through WebHDFS
+ * Name node, Router implement this for JspHelper to verify token
+ */
+ void verifyToken(T t, byte[] password) throws IOException;
+
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 4556b89..ba4b730 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap;
import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -56,6 +57,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeSt
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.common.TokenVerifier;
import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -217,7 +219,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DE
**********************************************************/
@InterfaceAudience.Private
public class NameNode extends ReconfigurableBase implements
- NameNodeStatusMXBean {
+ NameNodeStatusMXBean, TokenVerifier<DelegationTokenIdentifier> {
static{
HdfsConfiguration.init();
}
@@ -674,6 +676,11 @@ public class NameNode extends ReconfigurableBase implements
return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
}
+ @Override
+ public void verifyToken(DelegationTokenIdentifier id, byte[] password)
+ throws IOException {
+ namesystem.verifyToken(id, password);
+ }
/**
* Login as the configured user for the NameNode.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index a4c8415..a767c46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.common.TokenVerifier;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.web.AuthFilter;
@@ -311,6 +312,11 @@ public class NameNodeHttpServer {
return (NameNode)context.getAttribute(NAMENODE_ATTRIBUTE_KEY);
}
+ public static TokenVerifier
+ getTokenVerifierFromContext(ServletContext context) {
+ return (TokenVerifier) context.getAttribute(NAMENODE_ATTRIBUTE_KEY);
+ }
+
static Configuration getConfFromContext(ServletContext context) {
return (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: HDFS-14434. Ignore user.name query parameter in
secure WebHDFS. Contributed by KWON BYUNGCHANG
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit ba6b3a384863b57bc7eeeb736950f544e6ed8d6d
Author: Eric Yang <ey...@apache.org>
AuthorDate: Tue May 28 17:31:35 2019 -0400
HDFS-14434. Ignore user.name query parameter in secure WebHDFS.
Contributed by KWON BYUNGCHANG
(cherry picked from commit d78854b928bb877f26b11b5b212a100a79941f35)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
---
.../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +-
.../hadoop/hdfs/server/common/JspHelper.java | 8 +-
.../hadoop/hdfs/server/common/TestJspHelper.java | 88 +++++----
.../apache/hadoop/hdfs/web/TestWebHdfsTokens.java | 218 +++++++++++++--------
.../org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 47 +++--
5 files changed, 236 insertions(+), 141 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index b316bf1..90b15ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -167,6 +167,7 @@ public class WebHdfsFileSystem extends FileSystem
private InetSocketAddress nnAddrs[];
private int currentNNAddrIndex;
private boolean disallowFallbackToInsecureCluster;
+ private boolean isInsecureCluster;
private String restCsrfCustomHeader;
private Set<String> restCsrfMethodsToIgnore;
@@ -279,6 +280,7 @@ public class WebHdfsFileSystem extends FileSystem
this.workingDir = makeQualified(new Path(getHomeDirectoryString(ugi)));
this.canRefreshDelegationToken = UserGroupInformation.isSecurityEnabled();
+ this.isInsecureCluster = !this.canRefreshDelegationToken;
this.disallowFallbackToInsecureCluster = !conf.getBoolean(
CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
@@ -364,6 +366,7 @@ public class WebHdfsFileSystem extends FileSystem
LOG.debug("Fetched new token: {}", token);
} else { // security is disabled
canRefreshDelegationToken = false;
+ isInsecureCluster = true;
}
}
}
@@ -410,8 +413,7 @@ public class WebHdfsFileSystem extends FileSystem
if (cachedHomeDirectory == null) {
final HttpOpParam.Op op = GetOpParam.Op.GETHOMEDIRECTORY;
try {
- String pathFromDelegatedFS = new FsPathResponseRunner<String>(op, null,
- new UserParam(ugi)) {
+ String pathFromDelegatedFS = new FsPathResponseRunner<String>(op, null){
@Override
String decodeResponse(Map<?, ?> json) throws IOException {
return JsonUtilClient.getPath(json);
@@ -573,7 +575,8 @@ public class WebHdfsFileSystem extends FileSystem
return url;
}
- Param<?,?>[] getAuthParameters(final HttpOpParam.Op op) throws IOException {
+ private synchronized Param<?, ?>[] getAuthParameters(final HttpOpParam.Op op)
+ throws IOException {
List<Param<?,?>> authParams = Lists.newArrayList();
// Skip adding delegation token for token operations because these
// operations require authentication.
@@ -590,7 +593,12 @@ public class WebHdfsFileSystem extends FileSystem
authParams.add(new DoAsParam(userUgi.getShortUserName()));
userUgi = realUgi;
}
- authParams.add(new UserParam(userUgi.getShortUserName()));
+ UserParam userParam = new UserParam((userUgi.getShortUserName()));
+
+ //in insecure, use user.name parameter, in secure, use spnego auth
+ if(isInsecureCluster) {
+ authParams.add(userParam);
+ }
}
return authParams.toArray(new Param<?,?>[0]);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
index eb488e8..2c65c3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
@@ -118,12 +118,9 @@ public class JspHelper {
remoteUser = request.getRemoteUser();
final String tokenString = request.getParameter(DELEGATION_PARAMETER_NAME);
if (tokenString != null) {
- // Token-based connections need only verify the effective user, and
- // disallow proxying to different user. Proxy authorization checks
- // are not required since the checks apply to issuing a token.
+
+ // user.name, doas param is ignored in the token-based auth
ugi = getTokenUGI(context, request, tokenString, conf);
- checkUsername(ugi.getShortUserName(), usernameFromQuery);
- checkUsername(ugi.getShortUserName(), doAsUserFromQuery);
} else if (remoteUser == null) {
throw new IOException(
"Security enabled but user not authenticated by filter");
@@ -137,7 +134,6 @@ public class JspHelper {
if (ugi == null) { // security is off, or there's no token
ugi = UserGroupInformation.createRemoteUser(remoteUser);
- checkUsername(ugi.getShortUserName(), usernameFromQuery);
if (UserGroupInformation.isSecurityEnabled()) {
// This is not necessarily true, could have been auth'ed by user-facing
// filter
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
index 119db8c..5a1661c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
@@ -158,7 +158,7 @@ public class TestJspHelper {
Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
dtId, new DummySecretManager(0, 0, 0, 0));
String tokenString = token.encodeToUrlString();
-
+
// token with no auth-ed user
request = getMockRequest(null, null, null);
when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
@@ -168,7 +168,7 @@ public class TestJspHelper {
Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
Assert.assertEquals(ugi.getShortUserName(), user);
checkUgiFromToken(ugi);
-
+
// token with auth-ed user
request = getMockRequest(realUser, null, null);
when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
@@ -198,32 +198,40 @@ public class TestJspHelper {
Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
Assert.assertEquals(ugi.getShortUserName(), user);
checkUgiFromToken(ugi);
-
- // can't proxy with a token!
+
+ // if present token, ignore doas parameter
request = getMockRequest(null, null, "rogue");
when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
tokenString);
- try {
- JspHelper.getUGI(context, request, conf);
- Assert.fail("bad request allowed");
- } catch (IOException ioe) {
- Assert.assertEquals(
- "Usernames not matched: name=rogue != expected="+user,
- ioe.getMessage());
- }
-
- // can't proxy with a token!
+
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromToken(ugi);
+
+ // if present token, ignore user.name parameter
+ request = getMockRequest(null, "rogue", null);
+ when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
+ tokenString);
+
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromToken(ugi);
+
+ // if present token, ignore user.name and doas parameter
request = getMockRequest(null, user, "rogue");
when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
tokenString);
- try {
- JspHelper.getUGI(context, request, conf);
- Assert.fail("bad request allowed");
- } catch (IOException ioe) {
- Assert.assertEquals(
- "Usernames not matched: name=rogue != expected="+user,
- ioe.getMessage());
- }
+
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromToken(ugi);
+
}
@Test
@@ -271,16 +279,12 @@ public class TestJspHelper {
Assert.assertEquals(ugi.getShortUserName(), realUser);
checkUgiFromAuth(ugi);
- // ugi for remote user != real user
+ // if there is remote user via SPNEGO, ignore user.name param
request = getMockRequest(realUser, user, null);
- try {
- JspHelper.getUGI(context, request, conf);
- Assert.fail("bad request allowed");
- } catch (IOException ioe) {
- Assert.assertEquals(
- "Usernames not matched: name="+user+" != expected="+realUser,
- ioe.getMessage());
- }
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getShortUserName(), realUser);
+ checkUgiFromAuth(ugi);
}
@Test
@@ -335,17 +339,16 @@ public class TestJspHelper {
Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
Assert.assertEquals(ugi.getShortUserName(), user);
checkUgiFromAuth(ugi);
-
- // proxy ugi for user via remote user != real user
+
+ // if there is remote user via SPNEGO, ignore user.name, doas param
request = getMockRequest(realUser, user, user);
- try {
- JspHelper.getUGI(context, request, conf);
- Assert.fail("bad request allowed");
- } catch (IOException ioe) {
- Assert.assertEquals(
- "Usernames not matched: name="+user+" != expected="+realUser,
- ioe.getMessage());
- }
+ ugi = JspHelper.getUGI(context, request, conf);
+ Assert.assertNotNull(ugi.getRealUser());
+ Assert.assertEquals(ugi.getRealUser().getShortUserName(), realUser);
+ Assert.assertEquals(ugi.getShortUserName(), user);
+ checkUgiFromAuth(ugi);
+
+
// try to get get a proxy user with unauthorized user
try {
@@ -368,6 +371,9 @@ public class TestJspHelper {
}
}
+
+
+
private HttpServletRequest getMockRequest(String remoteUser, String user, String doAs) {
HttpServletRequest request = mock(HttpServletRequest.class);
when(request.getParameter(UserParam.NAME)).thenReturn(user);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
index b925e70..50ea62b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
@@ -20,21 +20,37 @@ package org.apache.hadoop.hdfs.web;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.SIMPLE;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.mockito.Matchers.*;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.*;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
-import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.security.PrivilegedExceptionAction;
import java.util.Map;
+import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -47,17 +63,18 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.web.resources.*;
import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.Whitebox;
import org.apache.hadoop.security.token.Token;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -66,16 +83,95 @@ public class TestWebHdfsTokens {
private static Configuration conf;
URI uri = null;
+ //secure cluster
+ private static MiniKdc kdc = null;
+ private static File baseDir;
+ private static File keytabFile;
+ private static String username = "webhdfs-tokens-test";
+ private static String principal;
+ private static String keystoresDir;
+ private static String sslConfDir;
+
@BeforeClass
public static void setUp() {
conf = new Configuration();
+ }
+
+ @AfterClass
+ public static void destroy() throws Exception {
+ if (kdc != null) {
+ kdc.stop();
+ FileUtil.fullyDelete(baseDir);
+ KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
+ }
+ }
+
+ private static void initEnv(){
SecurityUtil.setAuthenticationMethod(KERBEROS, conf);
- UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation.setConfiguration(conf);
UserGroupInformation.setLoginUser(
UserGroupInformation.createUserForTesting(
"LoginUser", new String[]{"supergroup"}));
}
+ private static void initSecureConf(Configuration secureConf)
+ throws Exception {
+
+ baseDir = GenericTestUtils.getTestDir(
+ TestWebHdfsTokens.class.getSimpleName());
+ FileUtil.fullyDelete(baseDir);
+ assertTrue(baseDir.mkdirs());
+
+ Properties kdcConf = MiniKdc.createConf();
+ kdc = new MiniKdc(kdcConf, baseDir);
+ kdc.start();
+
+ SecurityUtil.setAuthenticationMethod(
+ UserGroupInformation.AuthenticationMethod.KERBEROS, secureConf);
+ UserGroupInformation.setConfiguration(secureConf);
+ KerberosName.resetDefaultRealm();
+ assertTrue("Expected secureConfiguration to enable security",
+ UserGroupInformation.isSecurityEnabled());
+
+ keytabFile = new File(baseDir, username + ".keytab");
+ String keytab = keytabFile.getAbsolutePath();
+ // Windows will not reverse name lookup "127.0.0.1" to "localhost".
+ String krbInstance = Path.WINDOWS ? "127.0.0.1" : "localhost";
+ principal = username + "/" + krbInstance + "@" + kdc.getRealm();
+ String spnegoPrincipal = "HTTP/" + krbInstance + "@" + kdc.getRealm();
+ kdc.createPrincipal(keytabFile, username, username + "/" + krbInstance,
+ "HTTP/" + krbInstance);
+
+ secureConf.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, principal);
+ secureConf.set(DFS_NAMENODE_KEYTAB_FILE_KEY, keytab);
+ secureConf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, principal);
+ secureConf.set(DFS_DATANODE_KEYTAB_FILE_KEY, keytab);
+ secureConf.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ spnegoPrincipal);
+ secureConf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
+ secureConf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "authentication");
+ secureConf.set(DFS_HTTP_POLICY_KEY,
+ HttpConfig.Policy.HTTP_AND_HTTPS.name());
+ secureConf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
+ secureConf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY, "localhost:0");
+ secureConf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
+ secureConf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "localhost:0");
+ secureConf.setBoolean(DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+ secureConf.setBoolean(IGNORE_SECURE_PORTS_FOR_TESTING_KEY, true);
+
+ keystoresDir = baseDir.getAbsolutePath();
+ sslConfDir = KeyStoreTestUtil.getClasspathDir(TestWebHdfsTokens.class);
+ KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir,
+ secureConf, false);
+
+ secureConf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY,
+ KeyStoreTestUtil.getClientSSLConfigFileName());
+ secureConf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY,
+ KeyStoreTestUtil.getServerSSLConfigFileName());
+ }
+
+
+
private WebHdfsFileSystem spyWebhdfsInSecureSetup() throws IOException {
WebHdfsFileSystem fsOrig = new WebHdfsFileSystem();
fsOrig.initialize(URI.create("webhdfs://127.0.0.1:0"), conf);
@@ -85,6 +181,7 @@ public class TestWebHdfsTokens {
@Test(timeout = 5000)
public void testTokenForNonTokenOp() throws IOException {
+ initEnv();
WebHdfsFileSystem fs = spyWebhdfsInSecureSetup();
Token<?> token = mock(Token.class);
doReturn(token).when(fs).getDelegationToken(null);
@@ -105,16 +202,19 @@ public class TestWebHdfsTokens {
@Test(timeout = 5000)
public void testNoTokenForGetToken() throws IOException {
+ initEnv();
checkNoTokenForOperation(GetOpParam.Op.GETDELEGATIONTOKEN);
}
@Test(timeout = 5000)
public void testNoTokenForRenewToken() throws IOException {
+ initEnv();
checkNoTokenForOperation(PutOpParam.Op.RENEWDELEGATIONTOKEN);
}
@Test(timeout = 5000)
public void testNoTokenForCancelToken() throws IOException {
+ initEnv();
checkNoTokenForOperation(PutOpParam.Op.CANCELDELEGATIONTOKEN);
}
@@ -163,86 +263,42 @@ public class TestWebHdfsTokens {
@Test
public void testLazyTokenFetchForWebhdfs() throws Exception {
MiniDFSCluster cluster = null;
- WebHdfsFileSystem fs = null;
+ UserGroupInformation ugi = null;
try {
final Configuration clusterConf = new HdfsConfiguration(conf);
- SecurityUtil.setAuthenticationMethod(SIMPLE, clusterConf);
- clusterConf.setBoolean(DFSConfigKeys
- .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+ initSecureConf(clusterConf);
- // trick the NN into thinking security is enabled w/o it trying
- // to login from a keytab
- UserGroupInformation.setConfiguration(clusterConf);
cluster = new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).build();
cluster.waitActive();
- SecurityUtil.setAuthenticationMethod(KERBEROS, clusterConf);
- UserGroupInformation.setConfiguration(clusterConf);
-
+
+ ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(
+ principal, keytabFile.getAbsolutePath());
+
+ //test with swebhdfs
+ uri = DFSUtil.createUri(
+ "swebhdfs", cluster.getNameNode().getHttpsAddress());
+ validateLazyTokenFetch(ugi, clusterConf);
+
+ //test with webhdfs
uri = DFSUtil.createUri(
"webhdfs", cluster.getNameNode().getHttpAddress());
- validateLazyTokenFetch(clusterConf);
+ validateLazyTokenFetch(ugi, clusterConf);
+
} finally {
- IOUtils.cleanup(null, fs);
if (cluster != null) {
cluster.shutdown();
}
+
+ // Reset UGI so that other tests are not affected.
+ UserGroupInformation.reset();
+ UserGroupInformation.setConfiguration(new Configuration());
}
}
-
- @Test
- public void testLazyTokenFetchForSWebhdfs() throws Exception {
- MiniDFSCluster cluster = null;
- SWebHdfsFileSystem fs = null;
- String keystoresDir;
- String sslConfDir;
- try {
- final Configuration clusterConf = new HdfsConfiguration(conf);
- SecurityUtil.setAuthenticationMethod(SIMPLE, clusterConf);
- clusterConf.setBoolean(DFSConfigKeys
- .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
- String baseDir =
- GenericTestUtils.getTempPath(TestWebHdfsTokens.class.getSimpleName());
-
- clusterConf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
- clusterConf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
- clusterConf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
-
- File base = new File(baseDir);
- FileUtil.fullyDelete(base);
- base.mkdirs();
- keystoresDir = new File(baseDir).getAbsolutePath();
- sslConfDir = KeyStoreTestUtil.getClasspathDir(TestWebHdfsTokens.class);
- KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, clusterConf, false);
- clusterConf.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY,
- KeyStoreTestUtil.getClientSSLConfigFileName());
- clusterConf.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY,
- KeyStoreTestUtil.getServerSSLConfigFileName());
-
- // trick the NN into thinking security is enabled w/o it trying
- // to login from a keytab
- UserGroupInformation.setConfiguration(clusterConf);
- cluster = new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).build();
- cluster.waitActive();
- InetSocketAddress addr = cluster.getNameNode().getHttpsAddress();
- String nnAddr = NetUtils.getHostPortString(addr);
- clusterConf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, nnAddr);
- SecurityUtil.setAuthenticationMethod(KERBEROS, clusterConf);
- UserGroupInformation.setConfiguration(clusterConf);
-
- uri = DFSUtil.createUri(
- "swebhdfs", cluster.getNameNode().getHttpsAddress());
- validateLazyTokenFetch(clusterConf);
- } finally {
- IOUtils.cleanup(null, fs);
- if (cluster != null) {
- cluster.shutdown();
- }
- }
- KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
- }
+
@Test
public void testSetTokenServiceAndKind() throws Exception {
+ initEnv();
MiniDFSCluster cluster = null;
try {
@@ -297,16 +353,20 @@ public class TestWebHdfsTokens {
}
}
- private void validateLazyTokenFetch(final Configuration clusterConf) throws Exception{
- final String testUser = "DummyUser";
- UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
- testUser, new String[]{"supergroup"});
- WebHdfsFileSystem fs = ugi.doAs(new PrivilegedExceptionAction<WebHdfsFileSystem>() {
- @Override
- public WebHdfsFileSystem run() throws IOException {
- return spy((WebHdfsFileSystem) FileSystem.newInstance(uri, clusterConf));
- }
- });
+ private void validateLazyTokenFetch(UserGroupInformation ugi,
+ final Configuration clusterConf) throws Exception {
+
+ String testUser = ugi.getShortUserName();
+
+ WebHdfsFileSystem fs = ugi.doAs(
+ new PrivilegedExceptionAction<WebHdfsFileSystem>() {
+ @Override
+ public WebHdfsFileSystem run() throws IOException {
+ return spy((WebHdfsFileSystem) FileSystem.newInstance(uri,
+ clusterConf));
+ }
+ });
+
// verify token ops don't get a token
Assert.assertNull(fs.getRenewToken());
Token<?> token = fs.getDelegationToken(null);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
index 3e5c506..cb62288 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.web;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import java.io.IOException;
@@ -145,37 +146,47 @@ public class TestWebHdfsUrl {
Path fsPath = new Path("/");
String tokenString = webhdfs.getDelegationToken().encodeToUrlString();
+ String userParam = new UserParam(ugi.getShortUserName()).toString();
+
// send user
URL getTokenUrl = webhdfs.toUrl(GetOpParam.Op.GETDELEGATIONTOKEN, fsPath);
+ assertTrue("secure webhdfs SHOULD NOT use user.name parameter",
+ getTokenUrl.toString().indexOf(userParam) == -1);
checkQueryParams(
new String[]{
GetOpParam.Op.GETDELEGATIONTOKEN.toQueryString(),
- new UserParam(ugi.getShortUserName()).toString()
},
getTokenUrl);
+
+
// send user
URL renewTokenUrl = webhdfs.toUrl(PutOpParam.Op.RENEWDELEGATIONTOKEN,
fsPath, new TokenArgumentParam(tokenString));
+ assertTrue("secure webhdfs SHOULD NOT use user.name parameter",
+ renewTokenUrl.toString().indexOf(userParam) == -1);
checkQueryParams(
new String[]{
PutOpParam.Op.RENEWDELEGATIONTOKEN.toQueryString(),
- new UserParam(ugi.getShortUserName()).toString(),
new TokenArgumentParam(tokenString).toString(),
},
renewTokenUrl);
+
+
// send token
URL cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
fsPath, new TokenArgumentParam(tokenString));
+ assertTrue("secure webhdfs SHOULD NOT use user.name parameter",
+ cancelTokenUrl.toString().indexOf(userParam) == -1);
checkQueryParams(
new String[]{
PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(),
- new UserParam(ugi.getShortUserName()).toString(),
new TokenArgumentParam(tokenString).toString(),
},
cancelTokenUrl);
-
+
+
// send token
URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
checkQueryParams(
@@ -191,14 +202,16 @@ public class TestWebHdfsUrl {
// send user
cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
fsPath, new TokenArgumentParam(tokenString));
+ assertTrue("secure webhdfs SHOULD NOT use user.name parameter",
+ cancelTokenUrl.toString().indexOf(userParam) == -1);
checkQueryParams(
new String[]{
PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(),
- new UserParam(ugi.getShortUserName()).toString(),
new TokenArgumentParam(tokenString).toString(),
},
cancelTokenUrl);
+
// send user
fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
checkQueryParams(
@@ -226,40 +239,50 @@ public class TestWebHdfsUrl {
Path fsPath = new Path("/");
String tokenString = webhdfs.getDelegationToken().encodeToUrlString();
+ String userParam = new UserParam(ugi.getRealUser().
+ getShortUserName()).toString();
+
// send real+effective
URL getTokenUrl = webhdfs.toUrl(GetOpParam.Op.GETDELEGATIONTOKEN, fsPath);
+ assertTrue("secure webhdfs SHOULD NOT use user.name parameter",
+ getTokenUrl.toString().indexOf(userParam) == -1);
checkQueryParams(
new String[]{
GetOpParam.Op.GETDELEGATIONTOKEN.toQueryString(),
- new UserParam(ugi.getRealUser().getShortUserName()).toString(),
new DoAsParam(ugi.getShortUserName()).toString()
},
getTokenUrl);
+
+
// send real+effective
URL renewTokenUrl = webhdfs.toUrl(PutOpParam.Op.RENEWDELEGATIONTOKEN,
fsPath, new TokenArgumentParam(tokenString));
+ assertTrue("secure webhdfs SHOULD NOT use user.name parameter",
+ renewTokenUrl.toString().indexOf(userParam) == -1);
checkQueryParams(
new String[]{
PutOpParam.Op.RENEWDELEGATIONTOKEN.toQueryString(),
- new UserParam(ugi.getRealUser().getShortUserName()).toString(),
new DoAsParam(ugi.getShortUserName()).toString(),
new TokenArgumentParam(tokenString).toString(),
},
renewTokenUrl);
+
// send token
URL cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
fsPath, new TokenArgumentParam(tokenString));
+ assertTrue("secure webhdfs SHOULD NOT use user.name parameter",
+ cancelTokenUrl.toString().indexOf(userParam) == -1);
checkQueryParams(
new String[]{
PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(),
- new UserParam(ugi.getRealUser().getShortUserName()).toString(),
new DoAsParam(ugi.getShortUserName()).toString(),
new TokenArgumentParam(tokenString).toString(),
},
cancelTokenUrl);
-
+
+
// send token
URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
checkQueryParams(
@@ -275,15 +298,17 @@ public class TestWebHdfsUrl {
// send real+effective
cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
fsPath, new TokenArgumentParam(tokenString));
+ assertTrue("secure webhdfs SHOULD NOT use user.name parameter",
+ cancelTokenUrl.toString().indexOf(userParam) == -1);
checkQueryParams(
new String[]{
PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(),
- new UserParam(ugi.getRealUser().getShortUserName()).toString(),
new DoAsParam(ugi.getShortUserName()).toString(),
new TokenArgumentParam(tokenString).toString()
},
cancelTokenUrl);
-
+
+
// send real+effective
fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
checkQueryParams(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: HDFS-14647. NPE during secure namenode startup.
Contributed by Fengnan Li.
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 6cd9290401735a2c33a0ff0ae7324876ef9615e9
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Thu Jul 25 06:51:07 2019 +0530
HDFS-14647. NPE during secure namenode startup. Contributed by Fengnan Li.
(cherry picked from commit 62deab17a33cef723d73f8d8b9e37e5bddbc1813)
---
.../hadoop/hdfs/server/namenode/NameNode.java | 4 +++
.../hadoop/hdfs/server/common/TestJspHelper.java | 34 ++++++++++++++++++++++
2 files changed, 38 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index ba4b730..0fff970 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -679,6 +679,10 @@ public class NameNode extends ReconfigurableBase implements
@Override
public void verifyToken(DelegationTokenIdentifier id, byte[] password)
throws IOException {
+ // during startup namesystem is null, let client retry
+ if (namesystem == null) {
+ throw new RetriableException("Namenode is in startup mode");
+ }
namesystem.verifyToken(id, password);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
index 5a1661c..1aff766 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
@@ -21,12 +21,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authorize.AuthorizationException;
@@ -36,9 +38,11 @@ import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.mockito.Mockito;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
@@ -371,8 +375,38 @@ public class TestJspHelper {
}
}
+ @Test
+ public void testGetUgiDuringStartup() throws Exception {
+ conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/");
+ ServletContext context = mock(ServletContext.class);
+ String realUser = "TheDoctor";
+ String user = "TheNurse";
+ conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ HttpServletRequest request;
+ Text ownerText = new Text(user);
+ DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(
+ ownerText, ownerText, new Text(realUser));
+ Token<DelegationTokenIdentifier> token =
+ new Token<DelegationTokenIdentifier>(dtId,
+ new DummySecretManager(0, 0, 0, 0));
+ String tokenString = token.encodeToUrlString();
+ // token with auth-ed user
+ request = getMockRequest(realUser, null, null);
+ when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(
+ tokenString);
+
+ NameNode mockNN = mock(NameNode.class);
+ Mockito.doCallRealMethod().when(mockNN)
+ .verifyToken(Mockito.any(), Mockito.any());
+ when(context.getAttribute("name.node")).thenReturn(mockNN);
+
+ LambdaTestUtils.intercept(RetriableException.class,
+ "Namenode is in startup mode",
+ () -> JspHelper.getUGI(context, request, conf));
+ }
private HttpServletRequest getMockRequest(String remoteUser, String user, String doAs) {
HttpServletRequest request = mock(HttpServletRequest.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org