You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2014/09/03 00:58:42 UTC
[09/50] [abbrv] git commit: HADOOP-10880. Move HTTP delegation tokens
out of URL querystring to a header. (tucu)
HADOOP-10880. Move HTTP delegation tokens out of URL querystring to a header. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1ae479a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1ae479a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1ae479a
Branch: refs/heads/MR-2841
Commit: d1ae479aa5ae4d3e7ec80e35892e1699c378f813
Parents: c4c9a78
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Thu Aug 28 14:45:40 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Thu Aug 28 14:45:40 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../web/DelegationTokenAuthenticatedURL.java | 81 ++++++++++++++++----
.../DelegationTokenAuthenticationHandler.java | 14 +++-
.../web/DelegationTokenAuthenticator.java | 19 ++++-
...tionTokenAuthenticationHandlerWithMocks.java | 46 ++++++++++-
.../delegation/web/TestWebDelegationToken.java | 50 +++++++++++-
6 files changed, 187 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ecbaaab..641635b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -518,6 +518,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10998. Fix bash tab completion code to work (Jim Hester via aw)
+ HADOOP-10880. Move HTTP delegation tokens out of URL querystring to
+ a header. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
index d955ada..5aeb177 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
@@ -125,6 +125,8 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
}
}
+ private boolean useQueryStringforDelegationToken = false;
+
/**
* Creates an <code>DelegationTokenAuthenticatedURL</code>.
* <p/>
@@ -171,6 +173,34 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
}
/**
+ * Sets if delegation token should be transmitted in the URL query string.
+ * By default it is transmitted using the
+ * {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ * <p/>
+ * This method is provided to enable WebHDFS backwards compatibility.
+ *
+ * @param useQueryString <code>TRUE</code> if the token is transmitted in the
+ * URL query string, <code>FALSE</code> if the delegation token is transmitted
+ * using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP
+ * header.
+ */
+ @Deprecated
+ protected void setUseQueryStringForDelegationToken(boolean useQueryString) {
+ useQueryStringforDelegationToken = useQueryString;
+ }
+
+ /**
+ * Returns if delegation token is transmitted as a HTTP header.
+ *
+ * @return <code>TRUE</code> if the token is transmitted in the URL query
+ * string, <code>FALSE</code> if the delegation token is transmitted using the
+ * {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ */
+ public boolean useQueryStringForDelegationToken() {
+ return useQueryStringforDelegationToken;
+ }
+
+ /**
* Returns an authenticated {@link HttpURLConnection}, it uses a Delegation
* Token only if the given auth token is an instance of {@link Token} and
* it contains a Delegation Token, otherwise use the configured
@@ -235,23 +265,41 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
* @throws IOException if an IO error occurred.
* @throws AuthenticationException if an authentication exception occurred.
*/
+ @SuppressWarnings("unchecked")
public HttpURLConnection openConnection(URL url, Token token, String doAs)
throws IOException, AuthenticationException {
Preconditions.checkNotNull(url, "url");
Preconditions.checkNotNull(token, "token");
Map<String, String> extraParams = new HashMap<String, String>();
-
- // delegation token
- Credentials creds = UserGroupInformation.getCurrentUser().getCredentials();
- if (!creds.getAllTokens().isEmpty()) {
- InetSocketAddress serviceAddr = new InetSocketAddress(url.getHost(),
- url.getPort());
- Text service = SecurityUtil.buildTokenService(serviceAddr);
- org.apache.hadoop.security.token.Token<? extends TokenIdentifier> dt =
- creds.getToken(service);
- if (dt != null) {
- extraParams.put(KerberosDelegationTokenAuthenticator.DELEGATION_PARAM,
- dt.encodeToUrlString());
+ org.apache.hadoop.security.token.Token<? extends TokenIdentifier> dToken
+ = null;
+ // if we have valid auth token, it takes precedence over a delegation token
+ // and we don't even look for one.
+ if (!token.isSet()) {
+ // delegation token
+ Credentials creds = UserGroupInformation.getCurrentUser().
+ getCredentials();
+ if (!creds.getAllTokens().isEmpty()) {
+ InetSocketAddress serviceAddr = new InetSocketAddress(url.getHost(),
+ url.getPort());
+ Text service = SecurityUtil.buildTokenService(serviceAddr);
+ dToken = creds.getToken(service);
+ if (dToken != null) {
+ if (useQueryStringForDelegationToken()) {
+ // delegation token will go in the query string, injecting it
+ extraParams.put(
+ KerberosDelegationTokenAuthenticator.DELEGATION_PARAM,
+ dToken.encodeToUrlString());
+ } else {
+ // delegation token will go as request header, setting it in the
+ // auth-token to ensure no authentication handshake is triggered
+ // (if we have a delegation token, we are authenticated)
+ // the delegation token header is injected in the connection request
+ // at the end of this method.
+ token.delegationToken = (org.apache.hadoop.security.token.Token
+ <AbstractDelegationTokenIdentifier>) dToken;
+ }
+ }
}
}
@@ -261,7 +309,14 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
}
url = augmentURL(url, extraParams);
- return super.openConnection(url, token);
+ HttpURLConnection conn = super.openConnection(url, token);
+ if (!token.isSet() && !useQueryStringForDelegationToken() && dToken != null) {
+ // injecting the delegation token header in the connection request
+ conn.setRequestProperty(
+ DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER,
+ dToken.encodeToUrlString());
+ }
+ return conn;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index 670ec55..e4d9424 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -331,8 +331,7 @@ public abstract class DelegationTokenAuthenticationHandler
HttpServletResponse response)
throws IOException, AuthenticationException {
AuthenticationToken token;
- String delegationParam = ServletUtils.getParameter(request,
- KerberosDelegationTokenAuthenticator.DELEGATION_PARAM);
+ String delegationParam = getDelegationToken(request);
if (delegationParam != null) {
try {
Token<DelegationTokenIdentifier> dt =
@@ -356,4 +355,15 @@ public abstract class DelegationTokenAuthenticationHandler
return token;
}
+ private String getDelegationToken(HttpServletRequest request)
+ throws IOException {
+ String dToken = request.getHeader(
+ DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER);
+ if (dToken == null) {
+ dToken = ServletUtils.getParameter(request,
+ KerberosDelegationTokenAuthenticator.DELEGATION_PARAM);
+ }
+ return dToken;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index ec192da..18df56c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -56,6 +56,9 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
public static final String OP_PARAM = "op";
+ public static final String DELEGATION_TOKEN_HEADER =
+ "X-Hadoop-Delegation-Token";
+
public static final String DELEGATION_PARAM = "delegation";
public static final String TOKEN_PARAM = "token";
public static final String RENEWER_PARAM = "renewer";
@@ -101,15 +104,23 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
authenticator.setConnectionConfigurator(configurator);
}
- private boolean hasDelegationToken(URL url) {
- String queryStr = url.getQuery();
- return (queryStr != null) && queryStr.contains(DELEGATION_PARAM + "=");
+ private boolean hasDelegationToken(URL url, AuthenticatedURL.Token token) {
+ boolean hasDt = false;
+ if (token instanceof DelegationTokenAuthenticatedURL.Token) {
+ hasDt = ((DelegationTokenAuthenticatedURL.Token) token).
+ getDelegationToken() != null;
+ }
+ if (!hasDt) {
+ String queryStr = url.getQuery();
+ hasDt = (queryStr != null) && queryStr.contains(DELEGATION_PARAM + "=");
+ }
+ return hasDt;
}
@Override
public void authenticate(URL url, AuthenticatedURL.Token token)
throws IOException, AuthenticationException {
- if (!hasDelegationToken(url)) {
+ if (!hasDelegationToken(url, token)) {
authenticator.authenticate(url, token);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
index c9d255d..7880fa1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
@@ -284,11 +284,13 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
@Test
public void testAuthenticate() throws Exception {
- testValidDelegationToken();
- testInvalidDelegationToken();
+ testValidDelegationTokenQueryString();
+ testValidDelegationTokenHeader();
+ testInvalidDelegationTokenQueryString();
+ testInvalidDelegationTokenHeader();
}
- private void testValidDelegationToken() throws Exception {
+ private void testValidDelegationTokenQueryString() throws Exception {
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
Token<DelegationTokenIdentifier> dToken =
@@ -307,7 +309,26 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
Assert.assertTrue(token.isExpired());
}
- private void testInvalidDelegationToken() throws Exception {
+ private void testValidDelegationTokenHeader() throws Exception {
+ HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+ Token<DelegationTokenIdentifier> dToken =
+ handler.getTokenManager().createToken(
+ UserGroupInformation.getCurrentUser(), "user");
+ Mockito.when(request.getHeader(Mockito.eq(
+ DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER))).thenReturn(
+ dToken.encodeToUrlString());
+
+ AuthenticationToken token = handler.authenticate(request, response);
+ Assert.assertEquals(UserGroupInformation.getCurrentUser().
+ getShortUserName(), token.getUserName());
+ Assert.assertEquals(0, token.getExpires());
+ Assert.assertEquals(handler.getType(),
+ token.getType());
+ Assert.assertTrue(token.isExpired());
+ }
+
+ private void testInvalidDelegationTokenQueryString() throws Exception {
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
Mockito.when(request.getQueryString()).thenReturn(
@@ -323,4 +344,21 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
}
}
+ private void testInvalidDelegationTokenHeader() throws Exception {
+ HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+ Mockito.when(request.getHeader(Mockito.eq(
+ DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER))).thenReturn(
+ "invalid");
+
+ try {
+ handler.authenticate(request, response);
+ Assert.fail();
+ } catch (AuthenticationException ex) {
+ //NOP
+ } catch (Exception ex) {
+ Assert.fail();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1ae479a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
index 1b452f1..118abff 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java
@@ -149,6 +149,15 @@ public class TestWebDelegationToken {
throws ServletException, IOException {
resp.setStatus(HttpServletResponse.SC_OK);
resp.getWriter().write("ping");
+ if (req.getHeader(DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER)
+ != null) {
+ resp.setHeader("UsingHeader", "true");
+ }
+ if (req.getQueryString() != null &&
+ req.getQueryString().contains(
+ DelegationTokenAuthenticator.DELEGATION_PARAM + "=")) {
+ resp.setHeader("UsingQueryString", "true");
+ }
}
@Override
@@ -314,7 +323,20 @@ public class TestWebDelegationToken {
}
@Test
- public void testDelegationTokenAuthenticatorCalls() throws Exception {
+ public void testDelegationTokenAuthenticatorCallsWithHeader()
+ throws Exception {
+ testDelegationTokenAuthenticatorCalls(false);
+ }
+
+ @Test
+ public void testDelegationTokenAuthenticatorCallsWithQueryString()
+ throws Exception {
+ testDelegationTokenAuthenticatorCalls(true);
+ }
+
+
+ private void testDelegationTokenAuthenticatorCalls(final boolean useQS)
+ throws Exception {
final Server jetty = createJettyServer();
Context context = new Context();
context.setContextPath("/foo");
@@ -324,14 +346,15 @@ public class TestWebDelegationToken {
try {
jetty.start();
- URL nonAuthURL = new URL(getJettyURL() + "/foo/bar");
+ final URL nonAuthURL = new URL(getJettyURL() + "/foo/bar");
URL authURL = new URL(getJettyURL() + "/foo/bar?authenticated=foo");
URL authURL2 = new URL(getJettyURL() + "/foo/bar?authenticated=bar");
DelegationTokenAuthenticatedURL.Token token =
new DelegationTokenAuthenticatedURL.Token();
- DelegationTokenAuthenticatedURL aUrl =
+ final DelegationTokenAuthenticatedURL aUrl =
new DelegationTokenAuthenticatedURL();
+ aUrl.setUseQueryStringForDelegationToken(useQS);
try {
aUrl.getDelegationToken(nonAuthURL, token, FOO_USER);
@@ -379,6 +402,27 @@ public class TestWebDelegationToken {
Assert.assertTrue(ex.getMessage().contains("401"));
}
+ aUrl.getDelegationToken(authURL, token, "foo");
+
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ ugi.addToken(token.getDelegationToken());
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ HttpURLConnection conn = aUrl.openConnection(nonAuthURL, new DelegationTokenAuthenticatedURL.Token());
+ Assert.assertEquals(HttpServletResponse.SC_OK, conn.getResponseCode());
+ if (useQS) {
+ Assert.assertNull(conn.getHeaderField("UsingHeader"));
+ Assert.assertNotNull(conn.getHeaderField("UsingQueryString"));
+ } else {
+ Assert.assertNotNull(conn.getHeaderField("UsingHeader"));
+ Assert.assertNull(conn.getHeaderField("UsingQueryString"));
+ }
+ return null;
+ }
+ });
+
+
} finally {
jetty.stop();
}