You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2011/10/25 19:57:06 UTC
svn commit: r1188836 - in /hadoop/common/branches/branch-0.20-security: ./
src/core/org/apache/hadoop/security/token/delegation/
src/hdfs/org/apache/hadoop/hdfs/
src/hdfs/org/apache/hadoop/hdfs/server/namenode/
src/hdfs/org/apache/hadoop/hdfs/server/na...
Author: jitendra
Date: Tue Oct 25 17:57:05 2011
New Revision: 1188836
URL: http://svn.apache.org/viewvc?rev=1188836&view=rev
Log:
HDFS-2416. Distcp with a webhdfs uri on a secure cluster fails.
Added:
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/TokenArgumentParam.java
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
Modified:
hadoop/common/branches/branch-0.20-security/CHANGES.txt
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/ByteRangeInputStream.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/JspHelper.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/AuthFilter.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/UserProvider.java
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
hadoop/common/branches/branch-0.20-security/src/webapps/hdfs/nn_browsedfscontent.jsp
Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Tue Oct 25 17:57:05 2011
@@ -74,6 +74,8 @@ Release 0.20.205.1 - unreleased
getDefaultBlockSize() and getDefaultReplication() in WebHdfsFileSystem
and cleared content type in ExceptionHandler. (szetszwo)
+ HDFS-2416. Distcp with a webhdfs uri on a secure cluster fails. (jitendra)
+
Release 0.20.205.0 - 2011.10.06
NEW FEATURES
Modified: hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java Tue Oct 25 17:57:05 2011
@@ -217,6 +217,21 @@ extends AbstractDelegationTokenIdentifie
}
/**
+ * Verifies that the given identifier and password are valid and match.
+ * @param identifier Token identifier.
+ * @param password Password in the token.
+ * @throws InvalidToken
+ */
+ public synchronized void verifyToken(TokenIdent identifier, byte[] password)
+ throws InvalidToken {
+ byte[] storedPassword = retrievePassword(identifier);
+ if (!Arrays.equals(password, storedPassword)) {
+ throw new InvalidToken("token (" + identifier
+ + ") is invalid, password doesn't match");
+ }
+ }
+
+ /**
* Renew a delegation token.
* @param token the token to renew
* @param renewer the full principal name of the user doing the renewal
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/ByteRangeInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/ByteRangeInputStream.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/ByteRangeInputStream.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/ByteRangeInputStream.java Tue Oct 25 17:57:05 2011
@@ -40,13 +40,13 @@ import org.apache.hadoop.hdfs.web.resour
public class ByteRangeInputStream extends FSInputStream {
/**
- * This class wraps a URL to allow easy mocking when testing. The URL class
- * cannot be easily mocked because it is public.
+ * This class wraps a URL and provides method to open connection.
+ * It can be overridden to change how a connection is opened.
*/
- static class URLOpener {
+ public static class URLOpener {
protected URL url;
/** The url with offset parameter */
- private URL offsetUrl;
+ protected URL offsetUrl;
public URLOpener(URL u) {
url = u;
@@ -60,7 +60,7 @@ public class ByteRangeInputStream extend
return url;
}
- HttpURLConnection openConnection() throws IOException {
+ protected HttpURLConnection openConnection() throws IOException {
return (HttpURLConnection)offsetUrl.openConnection();
}
@@ -125,7 +125,13 @@ public class ByteRangeInputStream extend
this(new URLOpener(url), new URLOpener(null));
}
- ByteRangeInputStream(URLOpener o, URLOpener r) {
+ /**
+ * Create with the specified URLOpeners. Original url is used to open the
+ * stream for the first time. Resolved url is used in subsequent requests.
+ * @param o Original url
+ * @param r Resolved url
+ */
+ public ByteRangeInputStream(URLOpener o, URLOpener r) {
this.originalURL = o;
this.resolvedURL = r;
}
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java Tue Oct 25 17:57:05 2011
@@ -71,7 +71,7 @@ abstract class DfsServlet extends HttpSe
protected UserGroupInformation getUGI(HttpServletRequest request,
Configuration conf
) throws IOException {
- return JspHelper.getUGI(request, conf);
+ return JspHelper.getUGI(getServletContext(), request, conf);
}
/**
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/JspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/JspHelper.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/JspHelper.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/JspHelper.java Tue Oct 25 17:57:05 2011
@@ -31,6 +31,7 @@ import java.util.Comparator;
import java.util.Random;
import java.util.TreeSet;
+import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.jsp.JspWriter;
@@ -427,15 +428,30 @@ public class JspHelper {
return UserGroupInformation.createRemoteUser(strings[0]);
}
- /** Same as getUGI(request, conf, KERBEROS_SSL, true). */
+ /** Same as getUGI(null, request, conf). */
public static UserGroupInformation getUGI(final HttpServletRequest request,
final Configuration conf) throws IOException {
- return getUGI(request, conf, AuthenticationMethod.KERBEROS_SSL, true);
+ return getUGI(null, request, conf);
+ }
+
+ /**
+ * Get {@link UserGroupInformation} and possibly the delegation token out of
+ * the request.
+ * @param context the Servlet context
+ * @param request the http request
+ * @param conf configuration
+ * @throws AccessControlException if the request has no token
+ */
+ public static UserGroupInformation getUGI(ServletContext context,
+ HttpServletRequest request, Configuration conf) throws IOException {
+ return getUGI(context, request, conf, AuthenticationMethod.KERBEROS_SSL,
+ true);
}
/**
* Get {@link UserGroupInformation} and possibly the delegation token out of
* the request.
+ * @param context the Servlet context
* @param request the http request
* @param conf configuration
* @param secureAuthMethod the AuthenticationMethod used in secure mode.
@@ -443,8 +459,8 @@ public class JspHelper {
* @return a new user from the request
* @throws AccessControlException if the request has no token
*/
- public static UserGroupInformation getUGI(HttpServletRequest request,
- Configuration conf,
+ public static UserGroupInformation getUGI(ServletContext context,
+ HttpServletRequest request, Configuration conf,
final AuthenticationMethod secureAuthMethod,
final boolean tryUgiParameter) throws IOException {
final UserGroupInformation ugi;
@@ -464,6 +480,14 @@ public class JspHelper {
DataInputStream in = new DataInputStream(buf);
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.readFields(in);
+ if (context != null) {
+ NameNode nn = (NameNode) context.getAttribute("name.node");
+ if (nn != null) {
+ //Verify the token.
+ nn.getNamesystem().getDelegationTokenSecretManager()
+ .verifyToken(id, token.getPassword());
+ }
+ }
ugi = id.getUser();
checkUsername(ugi.getShortUserName(), usernameFromQuery);
checkUsername(ugi.getShortUserName(), user);
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Tue Oct 25 17:57:05 2011
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
import org.apache.hadoop.hdfs.web.resources.DelegationParam;
+import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
import org.apache.hadoop.hdfs.web.resources.DestinationParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
@@ -211,7 +212,9 @@ public class NamenodeWebHdfsMethods {
@QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
final ModificationTimeParam modificationTime,
@QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
- final AccessTimeParam accessTime
+ final AccessTimeParam accessTime,
+ @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT)
+ final TokenArgumentParam delegationTokenArgument
) throws IOException, InterruptedException {
if (LOG.isTraceEnabled()) {
@@ -278,7 +281,7 @@ public class NamenodeWebHdfsMethods {
case RENEWDELEGATIONTOKEN:
{
final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
- token.decodeFromUrlString(delegation.getValue());
+ token.decodeFromUrlString(delegationTokenArgument.getValue());
final long expiryTime = namenode.renewDelegationToken(token);
final String js = JsonUtil.toJsonString("long", expiryTime);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
@@ -286,7 +289,7 @@ public class NamenodeWebHdfsMethods {
case CANCELDELEGATIONTOKEN:
{
final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
- token.decodeFromUrlString(delegation.getValue());
+ token.decodeFromUrlString(delegationTokenArgument.getValue());
namenode.cancelDelegationToken(token);
return Response.ok().type(MediaType.APPLICATION_JSON).build();
}
@@ -353,6 +356,45 @@ public class NamenodeWebHdfsMethods {
}
private static final UriFsPathParam ROOT = new UriFsPathParam("");
+
+ /** Handle HTTP PUT request for root. */
+ @PUT
+ @Path("/")
+ @Consumes({"*/*"})
+ @Produces({MediaType.APPLICATION_JSON})
+ public Response putRoot(
+ @Context final UserGroupInformation ugi,
+ @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
+ final DelegationParam delegation,
+ @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
+ final PutOpParam op,
+ @QueryParam(DestinationParam.NAME) @DefaultValue(DestinationParam.DEFAULT)
+ final DestinationParam destination,
+ @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT)
+ final OwnerParam owner,
+ @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT)
+ final GroupParam group,
+ @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
+ final PermissionParam permission,
+ @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
+ final OverwriteParam overwrite,
+ @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
+ final BufferSizeParam bufferSize,
+ @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
+ final ReplicationParam replication,
+ @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
+ final BlockSizeParam blockSize,
+ @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
+ final ModificationTimeParam modificationTime,
+ @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
+ final AccessTimeParam accessTime,
+ @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT)
+ final TokenArgumentParam delegationTokenArgument
+ ) throws IOException, InterruptedException {
+ return put(ugi, delegation, ROOT, op, destination, owner, group,
+ permission, overwrite, bufferSize, replication, blockSize,
+ modificationTime, accessTime, delegationTokenArgument);
+ }
/** Handle HTTP GET request for the root. */
@GET
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/AuthFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/AuthFilter.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/AuthFilter.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/AuthFilter.java Tue Oct 25 17:57:05 2011
@@ -17,11 +17,17 @@
*/
package org.apache.hadoop.hdfs.web;
+import java.io.IOException;
import java.util.Properties;
+import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import org.apache.hadoop.hdfs.web.resources.DelegationParam;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
@@ -55,6 +61,21 @@ public class AuthFilter extends Authenti
p.setProperty(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, "true");
//set cookie path
p.setProperty(COOKIE_PATH, "/");
- return p;
+ return p;
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response,
+ FilterChain filterChain) throws IOException, ServletException {
+ HttpServletRequest httpRequest = (HttpServletRequest) request;
+ String tokenString = httpRequest
+ .getParameter(DelegationParam.NAME);
+ if (tokenString != null) {
+ //Token is present in the url, therefore token will be used for
+ //authentication, bypass kerberos authentication.
+ filterChain.doFilter(httpRequest, response);
+ return;
+ }
+ super.doFilter(request, response, filterChain);
}
}
\ No newline at end of file
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Tue Oct 25 17:57:05 2011
@@ -58,6 +58,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
+import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
import org.apache.hadoop.hdfs.web.resources.DestinationParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
@@ -255,7 +256,7 @@ public class WebHdfsFileSystem extends F
}
return url;
}
-
+
private String addDt2Query(String query) throws IOException {
if (UserGroupInformation.isSecurityEnabled()) {
synchronized (this) {
@@ -276,24 +277,42 @@ public class WebHdfsFileSystem extends F
final String query = op.toQueryString()
+ '&' + new UserParam(ugi)
+ Param.toSortedString("&", parameters);
- final URL url = getNamenodeURL(path, addDt2Query(query));
+ final URL url;
+ if (op.equals(PutOpParam.Op.RENEWDELEGATIONTOKEN)
+ || op.equals(GetOpParam.Op.GETDELEGATIONTOKEN)) {
+ // Skip adding delegation token for getting or renewing delegation token,
+ // because these operations require kerberos authentication.
+ url = getNamenodeURL(path, query);
+ } else {
+ url = getNamenodeURL(path, addDt2Query(query));
+ }
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
}
return url;
}
+ private HttpURLConnection getHttpUrlConnection(URL url)
+ throws IOException {
+ final HttpURLConnection conn;
+ try {
+ if (ugi.hasKerberosCredentials()) {
+ conn = new AuthenticatedURL(AUTH).openConnection(url, authToken);
+ } else {
+ conn = (HttpURLConnection)url.openConnection();
+ }
+ } catch (AuthenticationException e) {
+ throw new IOException("Authentication failed, url=" + url, e);
+ }
+ return conn;
+ }
+
private HttpURLConnection httpConnect(final HttpOpParam.Op op, final Path fspath,
final Param<?,?>... parameters) throws IOException {
final URL url = toUrl(op, fspath, parameters);
//connect and get response
- final HttpURLConnection conn;
- try {
- conn = new AuthenticatedURL(AUTH).openConnection(url, authToken);
- } catch(AuthenticationException e) {
- throw new IOException("Authentication failed, url=" + url, e);
- }
+ final HttpURLConnection conn = getHttpUrlConnection(url);
try {
conn.setRequestMethod(op.getType().toString());
conn.setDoOutput(op.getDoOutput());
@@ -303,7 +322,7 @@ public class WebHdfsFileSystem extends F
}
conn.connect();
return conn;
- } catch(IOException e) {
+ } catch (IOException e) {
conn.disconnect();
throw e;
}
@@ -488,7 +507,24 @@ public class WebHdfsFileSystem extends F
statistics.incrementReadOps(1);
final HttpOpParam.Op op = GetOpParam.Op.OPEN;
final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
- return new FSDataInputStream(new ByteRangeInputStream(url));
+ ByteRangeInputStream str = getByteRangeInputStream(url);
+ return new FSDataInputStream(str);
+ }
+
+ private class URLOpener extends ByteRangeInputStream.URLOpener {
+
+ public URLOpener(URL u) {
+ super(u);
+ }
+
+ @Override
+ public HttpURLConnection openConnection() throws IOException {
+ return getHttpUrlConnection(offsetUrl);
+ }
+ }
+
+ private ByteRangeInputStream getByteRangeInputStream(URL url) {
+ return new ByteRangeInputStream(new URLOpener(url), new URLOpener(null));
}
@Override
@@ -542,17 +578,19 @@ public class WebHdfsFileSystem extends F
private synchronized long renewDelegationToken(final Token<?> token
) throws IOException {
- delegationToken = token;
final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
- final Map<String, Object> m = run(op, null);
- return (Long)m.get("long");
+ TokenArgumentParam dtargParam = new TokenArgumentParam(
+ token.encodeToUrlString());
+ final Map<String, Object> m = run(op, null, dtargParam);
+ return (Long) m.get("long");
}
private synchronized void cancelDelegationToken(final Token<?> token
) throws IOException {
- delegationToken = token;
final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
- run(op, null);
+ TokenArgumentParam dtargParam = new TokenArgumentParam(
+ token.encodeToUrlString());
+ run(op, null, dtargParam);
}
@Override
@@ -627,13 +665,12 @@ public class WebHdfsFileSystem extends F
// update the kerberos credentials, if they are coming from a keytab
ugi.checkTGTAndReloginFromKeytab();
- return ugi.doAs(new PrivilegedExceptionAction<Long>() {
- @Override
- public Long run() throws Exception {
- final WebHdfsFileSystem webhdfs = getWebHdfs(token, conf);
- return webhdfs.renewDelegationToken(token);
- }
- });
+ try {
+ WebHdfsFileSystem webhdfs = getWebHdfs(token, conf);
+ return webhdfs.renewDelegationToken(token);
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
}
@Override
@@ -643,14 +680,12 @@ public class WebHdfsFileSystem extends F
// update the kerberos credentials, if they are coming from a keytab
ugi.checkTGTAndReloginFromKeytab();
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- final WebHdfsFileSystem webhdfs = getWebHdfs(token, conf);
- webhdfs.cancelDelegationToken(token);
- return null;
- }
- });
+ try {
+ final WebHdfsFileSystem webhdfs = getWebHdfs(token, conf);
+ webhdfs.cancelDelegationToken(token);
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
}
}
}
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/DelegationParam.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/DelegationParam.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/DelegationParam.java Tue Oct 25 17:57:05 2011
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.web.resou
import org.apache.hadoop.security.UserGroupInformation;
-/** Delegation token parameter. */
+/** Represents delegation token used for authentication. */
public class DelegationParam extends StringParam {
/** Parameter name. */
public static final String NAME = "delegation";
Added: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/TokenArgumentParam.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/TokenArgumentParam.java?rev=1188836&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/TokenArgumentParam.java (added)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/TokenArgumentParam.java Tue Oct 25 17:57:05 2011
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/**
+ * Represents delegation token parameter as method arguments. This is
+ * different from {@link DelegationParam}.
+ */
+public class TokenArgumentParam extends StringParam {
+ /** Parameter name. */
+ public static final String NAME = "token";
+ /** Default parameter value. */
+ public static final String DEFAULT = "";
+
+ private static final Domain DOMAIN = new Domain(NAME, null);
+
+ /**
+ * Constructor.
+ * @param str A string representation of the parameter value.
+ */
+ public TokenArgumentParam(final String str) {
+ super(DOMAIN, str != null && !str.equals(DEFAULT) ? str : null);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+}
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/UserProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/UserProvider.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/UserProvider.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/web/resources/UserProvider.java Tue Oct 25 17:57:05 2011
@@ -50,7 +50,7 @@ public class UserProvider
final Configuration conf = (Configuration) servletcontext
.getAttribute(JspHelper.CURRENT_CONF);
try {
- return JspHelper.getUGI(request, conf,
+ return JspHelper.getUGI(servletcontext, request, conf,
AuthenticationMethod.KERBEROS, false);
} catch (IOException e) {
throw new RuntimeException(e);
Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java Tue Oct 25 17:57:05 2011
@@ -154,7 +154,7 @@ public class TestDelegationToken {
DelegationTokenSecretManager dtSecretManager = cluster.getNameNode()
.getNamesystem().getDelegationTokenSecretManager();
DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
- Token<DelegationTokenIdentifier> token = dfs.getDelegationToken(new Text("JobTracker"));
+ final Token<DelegationTokenIdentifier> token = dfs.getDelegationToken(new Text("JobTracker"));
DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
byte[] tokenId = token.getIdentifier();
identifier.readFields(new DataInputStream(
@@ -162,6 +162,15 @@ public class TestDelegationToken {
LOG.info("A valid token should have non-null password, and should be renewed successfully");
Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier));
dtSecretManager.renewToken(token, "JobTracker");
+ UserGroupInformation.createRemoteUser("JobTracker").doAs(
+ new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ token.renew(config);
+ token.cancel(config);
+ return null;
+ }
+ });
}
@Test
@@ -182,13 +191,23 @@ public class TestDelegationToken {
}
});
- final Token<DelegationTokenIdentifier> token = webhdfs.getDelegationToken("JobTracker");
+ final Token<DelegationTokenIdentifier> token = webhdfs
+ .getDelegationToken("JobTracker");
DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
byte[] tokenId = token.getIdentifier();
- identifier.readFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
+ identifier
+ .readFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
LOG.info("A valid token should have non-null password, and should be renewed successfully");
Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier));
dtSecretManager.renewToken(token, "JobTracker");
+ ugi.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ token.renew(config);
+ token.cancel(config);
+ return null;
+ }
+ });
}
@Test
Added: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java?rev=1188836&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java (added)
+++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java Tue Oct 25 17:57:05 2011
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.web;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.web.resources.DelegationParam;
+import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
+import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
+import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.junit.Assert;
+import org.junit.Test;
+import static org.mockito.Mockito.mock;
+
+public class TestWebHdfsUrl {
+
+ @Test
+ public void testDelegationTokenInUrl() throws IOException {
+ Configuration conf = new Configuration();
+ final String uri = WebHdfsFileSystem.SCHEME + "://" + "127.0.0.1:9071";
+ // Turn on security
+ conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(
+ ugi.getUserName()), null, null);
+ FSNamesystem namesystem = mock(FSNamesystem.class);
+ DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(
+ 86400000, 86400000, 86400000, 86400000, namesystem);
+ dtSecretManager.startThreads();
+ Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
+ dtId, dtSecretManager);
+ token.setService(new Text("127.0.0.1:9071"));
+ token.setKind(WebHdfsFileSystem.TOKEN_KIND);
+ ugi.addToken(token);
+ final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) FileSystem.get(
+ URI.create(uri), conf);
+ String tokenString = token.encodeToUrlString();
+ Path fsPath = new Path("/");
+ URL renewTokenUrl = webhdfs.toUrl(PutOpParam.Op.RENEWDELEGATIONTOKEN,
+ fsPath, new TokenArgumentParam(tokenString));
+ URL cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
+ fsPath, new TokenArgumentParam(tokenString));
+ Assert.assertEquals(
+ generateUrlQueryPrefix(PutOpParam.Op.RENEWDELEGATIONTOKEN,
+ ugi.getUserName())
+ + "&token=" + tokenString, renewTokenUrl.getQuery());
+ Token<DelegationTokenIdentifier> delegationToken = new Token<DelegationTokenIdentifier>(
+ token);
+ delegationToken.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
+ Assert.assertEquals(
+ generateUrlQueryPrefix(PutOpParam.Op.CANCELDELEGATIONTOKEN,
+ ugi.getUserName())
+ + "&token="
+ + tokenString
+ + "&"
+ + DelegationParam.NAME
+ + "="
+ + delegationToken.encodeToUrlString(), cancelTokenUrl.getQuery());
+ }
+
+ private String generateUrlQueryPrefix(HttpOpParam.Op op, String username) {
+ return "op=" + op.toString() + "&user.name=" + username;
+ }
+}
Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/security/token/delegation/TestDelegationToken.java?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/security/token/delegation/TestDelegationToken.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/security/token/delegation/TestDelegationToken.java Tue Oct 25 17:57:05 2011
@@ -360,6 +360,8 @@ public class TestDelegationToken {
byte[] storedPassword = dtSecretManager.retrievePassword(id);
byte[] password = dtSecretManager.createPassword(id, key);
Assert.assertTrue(Arrays.equals(password, storedPassword));
+ //verify by secret manager api
+ dtSecretManager.verifyToken(id, password);
}
} finally {
dtSecretManager.stopThreads();
Modified: hadoop/common/branches/branch-0.20-security/src/webapps/hdfs/nn_browsedfscontent.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/webapps/hdfs/nn_browsedfscontent.jsp?rev=1188836&r1=1188835&r2=1188836&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/webapps/hdfs/nn_browsedfscontent.jsp (original)
+++ hadoop/common/branches/branch-0.20-security/src/webapps/hdfs/nn_browsedfscontent.jsp Tue Oct 25 17:57:05 2011
@@ -21,9 +21,9 @@
%>
<%!
static String getDelegationToken(final NameNode nn,
+ final UserGroupInformation ugi,
HttpServletRequest request, Configuration conf)
throws IOException, InterruptedException {
- final UserGroupInformation ugi = JspHelper.getUGI(request, conf);
Token<DelegationTokenIdentifier> token =
ugi.doAs(
new PrivilegedExceptionAction<Token<DelegationTokenIdentifier>>()
@@ -36,14 +36,16 @@
}
public void redirectToRandomDataNode(
- NameNode nn,
+ ServletContext context,
HttpServletRequest request,
- HttpServletResponse resp,
- Configuration conf
+ HttpServletResponse resp
) throws IOException, InterruptedException {
+ Configuration conf = (Configuration) context.getAttribute(JspHelper.CURRENT_CONF);
+ NameNode nn = (NameNode)context.getAttribute("name.node");
+ final UserGroupInformation ugi = JspHelper.getUGI(context, request, conf);
String tokenString = null;
if (UserGroupInformation.isSecurityEnabled()) {
- tokenString = getDelegationToken(nn, request, conf);
+ tokenString = getDelegationToken(nn, ugi, request, conf);
}
FSNamesystem fsn = nn.getNamesystem();
String datanode = fsn.randomDataNode();
@@ -76,9 +78,7 @@
<body>
<%
- NameNode nn = (NameNode)application.getAttribute("name.node");
- Configuration conf = (Configuration) application.getAttribute(JspHelper.CURRENT_CONF);
- redirectToRandomDataNode(nn, request, response, conf);
+ redirectToRandomDataNode(application, request, response);
%>
<hr>