You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2012/06/28 09:00:16 UTC
svn commit: r1354832 [1/2] - in
/hadoop/common/branches/HDFS-3092/hadoop-common-project:
hadoop-annotations/src/main/java/org/apache/hadoop/classification/
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/
hadoop-auth/src/main...
Author: szetszwo
Date: Thu Jun 28 06:59:38 2012
New Revision: 1354832
URL: http://svn.apache.org/viewvc?rev=1354832&view=rev
Log:
Merge r1346682 through r1354801 from trunk.
Added:
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/CMakeLists.txt
- copied unchanged from r1354801, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/config.h.cmake
- copied unchanged from r1354801, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/config.h.cmake
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
- copied unchanged from r1354801, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
- copied unchanged from r1354801, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
Removed:
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/.autom4te.cfg
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/Makefile.am
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/acinclude.m4
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/configure.ac
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/lib/Makefile.am
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am
Modified:
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceAudience.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceStability.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicy.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java
hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceAudience.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceAudience.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceAudience.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceAudience.java Thu Jun 28 06:59:38 2012
@@ -21,6 +21,24 @@ import java.lang.annotation.Documented;
/**
* Annotation to inform users of a package, class or method's intended audience.
+ * Currently the audience can be {@link Public}, {@link LimitedPrivate} or
+ * {@link Private}. <br>
+ * All public classes must have InterfaceAudience annotation. <br>
+ * <ul>
+ * <li>Public classes that are not marked with this annotation must be
+ * considered by default as {@link Private}.</li>
+ *
+ * <li>External applications must only use classes that are marked
+ * {@link Public}. Avoid using non public classes as these classes
+ * could be removed or change in incompatible ways.</li>
+ *
+ * <li>Hadoop projects must only use classes that are marked
+ * {@link LimitedPrivate} or {@link Public}</li>
+ *
+ * <li> Methods may have a different annotation that it is more restrictive
+ * compared to the audience classification of the class. Example: A class
+ * might be {@link Public}, but a method may be {@link LimitedPrivate}
+ * </li></ul>
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceStability.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceStability.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceStability.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-annotations/src/main/java/org/apache/hadoop/classification/InterfaceStability.java Thu Jun 28 06:59:38 2012
@@ -19,9 +19,20 @@ package org.apache.hadoop.classification
import java.lang.annotation.Documented;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+
/**
* Annotation to inform users of how much to rely on a particular package,
- * class or method not changing over time.
+ * class or method not changing over time. Currently the stability can be
+ * {@link Stable}, {@link Evolving} or {@link Unstable}. <br>
+ *
+ * <ul><li>All classes that are annotated with {@link Public} or
+ * {@link LimitedPrivate} must have InterfaceStability annotation. </li>
+ * <li>Classes that are {@link Private} are to be considered unstable unless
+ * a different InterfaceStability annotation states otherwise.</li>
+ * <li>Incompatible changes must not be made to classes marked as stable.</li>
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java Thu Jun 28 06:59:38 2012
@@ -266,6 +266,7 @@ public class AuthenticatedURL {
}
}
} else {
+ token.set(null);
throw new AuthenticationException("Authentication failed, status: " + conn.getResponseCode() +
", message: " + conn.getResponseMessage());
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java Thu Jun 28 06:59:38 2012
@@ -341,45 +341,50 @@ public class AuthenticationFilter implem
LOG.warn("AuthenticationToken ignored: " + ex.getMessage());
token = null;
}
- if (token == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Request [{}] triggering authentication", getRequestURL(httpRequest));
- }
- token = authHandler.authenticate(httpRequest, httpResponse);
- if (token != null && token != AuthenticationToken.ANONYMOUS) {
- token.setExpires(System.currentTimeMillis() + getValidity() * 1000);
- }
- newToken = true;
- }
- if (token != null) {
- unauthorizedResponse = false;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Request [{}] user [{}] authenticated", getRequestURL(httpRequest), token.getUserName());
+ if (authHandler.managementOperation(token, httpRequest, httpResponse)) {
+ if (token == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Request [{}] triggering authentication", getRequestURL(httpRequest));
+ }
+ token = authHandler.authenticate(httpRequest, httpResponse);
+ if (token != null && token.getExpires() != 0 &&
+ token != AuthenticationToken.ANONYMOUS) {
+ token.setExpires(System.currentTimeMillis() + getValidity() * 1000);
+ }
+ newToken = true;
}
- final AuthenticationToken authToken = token;
- httpRequest = new HttpServletRequestWrapper(httpRequest) {
-
- @Override
- public String getAuthType() {
- return authToken.getType();
+ if (token != null) {
+ unauthorizedResponse = false;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Request [{}] user [{}] authenticated", getRequestURL(httpRequest), token.getUserName());
}
+ final AuthenticationToken authToken = token;
+ httpRequest = new HttpServletRequestWrapper(httpRequest) {
- @Override
- public String getRemoteUser() {
- return authToken.getUserName();
- }
+ @Override
+ public String getAuthType() {
+ return authToken.getType();
+ }
- @Override
- public Principal getUserPrincipal() {
- return (authToken != AuthenticationToken.ANONYMOUS) ? authToken : null;
+ @Override
+ public String getRemoteUser() {
+ return authToken.getUserName();
+ }
+
+ @Override
+ public Principal getUserPrincipal() {
+ return (authToken != AuthenticationToken.ANONYMOUS) ? authToken : null;
+ }
+ };
+ if (newToken && !token.isExpired() && token != AuthenticationToken.ANONYMOUS) {
+ String signedToken = signer.sign(token.toString());
+ Cookie cookie = createCookie(signedToken);
+ httpResponse.addCookie(cookie);
}
- };
- if (newToken && token != AuthenticationToken.ANONYMOUS) {
- String signedToken = signer.sign(token.toString());
- Cookie cookie = createCookie(signedToken);
- httpResponse.addCookie(cookie);
+ filterChain.doFilter(httpRequest, httpResponse);
}
- filterChain.doFilter(httpRequest, httpResponse);
+ } else {
+ unauthorizedResponse = false;
}
} catch (AuthenticationException ex) {
unauthorizedMsg = ex.toString();
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java Thu Jun 28 06:59:38 2012
@@ -59,6 +59,34 @@ public interface AuthenticationHandler {
public void destroy();
/**
+ * Performs an authentication management operation.
+ * <p/>
+ * This is useful for handling operations like get/renew/cancel
+ * delegation tokens which are being handled as operations of the
+ * service end-point.
+ * <p/>
+ * If the method returns <code>TRUE</code> the request will continue normal
+ * processing, this means the method has not produced any HTTP response.
+ * <p/>
+ * If the method returns <code>FALSE</code> the request will end, this means
+ * the method has produced the corresponding HTTP response.
+ *
+ * @param token the authentication token if any, otherwise <code>NULL</code>.
+ * @param request the HTTP client request.
+ * @param response the HTTP client response.
+ * @return <code>TRUE</code> if the request should be processed as a regular
+ * request,
+ * <code>FALSE</code> otherwise.
+ *
+ * @throws IOException thrown if an IO error occurred.
+ * @throws AuthenticationException thrown if an Authentication error occurred.
+ */
+ public boolean managementOperation(AuthenticationToken token,
+ HttpServletRequest request,
+ HttpServletResponse response)
+ throws IOException, AuthenticationException;
+
+ /**
* Performs an authentication step for the given HTTP client request.
* <p/>
* This method is invoked by the {@link AuthenticationFilter} only if the HTTP client request is
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java Thu Jun 28 06:59:38 2012
@@ -115,10 +115,10 @@ public class AuthenticationToken impleme
*/
private void generateToken() {
StringBuffer sb = new StringBuffer();
- sb.append(USER_NAME).append("=").append(userName).append(ATTR_SEPARATOR);
- sb.append(PRINCIPAL).append("=").append(principal).append(ATTR_SEPARATOR);
- sb.append(TYPE).append("=").append(type).append(ATTR_SEPARATOR);
- sb.append(EXPIRES).append("=").append(expires);
+ sb.append(USER_NAME).append("=").append(getUserName()).append(ATTR_SEPARATOR);
+ sb.append(PRINCIPAL).append("=").append(getName()).append(ATTR_SEPARATOR);
+ sb.append(TYPE).append("=").append(getType()).append(ATTR_SEPARATOR);
+ sb.append(EXPIRES).append("=").append(getExpires());
token = sb.toString();
}
@@ -165,7 +165,7 @@ public class AuthenticationToken impleme
* @return if the token has expired.
*/
public boolean isExpired() {
- return expires != -1 && System.currentTimeMillis() > expires;
+ return getExpires() != -1 && System.currentTimeMillis() > getExpires();
}
/**
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java Thu Jun 28 06:59:38 2012
@@ -233,6 +233,27 @@ public class KerberosAuthenticationHandl
}
/**
+ * This is an empty implementation, it always returns <code>TRUE</code>.
+ *
+ *
+ *
+ * @param token the authentication token if any, otherwise <code>NULL</code>.
+ * @param request the HTTP client request.
+ * @param response the HTTP client response.
+ *
+ * @return <code>TRUE</code>
+ * @throws IOException it is never thrown.
+ * @throws AuthenticationException it is never thrown.
+ */
+ @Override
+ public boolean managementOperation(AuthenticationToken token,
+ HttpServletRequest request,
+ HttpServletResponse response)
+ throws IOException, AuthenticationException {
+ return true;
+ }
+
+ /**
* It enforces the the Kerberos SPNEGO authentication sequence returning an {@link AuthenticationToken} only
* after the Kerberos SPNEGO sequence has completed successfully.
* <p/>
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java Thu Jun 28 06:59:38 2012
@@ -94,6 +94,27 @@ public class PseudoAuthenticationHandler
}
/**
+ * This is an empty implementation, it always returns <code>TRUE</code>.
+ *
+ *
+ *
+ * @param token the authentication token if any, otherwise <code>NULL</code>.
+ * @param request the HTTP client request.
+ * @param response the HTTP client response.
+ *
+ * @return <code>TRUE</code>
+ * @throws IOException it is never thrown.
+ * @throws AuthenticationException it is never thrown.
+ */
+ @Override
+ public boolean managementOperation(AuthenticationToken token,
+ HttpServletRequest request,
+ HttpServletResponse response)
+ throws IOException, AuthenticationException {
+ return true;
+ }
+
+ /**
* Authenticates an HTTP client request.
* <p/>
* It extracts the {@link PseudoAuthenticator#USER_NAME} parameter from the query string and creates
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java Thu Jun 28 06:59:38 2012
@@ -13,6 +13,7 @@
*/
package org.apache.hadoop.security.authentication.client;
+import junit.framework.Assert;
import junit.framework.TestCase;
import org.mockito.Mockito;
@@ -100,11 +101,14 @@ public class TestAuthenticatedURL extend
headers.put("Set-Cookie", cookies);
Mockito.when(conn.getHeaderFields()).thenReturn(headers);
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ token.set("bar");
try {
- AuthenticatedURL.extractToken(conn, new AuthenticatedURL.Token());
+ AuthenticatedURL.extractToken(conn, token);
fail();
} catch (AuthenticationException ex) {
// Expected
+ Assert.assertFalse(token.isSet());
} catch (Exception ex) {
fail();
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java Thu Jun 28 06:59:38 2012
@@ -71,7 +71,9 @@ public class TestAuthenticationFilter ex
public static class DummyAuthenticationHandler implements AuthenticationHandler {
public static boolean init;
+ public static boolean managementOperationReturn;
public static boolean destroy;
+ public static boolean expired;
public static final String TYPE = "dummy";
@@ -83,6 +85,20 @@ public class TestAuthenticationFilter ex
@Override
public void init(Properties config) throws ServletException {
init = true;
+ managementOperationReturn =
+ config.getProperty("management.operation.return", "true").equals("true");
+ expired = config.getProperty("expired.token", "false").equals("true");
+ }
+
+ @Override
+ public boolean managementOperation(AuthenticationToken token,
+ HttpServletRequest request,
+ HttpServletResponse response)
+ throws IOException, AuthenticationException {
+ if (!managementOperationReturn) {
+ response.setStatus(HttpServletResponse.SC_ACCEPTED);
+ }
+ return managementOperationReturn;
}
@Override
@@ -102,7 +118,7 @@ public class TestAuthenticationFilter ex
String param = request.getParameter("authenticated");
if (param != null && param.equals("true")) {
token = new AuthenticationToken("u", "p", "t");
- token.setExpires(System.currentTimeMillis() + 1000);
+ token.setExpires((expired) ? 0 : System.currentTimeMillis() + 1000);
} else {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
}
@@ -170,10 +186,14 @@ public class TestAuthenticationFilter ex
filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameterNames()).thenReturn(
- new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements());
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ "management.operation.return")).elements());
filter.init(config);
assertTrue(DummyAuthenticationHandler.init);
} finally {
@@ -201,10 +221,14 @@ public class TestAuthenticationFilter ex
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameterNames()).thenReturn(
- new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements());
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ "management.operation.return")).elements());
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -221,12 +245,16 @@ public class TestAuthenticationFilter ex
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameter(AuthenticationFilter.SIGNATURE_SECRET)).thenReturn("secret");
Mockito.when(config.getInitParameterNames()).thenReturn(
- new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
- AuthenticationFilter.SIGNATURE_SECRET)).elements());
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ AuthenticationFilter.SIGNATURE_SECRET,
+ "management.operation.return")).elements());
filter.init(config);
AuthenticationToken token = new AuthenticationToken("u", "p", DummyAuthenticationHandler.TYPE);
@@ -250,12 +278,15 @@ public class TestAuthenticationFilter ex
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameter(AuthenticationFilter.SIGNATURE_SECRET)).thenReturn("secret");
Mockito.when(config.getInitParameterNames()).thenReturn(
- new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
- AuthenticationFilter.SIGNATURE_SECRET)).elements());
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ AuthenticationFilter.SIGNATURE_SECRET,
+ "management.operation.return")).elements());
filter.init(config);
AuthenticationToken token = new AuthenticationToken("u", "p", "invalidtype");
@@ -284,12 +315,16 @@ public class TestAuthenticationFilter ex
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameter(AuthenticationFilter.SIGNATURE_SECRET)).thenReturn("secret");
Mockito.when(config.getInitParameterNames()).thenReturn(
- new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
- AuthenticationFilter.SIGNATURE_SECRET)).elements());
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ AuthenticationFilter.SIGNATURE_SECRET,
+ "management.operation.return")).elements());
filter.init(config);
AuthenticationToken token = new AuthenticationToken("u", "p", "invalidtype");
@@ -318,10 +353,14 @@ public class TestAuthenticationFilter ex
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameterNames()).thenReturn(
- new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements());
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ "management.operation.return")).elements());
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -349,10 +388,16 @@ public class TestAuthenticationFilter ex
}
}
- private void _testDoFilterAuthentication(boolean withDomainPath, boolean invalidToken) throws Exception {
+ private void _testDoFilterAuthentication(boolean withDomainPath,
+ boolean invalidToken,
+ boolean expired) throws Exception {
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("true");
+ Mockito.when(config.getInitParameter("expired.token")).
+ thenReturn(Boolean.toString(expired));
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TOKEN_VALIDITY)).thenReturn("1000");
@@ -360,7 +405,9 @@ public class TestAuthenticationFilter ex
Mockito.when(config.getInitParameterNames()).thenReturn(
new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE,
AuthenticationFilter.AUTH_TOKEN_VALIDITY,
- AuthenticationFilter.SIGNATURE_SECRET)).elements());
+ AuthenticationFilter.SIGNATURE_SECRET,
+ "management.operation.return",
+ "expired.token")).elements());
if (withDomainPath) {
Mockito.when(config.getInitParameter(AuthenticationFilter.COOKIE_DOMAIN)).thenReturn(".foo.com");
@@ -370,7 +417,8 @@ public class TestAuthenticationFilter ex
AuthenticationFilter.AUTH_TOKEN_VALIDITY,
AuthenticationFilter.SIGNATURE_SECRET,
AuthenticationFilter.COOKIE_DOMAIN,
- AuthenticationFilter.COOKIE_PATH)).elements());
+ AuthenticationFilter.COOKIE_PATH,
+ "management.operation.return")).elements());
}
filter.init(config);
@@ -416,26 +464,32 @@ public class TestAuthenticationFilter ex
filter.doFilter(request, response, chain);
- assertNotNull(setCookie[0]);
- assertEquals(AuthenticatedURL.AUTH_COOKIE, setCookie[0].getName());
- assertTrue(setCookie[0].getValue().contains("u="));
- assertTrue(setCookie[0].getValue().contains("p="));
- assertTrue(setCookie[0].getValue().contains("t="));
- assertTrue(setCookie[0].getValue().contains("e="));
- assertTrue(setCookie[0].getValue().contains("s="));
- assertTrue(calledDoFilter[0]);
-
- Signer signer = new Signer("secret".getBytes());
- String value = signer.verifyAndExtract(setCookie[0].getValue());
- AuthenticationToken token = AuthenticationToken.parse(value);
- assertEquals(System.currentTimeMillis() + 1000 * 1000, token.getExpires(), 100);
-
- if (withDomainPath) {
- assertEquals(".foo.com", setCookie[0].getDomain());
- assertEquals("/bar", setCookie[0].getPath());
+ if (expired) {
+ Mockito.verify(response, Mockito.never()).
+ addCookie(Mockito.any(Cookie.class));
} else {
- assertNull(setCookie[0].getDomain());
- assertNull(setCookie[0].getPath());
+ assertNotNull(setCookie[0]);
+ assertEquals(AuthenticatedURL.AUTH_COOKIE, setCookie[0].getName());
+ assertTrue(setCookie[0].getValue().contains("u="));
+ assertTrue(setCookie[0].getValue().contains("p="));
+ assertTrue(setCookie[0].getValue().contains("t="));
+ assertTrue(setCookie[0].getValue().contains("e="));
+ assertTrue(setCookie[0].getValue().contains("s="));
+ assertTrue(calledDoFilter[0]);
+
+ Signer signer = new Signer("secret".getBytes());
+ String value = signer.verifyAndExtract(setCookie[0].getValue());
+ AuthenticationToken token = AuthenticationToken.parse(value);
+ assertEquals(System.currentTimeMillis() + 1000 * 1000,
+ token.getExpires(), 100);
+
+ if (withDomainPath) {
+ assertEquals(".foo.com", setCookie[0].getDomain());
+ assertEquals("/bar", setCookie[0].getPath());
+ } else {
+ assertNull(setCookie[0].getDomain());
+ assertNull(setCookie[0].getPath());
+ }
}
} finally {
filter.destroy();
@@ -443,25 +497,33 @@ public class TestAuthenticationFilter ex
}
public void testDoFilterAuthentication() throws Exception {
- _testDoFilterAuthentication(false, false);
+ _testDoFilterAuthentication(false, false, false);
+ }
+
+ public void testDoFilterAuthenticationImmediateExpiration() throws Exception {
+ _testDoFilterAuthentication(false, false, true);
}
public void testDoFilterAuthenticationWithInvalidToken() throws Exception {
- _testDoFilterAuthentication(false, true);
+ _testDoFilterAuthentication(false, true, false);
}
public void testDoFilterAuthenticationWithDomainPath() throws Exception {
- _testDoFilterAuthentication(true, false);
+ _testDoFilterAuthentication(true, false, false);
}
public void testDoFilterAuthenticated() throws Exception {
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameterNames()).thenReturn(
- new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements());
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ "management.operation.return")).elements());
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -503,10 +565,14 @@ public class TestAuthenticationFilter ex
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameterNames()).thenReturn(
- new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements());
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ "management.operation.return")).elements());
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -563,10 +629,14 @@ public class TestAuthenticationFilter ex
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameterNames()).thenReturn(
- new Vector<String>(Arrays.asList(AuthenticationFilter.AUTH_TYPE)).elements());
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ "management.operation.return")).elements());
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
@@ -618,4 +688,50 @@ public class TestAuthenticationFilter ex
}
}
+ public void testManagementOperation() throws Exception {
+ AuthenticationFilter filter = new AuthenticationFilter();
+ try {
+ FilterConfig config = Mockito.mock(FilterConfig.class);
+ Mockito.when(config.getInitParameter("management.operation.return")).
+ thenReturn("false");
+ Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).
+ thenReturn(DummyAuthenticationHandler.class.getName());
+ Mockito.when(config.getInitParameterNames()).thenReturn(
+ new Vector<String>(
+ Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+ "management.operation.return")).elements());
+ filter.init(config);
+
+ HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+ Mockito.when(request.getRequestURL()).
+ thenReturn(new StringBuffer("http://foo:8080/bar"));
+
+ HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+ FilterChain chain = Mockito.mock(FilterChain.class);
+
+ filter.doFilter(request, response, chain);
+ Mockito.verify(response).setStatus(HttpServletResponse.SC_ACCEPTED);
+ Mockito.verifyNoMoreInteractions(response);
+
+ Mockito.reset(request);
+ Mockito.reset(response);
+
+ AuthenticationToken token = new AuthenticationToken("u", "p", "t");
+ token.setExpires(System.currentTimeMillis() + 1000);
+ Signer signer = new Signer("secret".getBytes());
+ String tokenSigned = signer.sign(token.toString());
+ Cookie cookie = new Cookie(AuthenticatedURL.AUTH_COOKIE, tokenSigned);
+ Mockito.when(request.getCookies()).thenReturn(new Cookie[]{cookie});
+
+ filter.doFilter(request, response, chain);
+
+ Mockito.verify(response).setStatus(HttpServletResponse.SC_ACCEPTED);
+ Mockito.verifyNoMoreInteractions(response);
+
+ } finally {
+ filter.destroy();
+ }
+ }
+
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt Thu Jun 28 06:59:38 2012
@@ -9,11 +9,11 @@ Trunk (unreleased changes)
NEW FEATURES
- HADOOP-8135. Add ByteBufferReadable interface to FSDataInputStream. (Henry
- Robinson via atm)
+ HADOOP-8469. Make NetworkTopology class pluggable. (Junping Du via
+ szetszwo)
- HDFS-3042. Automatic failover support for NameNode HA (todd)
- (see dedicated section below for breakdown of subtasks)
+ HADOOP-8470. Add NetworkTopologyWithNodeGroup, a 4-layer implementation
+ of NetworkTopology. (Junping Du via szetszwo)
IMPROVEMENTS
@@ -55,9 +55,6 @@ Trunk (unreleased changes)
HADOOP-7994. Remove getProtocolVersion and getProtocolSignature from the
client side translator and server side implementation. (jitendra)
- HADOOP-8244. Improve comments on ByteBufferReadable.read. (Henry Robinson
- via atm)
-
HADOOP-7757. Test file reference count is at least 3x actual value (Jon
Eagles via bobby)
@@ -82,6 +79,9 @@ Trunk (unreleased changes)
HADOOP-7659. fs -getmerge isn't guaranteed to work well over non-HDFS
filesystems (harsh)
+ HADOOP-8059. Add javadoc to InterfaceAudience and InterfaceStability.
+ (Brandon Li via suresh)
+
BUG FIXES
HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName.
@@ -165,39 +165,7 @@ Trunk (unreleased changes)
HADOOP-7761. Improve the performance of raw comparisons. (todd)
- BREAKDOWN OF HDFS-3042 SUBTASKS
-
- HADOOP-8220. ZKFailoverController doesn't handle failure to become active
- correctly (todd)
-
- HADOOP-8228. Auto HA: Refactor tests and add stress tests. (todd)
-
- HADOOP-8215. Security support for ZK Failover controller (todd)
-
- HADOOP-8245. Fix flakiness in TestZKFailoverController (todd)
-
- HADOOP-8257. TestZKFailoverControllerStress occasionally fails with Mockito
- error (todd)
-
- HADOOP-8260. Replace ClientBaseWithFixes with our own modified copy of the
- class (todd)
-
- HADOOP-8246. Auto-HA: automatically scope znode by nameservice ID (todd)
-
- HADOOP-8247. Add a config to enable auto-HA, which disables manual
- FailoverController (todd)
-
- HADOOP-8306. ZKFC: improve error message when ZK is not running. (todd)
-
- HADOOP-8279. Allow manual failover to be invoked when auto-failover is
- enabled. (todd)
-
- HADOOP-8276. Auto-HA: add config for java options to pass to zkfc daemon
- (todd via eli)
-
- HADOOP-8405. ZKFC tests leak ZK instances. (todd)
-
-Release 2.0.1-alpha - UNRELEASED
+Branch-2 ( Unreleased changes )
INCOMPATIBLE CHANGES
@@ -206,6 +174,17 @@ Release 2.0.1-alpha - UNRELEASED
NEW FEATURES
+ HDFS-3042. Automatic failover support for NameNode HA (todd)
+ (see dedicated section below for breakdown of subtasks)
+
+ HADOOP-8135. Add ByteBufferReadable interface to FSDataInputStream. (Henry
+ Robinson via atm)
+
+ HADOOP-8458. Add management hook to AuthenticationHandler to enable
+ delegation token operations support (tucu)
+
+ HADOOP-8465. hadoop-auth should support ephemeral authentication (tucu)
+
IMPROVEMENTS
HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual
@@ -228,6 +207,14 @@ Release 2.0.1-alpha - UNRELEASED
HADOOP-8450. Remove src/test/system. (eli)
+ HADOOP-8244. Improve comments on ByteBufferReadable.read. (Henry Robinson
+ via atm)
+
+ HADOOP-8368. Use CMake rather than autotools to build native code (ccccabe via tucu)
+
+ HADOOP-8524. Allow users to get source of a Configuration
+ parameter (harsh)
+
BUG FIXES
HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname
@@ -267,6 +254,54 @@ Release 2.0.1-alpha - UNRELEASED
HADOOP-8481. update BUILDING.txt to talk about cmake rather than autotools.
(Colin Patrick McCabe via eli)
+ HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. (eli)
+
+ HADOOP-8488. test-patch.sh gives +1 even if the native build fails.
+ (Colin Patrick McCabe via eli)
+
+ HADOOP-8507. Avoid OOM while deserializing DelegationTokenIdentifer.
+ (Colin Patrick McCabe via eli)
+
+ HADOOP-8433. Don't set HADOOP_LOG_DIR in hadoop-env.sh.
+ (Brahma Reddy Battula via eli)
+
+ HADOOP-8509. JarFinder duplicate entry: META-INF/MANIFEST.MF exception (tucu)
+
+ HADOOP-8512. AuthenticatedURL should reset the Token when the server returns
+ other than OK on authentication (tucu)
+
+ BREAKDOWN OF HDFS-3042 SUBTASKS
+
+ HADOOP-8220. ZKFailoverController doesn't handle failure to become active
+ correctly (todd)
+
+ HADOOP-8228. Auto HA: Refactor tests and add stress tests. (todd)
+
+ HADOOP-8215. Security support for ZK Failover controller (todd)
+
+ HADOOP-8245. Fix flakiness in TestZKFailoverController (todd)
+
+ HADOOP-8257. TestZKFailoverControllerStress occasionally fails with Mockito
+ error (todd)
+
+ HADOOP-8260. Replace ClientBaseWithFixes with our own modified copy of the
+ class (todd)
+
+ HADOOP-8246. Auto-HA: automatically scope znode by nameservice ID (todd)
+
+ HADOOP-8247. Add a config to enable auto-HA, which disables manual
+ FailoverController (todd)
+
+ HADOOP-8306. ZKFC: improve error message when ZK is not running. (todd)
+
+ HADOOP-8279. Allow manual failover to be invoked when auto-failover is
+ enabled. (todd)
+
+ HADOOP-8276. Auto-HA: add config for java options to pass to zkfc daemon
+ (todd via eli)
+
+ HADOOP-8405. ZKFC tests leak ZK instances. (todd)
+
Release 2.0.0-alpha - 05-23-2012
INCOMPATIBLE CHANGES
@@ -696,6 +731,12 @@ Release 0.23.3 - UNRELEASED
HADOOP-8373. Port RPC.getServerAddress to 0.23 (Daryn Sharp via bobby)
+ HADOOP-8495. Update Netty to avoid leaking file descriptors during shuffle
+ (Jason Lowe via tgraves)
+
+ HADOOP-8129. ViewFileSystemTestSetup setupForViewFileSystem is erring
+ (Ahmed Radwan and Ravi Prakash via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1346682-1354801
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/pom.xml?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/pom.xml Thu Jun 28 06:59:38 2012
@@ -536,32 +536,11 @@
<snappy.prefix>/usr/local</snappy.prefix>
<snappy.lib>${snappy.prefix}/lib</snappy.lib>
<snappy.include>${snappy.prefix}/include</snappy.include>
+ <runas.home></runas.home>
</properties>
<build>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>compile</id>
- <phase>compile</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <mkdir dir="${project.build.directory}/native/javah"/>
- <copy toDir="${project.build.directory}/native">
- <fileset dir="${basedir}/src/main/native"/>
- </copy>
- <mkdir dir="${project.build.directory}/native/m4"/>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>native-maven-plugin</artifactId>
<executions>
@@ -590,73 +569,27 @@
</executions>
</plugin>
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>make-maven-plugin</artifactId>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
- <id>compile</id>
+ <id>make</id>
<phase>compile</phase>
- <goals>
- <goal>autoreconf</goal>
- <goal>configure</goal>
- <goal>make-install</goal>
- </goals>
+ <goals><goal>run</goal></goals>
+ <configuration>
+ <target>
+ <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
+ <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
+ <env key="CFLAGS" value="-I${snappy.include}"/>
+ <env key="LDFLAGS" value="-L${snappy.lib}"/>
+ </exec>
+ <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
+ <arg line="VERBOSE=1"/>
+ </exec>
+ </target>
+ </configuration>
</execution>
</executions>
- <configuration>
- <!-- autoreconf settings -->
- <workDir>${project.build.directory}/native</workDir>
- <arguments>
- <argument>-i</argument>
- <argument>-f</argument>
- </arguments>
-
- <!-- configure settings -->
- <configureEnvironment>
- <property>
- <name>OS_NAME</name>
- <value>${os.name}</value>
- </property>
- <property>
- <name>OS_ARCH</name>
- <value>${os.arch}</value>
- </property>
- <property>
- <name>JVM_DATA_MODEL</name>
- <value>${sun.arch.data.model}</value>
- </property>
- </configureEnvironment>
- <configureOptions>
- <configureOption>CPPFLAGS=-I${snappy.include}</configureOption>
- <configureOption>LDFLAGS=-L${snappy.lib}</configureOption>
- </configureOptions>
- <configureWorkDir>${project.build.directory}/native</configureWorkDir>
- <prefix>/usr/local</prefix>
-
- <!-- make settings -->
- <installEnvironment>
- <property>
- <name>OS_NAME</name>
- <value>${os.name}</value>
- </property>
- <property>
- <name>OS_ARCH</name>
- <value>${os.arch}</value>
- </property>
- <property>
- <name>JVM_DATA_MODEL</name>
- <value>${sun.arch.data.model}</value>
- </property>
- <property>
- <name>HADOOP_NATIVE_SRCDIR</name>
- <value>${project.build.directory}/native</value>
- </property>
- </installEnvironment>
-
- <!-- configure & make settings -->
- <destDir>${project.build.directory}/native/target</destDir>
-
- </configuration>
</plugin>
</plugins>
</build>
@@ -700,7 +633,7 @@
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
- <id>compile</id>
+ <id>kdc</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh Thu Jun 28 06:59:38 2012
@@ -61,7 +61,7 @@ export HADOOP_CLIENT_OPTS="-Xmx128m $HAD
export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
# Where log files are stored. $HADOOP_HOME/logs by default.
-export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
+#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
# Where log files are stored in the secure data environment.
export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
Propchange: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1346682-1354801
Propchange: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1346682-1354801
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Thu Jun 28 06:59:38 2012
@@ -1071,6 +1071,38 @@ public class Configuration implements It
}
/**
+ * Gets the absolute path to the resource object (file, URL, etc.), for a given
+ * property name.
+ *
+ * @param name - The property name to get the source of.
+ * @return null - If the property or its source wasn't found or if the property
+ * was defined in code (i.e. in a Configuration instance, not from a physical
+ * resource). Otherwise, returns the absolute path of the resource that loaded
+ * the property name, as a String.
+ */
+ @InterfaceStability.Unstable
+ public synchronized String getPropertySource(String name) {
+ if (properties == null) {
+ // If properties is null, it means a resource was newly added
+ // but the props were cleared so as to load it upon future
+ // requests. So lets force a load by asking a properties list.
+ getProps();
+ }
+ // Return a null right away if our properties still
+ // haven't loaded or the resource mapping isn't defined
+ if (properties == null || updatingResource == null) {
+ return null;
+ } else {
+ String source = updatingResource.get(name);
+ if (source == null || source.equals(UNKNOWN_RESOURCE)) {
+ return null;
+ } else {
+ return source;
+ }
+ }
+ }
+
+ /**
* A class that represents a set of positive integer ranges. It parses
* strings of the form: "2-3,5,7-" where ranges are separated by comma and
* the lower/upper bounds are separated by dash. Either the lower or upper
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java Thu Jun 28 06:59:38 2012
@@ -63,7 +63,9 @@ public class CommonConfigurationKeysPubl
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY =
"net.topology.node.switch.mapping.impl";
-
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String NET_TOPOLOGY_IMPL_KEY =
+ "net.topology.impl";
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY =
"net.topology.table.file.name";
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java Thu Jun 28 06:59:38 2012
@@ -254,7 +254,7 @@ public class FileStatus implements Writa
// Writable
//////////////////////////////////////////////////
public void write(DataOutput out) throws IOException {
- Text.writeString(out, getPath().toString(), Text.ONE_MEGABYTE);
+ Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN);
out.writeLong(getLen());
out.writeBoolean(isDirectory());
out.writeShort(getReplication());
@@ -262,16 +262,16 @@ public class FileStatus implements Writa
out.writeLong(getModificationTime());
out.writeLong(getAccessTime());
getPermission().write(out);
- Text.writeString(out, getOwner(), Text.ONE_MEGABYTE);
- Text.writeString(out, getGroup(), Text.ONE_MEGABYTE);
+ Text.writeString(out, getOwner(), Text.DEFAULT_MAX_LEN);
+ Text.writeString(out, getGroup(), Text.DEFAULT_MAX_LEN);
out.writeBoolean(isSymlink());
if (isSymlink()) {
- Text.writeString(out, getSymlink().toString(), Text.ONE_MEGABYTE);
+ Text.writeString(out, getSymlink().toString(), Text.DEFAULT_MAX_LEN);
}
}
public void readFields(DataInput in) throws IOException {
- String strPath = Text.readString(in, Text.ONE_MEGABYTE);
+ String strPath = Text.readString(in, Text.DEFAULT_MAX_LEN);
this.path = new Path(strPath);
this.length = in.readLong();
this.isdir = in.readBoolean();
@@ -280,10 +280,10 @@ public class FileStatus implements Writa
modification_time = in.readLong();
access_time = in.readLong();
permission.readFields(in);
- owner = Text.readString(in, Text.ONE_MEGABYTE);
- group = Text.readString(in, Text.ONE_MEGABYTE);
+ owner = Text.readString(in, Text.DEFAULT_MAX_LEN);
+ group = Text.readString(in, Text.DEFAULT_MAX_LEN);
if (in.readBoolean()) {
- this.symlink = new Path(Text.readString(in, Text.ONE_MEGABYTE));
+ this.symlink = new Path(Text.readString(in, Text.DEFAULT_MAX_LEN));
} else {
this.symlink = null;
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java Thu Jun 28 06:59:38 2012
@@ -84,8 +84,8 @@ public class PermissionStatus implements
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
- username = Text.readString(in, Text.ONE_MEGABYTE);
- groupname = Text.readString(in, Text.ONE_MEGABYTE);
+ username = Text.readString(in, Text.DEFAULT_MAX_LEN);
+ groupname = Text.readString(in, Text.DEFAULT_MAX_LEN);
permission = FsPermission.read(in);
}
@@ -110,8 +110,8 @@ public class PermissionStatus implements
String username,
String groupname,
FsPermission permission) throws IOException {
- Text.writeString(out, username, Text.ONE_MEGABYTE);
- Text.writeString(out, groupname, Text.ONE_MEGABYTE);
+ Text.writeString(out, username, Text.DEFAULT_MAX_LEN);
+ Text.writeString(out, groupname, Text.DEFAULT_MAX_LEN);
permission.write(out);
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java Thu Jun 28 06:59:38 2012
@@ -52,7 +52,9 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.jmx.JMXJsonServlet;
import org.apache.hadoop.log.LogLevel;
import org.apache.hadoop.metrics.MetricsServlet;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.ReflectionUtils;
import org.mortbay.io.Buffer;
@@ -606,6 +608,24 @@ public class HttpServer implements Filte
sslListener.setNeedClientAuth(needCertsAuth);
webServer.addConnector(sslListener);
}
+
+ protected void initSpnego(Configuration conf,
+ String usernameConfKey, String keytabConfKey) throws IOException {
+ Map<String, String> params = new HashMap<String, String>();
+ String principalInConf = conf.get(usernameConfKey);
+ if (principalInConf != null && !principalInConf.isEmpty()) {
+ params.put("kerberos.principal",
+ SecurityUtil.getServerPrincipal(principalInConf, listener.getHost()));
+ }
+ String httpKeytab = conf.get(keytabConfKey);
+ if (httpKeytab != null && !httpKeytab.isEmpty()) {
+ params.put("kerberos.keytab", httpKeytab);
+ }
+ params.put(AuthenticationFilter.AUTH_TYPE, "kerberos");
+
+ defineFilter(webAppContext, SPNEGO_FILTER,
+ AuthenticationFilter.class.getName(), params, null);
+ }
/**
* Start the server. Does not wait for the server to start.
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java Thu Jun 28 06:59:38 2012
@@ -20,6 +20,9 @@ package org.apache.hadoop.io;
import java.io.*;
import java.net.Socket;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
import org.apache.commons.logging.Log;
@@ -245,4 +248,34 @@ public class IOUtils {
public void write(int b) throws IOException {
}
}
+
+ /**
+ * Write a ByteBuffer to a WritableByteChannel, handling short writes.
+ *
+ * @param bc The WritableByteChannel to write to
+ * @param buf The input buffer
+ * @throws IOException On I/O error
+ */
+ public static void writeFully(WritableByteChannel bc, ByteBuffer buf)
+ throws IOException {
+ do {
+ bc.write(buf);
+ } while (buf.remaining() > 0);
+ }
+
+ /**
+ * Write a ByteBuffer to a FileChannel at a given offset,
+ * handling short writes.
+ *
+ * @param fc The FileChannel to write to
+ * @param buf The input buffer
+ * @param offset The offset in the file to start writing at
+ * @throws IOException On I/O error
+ */
+ public static void writeFully(FileChannel fc, ByteBuffer buf,
+ long offset) throws IOException {
+ do {
+ offset += fc.write(buf, offset);
+ } while (buf.remaining() > 0);
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java Thu Jun 28 06:59:38 2012
@@ -287,6 +287,20 @@ public class Text extends BinaryComparab
in.readFully(bytes, 0, newLength);
length = newLength;
}
+
+ public void readFields(DataInput in, int maxLength) throws IOException {
+ int newLength = WritableUtils.readVInt(in);
+ if (newLength < 0) {
+ throw new IOException("tried to deserialize " + newLength +
+ " bytes of data! newLength must be non-negative.");
+ } else if (newLength >= maxLength) {
+ throw new IOException("tried to deserialize " + newLength +
+ " bytes of data, but maxLength = " + maxLength);
+ }
+ setCapacity(newLength, false);
+ in.readFully(bytes, 0, newLength);
+ length = newLength;
+ }
/** Skips over one Text in the input. */
public static void skip(DataInput in) throws IOException {
@@ -304,6 +318,16 @@ public class Text extends BinaryComparab
out.write(bytes, 0, length);
}
+ public void write(DataOutput out, int maxLength) throws IOException {
+ if (length > maxLength) {
+ throw new IOException("data was too long to write! Expected " +
+ "less than or equal to " + maxLength + " bytes, but got " +
+ length + " bytes.");
+ }
+ WritableUtils.writeVInt(out, length);
+ out.write(bytes, 0, length);
+ }
+
/** Returns true iff <code>o</code> is a Text with the same contents. */
public boolean equals(Object o) {
if (o instanceof Text)
@@ -417,7 +441,7 @@ public class Text extends BinaryComparab
return bytes;
}
- static final public int ONE_MEGABYTE = 1024 * 1024;
+ static final public int DEFAULT_MAX_LEN = 1024 * 1024;
/** Read a UTF8 encoded string from in
*/
@@ -432,7 +456,7 @@ public class Text extends BinaryComparab
*/
public static String readString(DataInput in, int maxLength)
throws IOException {
- int length = WritableUtils.readVIntInRange(in, 0, maxLength - 1);
+ int length = WritableUtils.readVIntInRange(in, 0, maxLength);
byte [] bytes = new byte[length];
in.readFully(bytes, 0, length);
return decode(bytes);
@@ -454,9 +478,9 @@ public class Text extends BinaryComparab
throws IOException {
ByteBuffer bytes = encode(s);
int length = bytes.limit();
- if (length >= maxLength) {
+ if (length > maxLength) {
throw new IOException("string was too long to write! Expected " +
- "less than " + maxLength + " bytes, but got " +
+ "less than or equal to " + maxLength + " bytes, but got " +
length + " bytes.");
}
WritableUtils.writeVInt(out, length);
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java Thu Jun 28 06:59:38 2012
@@ -33,7 +33,7 @@ import org.apache.hadoop.ipc.RpcInvocati
class RetryInvocationHandler implements RpcInvocationHandler {
public static final Log LOG = LogFactory.getLog(RetryInvocationHandler.class);
- private FailoverProxyProvider proxyProvider;
+ private final FailoverProxyProvider proxyProvider;
/**
* The number of times the associated proxyProvider has ever been failed over.
@@ -41,26 +41,25 @@ class RetryInvocationHandler implements
private long proxyProviderFailoverCount = 0;
private volatile boolean hasMadeASuccessfulCall = false;
- private RetryPolicy defaultPolicy;
- private Map<String,RetryPolicy> methodNameToPolicyMap;
+ private final RetryPolicy defaultPolicy;
+ private final Map<String,RetryPolicy> methodNameToPolicyMap;
private Object currentProxy;
public RetryInvocationHandler(FailoverProxyProvider proxyProvider,
RetryPolicy retryPolicy) {
- this.proxyProvider = proxyProvider;
- this.defaultPolicy = retryPolicy;
- this.methodNameToPolicyMap = Collections.emptyMap();
- this.currentProxy = proxyProvider.getProxy();
+ this(proxyProvider, retryPolicy, Collections.<String, RetryPolicy>emptyMap());
}
-
+
public RetryInvocationHandler(FailoverProxyProvider proxyProvider,
+ RetryPolicy defaultPolicy,
Map<String, RetryPolicy> methodNameToPolicyMap) {
this.proxyProvider = proxyProvider;
- this.defaultPolicy = RetryPolicies.TRY_ONCE_THEN_FAIL;
+ this.defaultPolicy = defaultPolicy;
this.methodNameToPolicyMap = methodNameToPolicyMap;
this.currentProxy = proxyProvider.getProxy();
}
+ @Override
public Object invoke(Object proxy, Method method, Object[] args)
throws Throwable {
RetryPolicy policy = methodNameToPolicyMap.get(method.getName());
Modified: hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java Thu Jun 28 06:59:38 2012
@@ -22,10 +22,13 @@ import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.SocketException;
import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Map.Entry;
+import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
@@ -33,8 +36,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.StandbyException;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* <p>
* A collection of useful implementations of {@link RetryPolicy}.
@@ -44,7 +45,12 @@ public class RetryPolicies {
public static final Log LOG = LogFactory.getLog(RetryPolicies.class);
- private static final Random RAND = new Random();
+ private static ThreadLocal<Random> RANDOM = new ThreadLocal<Random>() {
+ @Override
+ protected Random initialValue() {
+ return new Random();
+ }
+ };
/**
* <p>
@@ -157,17 +163,35 @@ public class RetryPolicies {
}
}
+ /**
+ * Retry up to maxRetries.
+ * The actual sleep time of the n-th retry is f(n, sleepTime),
+ * where f is a function provided by the subclass implementation.
+ *
+ * The object of the subclasses should be immutable;
+ * otherwise, the subclass must override hashCode(), equals(..) and toString().
+ */
static abstract class RetryLimited implements RetryPolicy {
- int maxRetries;
- long sleepTime;
- TimeUnit timeUnit;
-
- public RetryLimited(int maxRetries, long sleepTime, TimeUnit timeUnit) {
+ final int maxRetries;
+ final long sleepTime;
+ final TimeUnit timeUnit;
+
+ private String myString;
+
+ RetryLimited(int maxRetries, long sleepTime, TimeUnit timeUnit) {
+ if (maxRetries < 0) {
+ throw new IllegalArgumentException("maxRetries = " + maxRetries+" < 0");
+ }
+ if (sleepTime < 0) {
+ throw new IllegalArgumentException("sleepTime = " + sleepTime + " < 0");
+ }
+
this.maxRetries = maxRetries;
this.sleepTime = sleepTime;
this.timeUnit = timeUnit;
}
+ @Override
public RetryAction shouldRetry(Exception e, int retries, int failovers,
boolean isMethodIdempotent) throws Exception {
if (retries >= maxRetries) {
@@ -178,6 +202,30 @@ public class RetryPolicies {
}
protected abstract long calculateSleepTime(int retries);
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ @Override
+ public boolean equals(final Object that) {
+ if (this == that) {
+ return true;
+ } else if (that == null || this.getClass() != that.getClass()) {
+ return false;
+ }
+ return this.toString().equals(that.toString());
+ }
+
+ @Override
+ public String toString() {
+ if (myString == null) {
+ myString = getClass().getSimpleName() + "(maxRetries=" + maxRetries
+ + ", sleepTime=" + sleepTime + " " + timeUnit + ")";
+ }
+ return myString;
+ }
}
static class RetryUpToMaximumCountWithFixedSleep extends RetryLimited {
@@ -208,6 +256,169 @@ public class RetryPolicies {
}
}
+ /**
+ * Given pairs of number of retries and sleep time (n0, t0), (n1, t1), ...,
+ * the first n0 retries sleep t0 milliseconds on average,
+ * the following n1 retries sleep t1 milliseconds on average, and so on.
+ *
+ * For all the sleep, the actual sleep time is randomly uniform distributed
+ * in the close interval [0.5t, 1.5t], where t is the sleep time specified.
+ *
+ * The objects of this class are immutable.
+ */
+ public static class MultipleLinearRandomRetry implements RetryPolicy {
+ /** Pairs of numRetries and sleepSeconds */
+ public static class Pair {
+ final int numRetries;
+ final int sleepMillis;
+
+ public Pair(final int numRetries, final int sleepMillis) {
+ if (numRetries < 0) {
+ throw new IllegalArgumentException("numRetries = " + numRetries+" < 0");
+ }
+ if (sleepMillis < 0) {
+ throw new IllegalArgumentException("sleepMillis = " + sleepMillis + " < 0");
+ }
+
+ this.numRetries = numRetries;
+ this.sleepMillis = sleepMillis;
+ }
+
+ @Override
+ public String toString() {
+ return numRetries + "x" + sleepMillis + "ms";
+ }
+ }
+
+ private final List<Pair> pairs;
+ private String myString;
+
+ public MultipleLinearRandomRetry(List<Pair> pairs) {
+ if (pairs == null || pairs.isEmpty()) {
+ throw new IllegalArgumentException("pairs must be neither null nor empty.");
+ }
+ this.pairs = Collections.unmodifiableList(pairs);
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception e, int curRetry, int failovers,
+ boolean isMethodIdempotent) throws Exception {
+ final Pair p = searchPair(curRetry);
+ if (p == null) {
+ //no more retries.
+ return RetryAction.FAIL;
+ }
+
+ //calculate sleep time and return.
+ final double ratio = RANDOM.get().nextDouble() + 0.5;//0.5 <= ratio <=1.5
+ final long sleepTime = Math.round(p.sleepMillis * ratio);
+ return new RetryAction(RetryAction.RetryDecision.RETRY, sleepTime);
+ }
+
+ /**
+ * Given the current number of retry, search the corresponding pair.
+ * @return the corresponding pair,
+ * or null if the current number of retry > maximum number of retry.
+ */
+ private Pair searchPair(int curRetry) {
+ int i = 0;
+ for(; i < pairs.size() && curRetry > pairs.get(i).numRetries; i++) {
+ curRetry -= pairs.get(i).numRetries;
+ }
+ return i == pairs.size()? null: pairs.get(i);
+ }
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ @Override
+ public boolean equals(final Object that) {
+ if (this == that) {
+ return true;
+ } else if (that == null || this.getClass() != that.getClass()) {
+ return false;
+ }
+ return this.toString().equals(that.toString());
+ }
+
+ @Override
+ public String toString() {
+ if (myString == null) {
+ myString = getClass().getSimpleName() + pairs;
+ }
+ return myString;
+ }
+
+ /**
+ * Parse the given string as a MultipleLinearRandomRetry object.
+ * The format of the string is "t_1, n_1, t_2, n_2, ...",
+ * where t_i and n_i are the i-th pair of sleep time and number of retires.
+ * Note that the white spaces in the string are ignored.
+ *
+ * @return the parsed object, or null if the parsing fails.
+ */
+ public static MultipleLinearRandomRetry parseCommaSeparatedString(String s) {
+ final String[] elements = s.split(",");
+ if (elements.length == 0) {
+ LOG.warn("Illegal value: there is no element in \"" + s + "\".");
+ return null;
+ }
+ if (elements.length % 2 != 0) {
+ LOG.warn("Illegal value: the number of elements in \"" + s + "\" is "
+ + elements.length + " but an even number of elements is expected.");
+ return null;
+ }
+
+ final List<RetryPolicies.MultipleLinearRandomRetry.Pair> pairs
+ = new ArrayList<RetryPolicies.MultipleLinearRandomRetry.Pair>();
+
+ for(int i = 0; i < elements.length; ) {
+ //parse the i-th sleep-time
+ final int sleep = parsePositiveInt(elements, i++, s);
+ if (sleep == -1) {
+ return null; //parse fails
+ }
+
+ //parse the i-th number-of-retries
+ final int retries = parsePositiveInt(elements, i++, s);
+ if (retries == -1) {
+ return null; //parse fails
+ }
+
+ pairs.add(new RetryPolicies.MultipleLinearRandomRetry.Pair(retries, sleep));
+ }
+ return new RetryPolicies.MultipleLinearRandomRetry(pairs);
+ }
+
+ /**
+ * Parse the i-th element as an integer.
+ * @return -1 if the parsing fails or the parsed value <= 0;
+ * otherwise, return the parsed value.
+ */
+ private static int parsePositiveInt(final String[] elements,
+ final int i, final String originalString) {
+ final String s = elements[i].trim();
+ final int n;
+ try {
+ n = Integer.parseInt(s);
+ } catch(NumberFormatException nfe) {
+ LOG.warn("Failed to parse \"" + s + "\", which is the index " + i
+ + " element in \"" + originalString + "\"", nfe);
+ return -1;
+ }
+
+ if (n <= 0) {
+ LOG.warn("The value " + n + " <= 0: it is parsed from the string \""
+ + s + "\" which is the index " + i + " element in \""
+ + originalString + "\"");
+ return -1;
+ }
+ return n;
+ }
+ }
+
static class ExceptionDependentRetry implements RetryPolicy {
RetryPolicy defaultPolicy;
@@ -265,6 +476,14 @@ public class RetryPolicies {
public ExponentialBackoffRetry(
int maxRetries, long sleepTime, TimeUnit timeUnit) {
super(maxRetries, sleepTime, timeUnit);
+
+ if (maxRetries < 0) {
+ throw new IllegalArgumentException("maxRetries = " + maxRetries + " < 0");
+ } else if (maxRetries >= Long.SIZE - 1) {
+ //calculateSleepTime may overflow.
+ throw new IllegalArgumentException("maxRetries = " + maxRetries
+ + " >= " + (Long.SIZE - 1));
+ }
}
@Override
@@ -353,11 +572,10 @@ public class RetryPolicies {
* @param cap value at which to cap the base sleep time
* @return an amount of time to sleep
*/
- @VisibleForTesting
- public static long calculateExponentialTime(long time, int retries,
+ private static long calculateExponentialTime(long time, int retries,
long cap) {
- long baseTime = Math.min(time * ((long)1 << retries), cap);
- return (long) (baseTime * (RAND.nextFloat() + 0.5));
+ long baseTime = Math.min(time * (1L << retries), cap);
+ return (long) (baseTime * (RANDOM.get().nextDouble() + 0.5));
}
private static long calculateExponentialTime(long time, int retries) {