You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2012/11/13 21:21:44 UTC
svn commit: r1408938 - in
/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common: ./
src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/fs/permission/
src/main/java/org/apache/hadoop/ipc/
src/main/java/org/apache/hadoop/securi...
Author: szetszwo
Date: Tue Nov 13 20:21:39 2012
New Revision: 1408938
URL: http://svn.apache.org/viewvc?rev=1408938&view=rev
Log:
Merge r1407704 through r1408926 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/site/resources/
- copied from r1408926, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/resources/
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/site/resources/css/
- copied from r1408926, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/resources/css/
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/site/resources/css/site.css
- copied unchanged from r1408926, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/resources/css/site.css
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/site/site.xml
- copied unchanged from r1408926, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/site.xml
Modified:
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1408938&r1=1408937&r2=1408938&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt Tue Nov 13 20:21:39 2012
@@ -361,6 +361,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8860. Split MapReduce and YARN sections in documentation navigation.
(tomwhite via tucu)
+ HADOOP-9021. Enforce configured SASL method on the server (daryn via
+ bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
@@ -421,6 +424,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-7115. Add a cache for getpwuid_r and getpwgid_r calls (tucu)
+ HADOOP-8999. SASL negotiation is flawed (daryn)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -1140,6 +1145,12 @@ Release 0.23.5 - UNRELEASED
HADOOP-8986. Server$Call object is never released after it is sent (bobby)
+ HADOOP-9022. Hadoop distcp tool fails to copy file if -m 0 specified
+ (Jonathan Eagles vai bobby)
+
+ HADOOP-9025. org.apache.hadoop.tools.TestCopyListing failing (Jonathan
+ Eagles via jlowe)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1407704-1408926
Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1407704-1408926
Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1407704-1408926
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=1408938&r1=1408937&r2=1408938&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java Tue Nov 13 20:21:39 2012
@@ -184,7 +184,18 @@ public class FsPermission implements Wri
return str;
}
- /** Apply a umask to this permission and return a new one */
+ /**
+ * Apply a umask to this permission and return a new one.
+ *
+ * The umask is used by create, mkdir, and other Hadoop filesystem operations.
+ * The mode argument for these operations is modified by removing the bits
+ * which are set in the umask. Thus, the umask limits the permissions which
+ * newly created files and directories get.
+ *
+ * @param umask The umask to use
+ *
+ * @return The effective permission
+ */
public FsPermission applyUMask(FsPermission umask) {
return new FsPermission(useraction.and(umask.useraction.not()),
groupaction.and(umask.groupaction.not()),
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1408938&r1=1408937&r2=1408938&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Tue Nov 13 20:21:39 2012
@@ -45,6 +45,7 @@ import java.security.PrivilegedException
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -87,7 +88,9 @@ import org.apache.hadoop.security.SaslRp
import org.apache.hadoop.security.SaslRpcServer.SaslDigestCallbackHandler;
import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler;
import org.apache.hadoop.security.SaslRpcServer.SaslStatus;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.authorize.PolicyProvider;
@@ -113,7 +116,7 @@ import com.google.common.annotations.Vis
@InterfaceStability.Evolving
public abstract class Server {
private final boolean authorize;
- private boolean isSecurityEnabled;
+ private EnumSet<AuthMethod> enabledAuthMethods;
private ExceptionsHandler exceptionsHandler = new ExceptionsHandler();
public void addTerseExceptions(Class<?>... exceptionClass) {
@@ -1217,6 +1220,10 @@ public abstract class Server {
AUDITLOG.warn(AUTH_FAILED_FOR + clientIP + ":" + attemptingUser);
throw e;
}
+ if (replyToken == null && authMethod == AuthMethod.PLAIN) {
+ // client needs at least response to know if it should use SIMPLE
+ replyToken = new byte[0];
+ }
if (replyToken != null) {
if (LOG.isDebugEnabled())
LOG.debug("Will send token of size " + replyToken.length
@@ -1334,34 +1341,9 @@ public abstract class Server {
if (authMethod == null) {
throw new IOException("Unable to read authentication method");
}
- boolean useSaslServer = isSecurityEnabled;
- final boolean clientUsingSasl;
- switch (authMethod) {
- case SIMPLE: { // no sasl for simple
- clientUsingSasl = false;
- break;
- }
- case DIGEST: { // always allow tokens if there's a secret manager
- useSaslServer |= (secretManager != null);
- clientUsingSasl = true;
- break;
- }
- default: {
- clientUsingSasl = true;
- break;
- }
- }
- if (useSaslServer) {
- saslServer = createSaslServer(authMethod);
- } else if (clientUsingSasl) { // security is off
- doSaslReply(SaslStatus.SUCCESS, new IntWritable(
- SaslRpcServer.SWITCH_TO_SIMPLE_AUTH), null, null);
- authMethod = AuthMethod.SIMPLE;
- // client has already sent the initial Sasl message and we
- // should ignore it. Both client and server should fall back
- // to simple auth from now on.
- skipInitialSaslHandshake = true;
- }
+
+ // this may create a SASL server, or switch us into SIMPLE
+ authMethod = initializeAuthContext(authMethod);
connectionHeaderBuf = null;
connectionHeaderRead = true;
@@ -1409,10 +1391,24 @@ public abstract class Server {
}
}
- private SaslServer createSaslServer(AuthMethod authMethod)
+ private AuthMethod initializeAuthContext(AuthMethod authMethod)
throws IOException {
try {
- return createSaslServerInternal(authMethod);
+ if (enabledAuthMethods.contains(authMethod)) {
+ saslServer = createSaslServer(authMethod);
+ } else if (enabledAuthMethods.contains(AuthMethod.SIMPLE)) {
+ doSaslReply(SaslStatus.SUCCESS, new IntWritable(
+ SaslRpcServer.SWITCH_TO_SIMPLE_AUTH), null, null);
+ authMethod = AuthMethod.SIMPLE;
+ // client has already sent the initial Sasl message and we
+ // should ignore it. Both client and server should fall back
+ // to simple auth from now on.
+ skipInitialSaslHandshake = true;
+ } else {
+ throw new AccessControlException(
+ authMethod + " authentication is not enabled."
+ + " Available:" + enabledAuthMethods);
+ }
} catch (IOException ioe) {
final String ioeClass = ioe.getClass().getName();
final String ioeMessage = ioe.getLocalizedMessage();
@@ -1425,9 +1421,10 @@ public abstract class Server {
}
throw ioe;
}
+ return authMethod;
}
- private SaslServer createSaslServerInternal(AuthMethod authMethod)
+ private SaslServer createSaslServer(AuthMethod authMethod)
throws IOException {
SaslServer saslServer = null;
String hostname = null;
@@ -1436,18 +1433,9 @@ public abstract class Server {
switch (authMethod) {
case SIMPLE: {
- throw new AccessControlException("Authorization ("
- + CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION
- + ") is enabled but authentication ("
- + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION
- + ") is configured as simple. Please configure another method "
- + "like kerberos or digest.");
+ return null; // no sasl for simple
}
case DIGEST: {
- if (secretManager == null) {
- throw new AccessControlException(
- "Server is not configured to do DIGEST authentication.");
- }
secretManager.checkAvailableForRead();
hostname = SaslRpcServer.SASL_DEFAULT_REALM;
saslCallback = new SaslDigestCallbackHandler(secretManager, this);
@@ -1469,6 +1457,7 @@ public abstract class Server {
break;
}
default:
+ // we should never be able to get here
throw new AccessControlException(
"Server does not support SASL " + authMethod);
}
@@ -1908,7 +1897,9 @@ public abstract class Server {
this.authorize =
conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
false);
- this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
+
+ // configure supported authentications
+ this.enabledAuthMethods = getAuthMethods(secretManager, conf);
// Start the listener here and let it bind to the port
listener = new Listener();
@@ -1929,6 +1920,31 @@ public abstract class Server {
this.exceptionsHandler.addTerseExceptions(StandbyException.class);
}
+ // get the security type from the conf. implicitly include token support
+ // if a secret manager is provided, or fail if token is the conf value but
+ // there is no secret manager
+ private EnumSet<AuthMethod> getAuthMethods(SecretManager<?> secretManager,
+ Configuration conf) {
+ AuthenticationMethod confAuthenticationMethod =
+ SecurityUtil.getAuthenticationMethod(conf);
+ EnumSet<AuthMethod> authMethods =
+ EnumSet.of(confAuthenticationMethod.getAuthMethod());
+
+ if (confAuthenticationMethod == AuthenticationMethod.TOKEN) {
+ if (secretManager == null) {
+ throw new IllegalArgumentException(AuthenticationMethod.TOKEN +
+ " authentication requires a secret manager");
+ }
+ } else if (secretManager != null) {
+ LOG.debug(AuthenticationMethod.TOKEN +
+ " authentication enabled for secret manager");
+ authMethods.add(AuthenticationMethod.TOKEN.getAuthMethod());
+ }
+
+ LOG.debug("Server accepts auth methods:" + authMethods);
+ return authMethods;
+ }
+
private void closeConnection(Connection connection) {
synchronized (connectionList) {
if (connectionList.remove(connection))
@@ -2045,16 +2061,6 @@ public abstract class Server {
return conf;
}
- /** for unit testing only, should be called before server is started */
- void disableSecurity() {
- this.isSecurityEnabled = false;
- }
-
- /** for unit testing only, should be called before server is started */
- void enableSecurity() {
- this.isSecurityEnabled = true;
- }
-
/** Sets the socket buffer size used for responding to RPCs */
public void setSocketSendBufSize(int size) { this.socketSendBufferSize = size; }
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java?rev=1408938&r1=1408937&r2=1408938&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java Tue Nov 13 20:21:39 2012
@@ -145,15 +145,13 @@ public class SaslRpcClient {
byte[] saslToken = new byte[0];
if (saslClient.hasInitialResponse())
saslToken = saslClient.evaluateChallenge(saslToken);
- if (saslToken != null) {
+ while (saslToken != null) {
outStream.writeInt(saslToken.length);
outStream.write(saslToken, 0, saslToken.length);
outStream.flush();
if (LOG.isDebugEnabled())
LOG.debug("Have sent token of size " + saslToken.length
+ " from initSASLContext.");
- }
- if (!saslClient.isComplete()) {
readStatus(inStream);
int len = inStream.readInt();
if (len == SaslRpcServer.SWITCH_TO_SIMPLE_AUTH) {
@@ -161,32 +159,18 @@ public class SaslRpcClient {
LOG.debug("Server asks us to fall back to simple auth.");
saslClient.dispose();
return false;
+ } else if ((len == 0) && saslClient.isComplete()) {
+ break;
}
saslToken = new byte[len];
if (LOG.isDebugEnabled())
LOG.debug("Will read input token of size " + saslToken.length
+ " for processing by initSASLContext");
inStream.readFully(saslToken);
- }
-
- while (!saslClient.isComplete()) {
saslToken = saslClient.evaluateChallenge(saslToken);
- if (saslToken != null) {
- if (LOG.isDebugEnabled())
- LOG.debug("Will send token of size " + saslToken.length
- + " from initSASLContext.");
- outStream.writeInt(saslToken.length);
- outStream.write(saslToken, 0, saslToken.length);
- outStream.flush();
- }
- if (!saslClient.isComplete()) {
- readStatus(inStream);
- saslToken = new byte[inStream.readInt()];
- if (LOG.isDebugEnabled())
- LOG.debug("Will read input token of size " + saslToken.length
- + " for processing by initSASLContext");
- inStream.readFully(saslToken);
- }
+ }
+ if (!saslClient.isComplete()) { // shouldn't happen
+ throw new SaslException("Internal negotiation error");
}
if (LOG.isDebugEnabled()) {
LOG.debug("SASL client context established. Negotiated QoP: "
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1408938&r1=1408937&r2=1408938&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java Tue Nov 13 20:21:39 2012
@@ -240,6 +240,7 @@ public class UserGroupInformation {
AuthenticationMethod auth = SecurityUtil.getAuthenticationMethod(conf);
switch (auth) {
case SIMPLE:
+ case TOKEN:
useKerberos = false;
break;
case KERBEROS:
Propchange: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1407704-1408926
Modified: hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java?rev=1408938&r1=1408937&r2=1408938&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java Tue Nov 13 20:21:39 2012
@@ -569,10 +569,13 @@ public class TestSaslRPC {
private static Pattern KrbFailed =
Pattern.compile(".*Failed on local exception:.* " +
"Failed to specify server's Kerberos principal name.*");
- private static Pattern Denied =
- Pattern.compile(".*Authorization .* is enabled .*");
- private static Pattern NoDigest =
- Pattern.compile(".*Server is not configured to do DIGEST auth.*");
+ private static Pattern Denied(AuthenticationMethod method) {
+ return Pattern.compile(".*RemoteException.*AccessControlException.*: "
+ +method.getAuthMethod() + " authentication is not enabled.*");
+ }
+ private static Pattern NoTokenAuth =
+ Pattern.compile(".*IllegalArgumentException: " +
+ "TOKEN authentication requires a secret manager");
/*
* simple server
@@ -605,12 +608,39 @@ public class TestSaslRPC {
}
/*
+ * token server
+ */
+ @Test
+ public void testTokenOnlyServer() throws Exception {
+ assertAuthEquals(Denied(SIMPLE), getAuthMethod(SIMPLE, TOKEN));
+ assertAuthEquals(KrbFailed, getAuthMethod(KERBEROS, TOKEN));
+ }
+
+ @Test
+ public void testTokenOnlyServerWithTokens() throws Exception {
+ assertAuthEquals(TOKEN, getAuthMethod(SIMPLE, TOKEN, true));
+ assertAuthEquals(TOKEN, getAuthMethod(KERBEROS, TOKEN, true));
+ forceSecretManager = false;
+ assertAuthEquals(NoTokenAuth, getAuthMethod(SIMPLE, TOKEN, true));
+ assertAuthEquals(NoTokenAuth, getAuthMethod(KERBEROS, TOKEN, true));
+ }
+
+ @Test
+ public void testTokenOnlyServerWithInvalidTokens() throws Exception {
+ assertAuthEquals(BadToken, getAuthMethod(SIMPLE, TOKEN, false));
+ assertAuthEquals(BadToken, getAuthMethod(KERBEROS, TOKEN, false));
+ forceSecretManager = false;
+ assertAuthEquals(NoTokenAuth, getAuthMethod(SIMPLE, TOKEN, false));
+ assertAuthEquals(NoTokenAuth, getAuthMethod(KERBEROS, TOKEN, false));
+ }
+
+ /*
* kerberos server
*/
@Test
public void testKerberosServer() throws Exception {
- assertAuthEquals(Denied, getAuthMethod(SIMPLE, KERBEROS));
- assertAuthEquals(KrbFailed, getAuthMethod(KERBEROS, KERBEROS));
+ assertAuthEquals(Denied(SIMPLE), getAuthMethod(SIMPLE, KERBEROS));
+ assertAuthEquals(KrbFailed, getAuthMethod(KERBEROS, KERBEROS));
}
@Test
@@ -620,8 +650,8 @@ public class TestSaslRPC {
assertAuthEquals(TOKEN, getAuthMethod(KERBEROS, KERBEROS, true));
// can't fallback to simple when using kerberos w/o tokens
forceSecretManager = false;
- assertAuthEquals(NoDigest, getAuthMethod(SIMPLE, KERBEROS, true));
- assertAuthEquals(NoDigest, getAuthMethod(KERBEROS, KERBEROS, true));
+ assertAuthEquals(Denied(TOKEN), getAuthMethod(SIMPLE, KERBEROS, true));
+ assertAuthEquals(Denied(TOKEN), getAuthMethod(KERBEROS, KERBEROS, true));
}
@Test
@@ -629,8 +659,8 @@ public class TestSaslRPC {
assertAuthEquals(BadToken, getAuthMethod(SIMPLE, KERBEROS, false));
assertAuthEquals(BadToken, getAuthMethod(KERBEROS, KERBEROS, false));
forceSecretManager = false;
- assertAuthEquals(NoDigest, getAuthMethod(SIMPLE, KERBEROS, true));
- assertAuthEquals(NoDigest, getAuthMethod(KERBEROS, KERBEROS, true));
+ assertAuthEquals(Denied(TOKEN), getAuthMethod(SIMPLE, KERBEROS, false));
+ assertAuthEquals(Denied(TOKEN), getAuthMethod(KERBEROS, KERBEROS, false));
}