You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/12/09 23:52:05 UTC
svn commit: r1549699 - in
/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common: ./
src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/security/
src/main/java/org/apache/hadoop/util/ src/ma...
Author: cnauroth
Date: Mon Dec 9 22:52:02 2013
New Revision: 1549699
URL: http://svn.apache.org/r1549699
Log:
Merge trunk to HDFS-4685.
Modified:
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1549699&r1=1549698&r2=1549699&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt Mon Dec 9 22:52:02 2013
@@ -487,6 +487,9 @@ Release 2.3.0 - UNRELEASED
OPTIMIZATIONS
+ HADOOP-10142. Avoid groups lookup for unprivileged users such as "dr.who"
+ (vinay via cmccabe)
+
BUG FIXES
HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9)
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1548386-1549698
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1548386-1549698
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1548386-1549698
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1549699&r1=1549698&r2=1549699&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Mon Dec 9 22:52:02 2013
@@ -204,6 +204,14 @@ public class CommonConfigurationKeys ext
public static final String DEFAULT_HADOOP_HTTP_STATIC_USER =
"dr.who";
+ /**
+ * User->groups static mapping to override the groups lookup
+ */
+ public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES =
+ "hadoop.user.group.static.mapping.overrides";
+ public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT =
+ "dr.who=;";
+
/** Enable/Disable aliases serving from jetty */
public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES =
"hadoop.jetty.logs.serve.aliases";
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java?rev=1549699&r1=1549698&r2=1549699&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java Mon Dec 9 22:52:02 2013
@@ -18,15 +18,20 @@
package org.apache.hadoop.security;
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.commons.logging.Log;
@@ -49,6 +54,8 @@ public class Groups {
private final Map<String, CachedGroups> userToGroupsMap =
new ConcurrentHashMap<String, CachedGroups>();
+ private final Map<String, List<String>> staticUserToGroupsMap =
+ new HashMap<String, List<String>>();
private final long cacheTimeout;
private final long warningDeltaMs;
@@ -66,12 +73,43 @@ public class Groups {
warningDeltaMs =
conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS,
CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT);
-
+ parseStaticMapping(conf);
+
if(LOG.isDebugEnabled())
LOG.debug("Group mapping impl=" + impl.getClass().getName() +
"; cacheTimeout=" + cacheTimeout + "; warningDeltaMs=" +
warningDeltaMs);
}
+
+ /*
+ * Parse the hadoop.user.group.static.mapping.overrides configuration to
+ * staticUserToGroupsMap
+ */
+ private void parseStaticMapping(Configuration conf) {
+ String staticMapping = conf.get(
+ CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
+ CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT);
+ Collection<String> mappings = StringUtils.getStringCollection(
+ staticMapping, ";");
+ for (String users : mappings) {
+ Collection<String> userToGroups = StringUtils.getStringCollection(users,
+ "=");
+ if (userToGroups.size() < 1 || userToGroups.size() > 2) {
+ throw new HadoopIllegalArgumentException("Configuration "
+ + CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES
+ + " is invalid");
+ }
+ String[] userToGroupsArray = userToGroups.toArray(new String[userToGroups
+ .size()]);
+ String user = userToGroupsArray[0];
+ List<String> groups = Collections.emptyList();
+ if (userToGroupsArray.length == 2) {
+ groups = (List<String>) StringUtils
+ .getStringCollection(userToGroupsArray[1]);
+ }
+ staticUserToGroupsMap.put(user, groups);
+ }
+ }
/**
* Get the group memberships of a given user.
@@ -80,6 +118,11 @@ public class Groups {
* @throws IOException
*/
public List<String> getGroups(String user) throws IOException {
+ // No need to lookup for groups of static users
+ List<String> staticMapping = staticUserToGroupsMap.get(user);
+ if (staticMapping != null) {
+ return staticMapping;
+ }
// Return cached value if available
CachedGroups groups = userToGroupsMap.get(user);
long startMs = Time.monotonicNow();
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1549699&r1=1549698&r2=1549699&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java Mon Dec 9 22:52:02 2013
@@ -325,10 +325,24 @@ public class StringUtils {
* @return an <code>ArrayList</code> of string values
*/
public static Collection<String> getStringCollection(String str){
+ String delim = ",";
+ return getStringCollection(str, delim);
+ }
+
+ /**
+ * Returns a collection of strings.
+ *
+ * @param str
+ * String to parse
+ * @param delim
+ * delimiter to separate the values
+ * @return Collection of parsed elements.
+ */
+ public static Collection<String> getStringCollection(String str, String delim) {
List<String> values = new ArrayList<String>();
if (str == null)
return values;
- StringTokenizer tokenizer = new StringTokenizer (str,",");
+ StringTokenizer tokenizer = new StringTokenizer(str, delim);
values = new ArrayList<String>();
while (tokenizer.hasMoreTokens()) {
values.add(tokenizer.nextToken());
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1549699&r1=1549698&r2=1549699&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Mon Dec 9 22:52:02 2013
@@ -1261,4 +1261,18 @@
Specify the port number used by Hadoop mount daemon.
</description>
</property>
+
+<property>
+ <name>hadoop.user.group.static.mapping.overrides</name>
+ <value>dr.who=;</value>
+ <description>
+ Static mapping of user to groups. This will override the groups if
+ available in the system for the specified user. In otherwords, groups
+ look-up will not happen for these users, instead groups mapped in this
+ configuration will be used.
+ Mapping should be in this format.
+ user1=group1,group2;user2=;user3=group2;
+ Default, "dr.who=;" will consider "dr.who" as user without groups.
+ </description>
+</property>
</configuration>
Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1548386-1549698
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java?rev=1549699&r1=1549698&r2=1549699&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java Mon Dec 9 22:52:02 2013
@@ -19,14 +19,17 @@ package org.apache.hadoop.security;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import org.apache.commons.logging.Log;
@@ -40,10 +43,12 @@ import org.apache.hadoop.security.ShellB
public class TestGroupsCaching {
public static final Log LOG = LogFactory.getLog(TestGroupsCaching.class);
- private static Configuration conf = new Configuration();
private static String[] myGroups = {"grp1", "grp2"};
+ private Configuration conf;
- static {
+ @Before
+ public void setup() {
+ conf = new Configuration();
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
FakeGroupMapping.class,
ShellBasedUnixGroupsMapping.class);
@@ -88,7 +93,7 @@ public class TestGroupsCaching {
}
@Test
- public void TestGroupsCaching() throws Exception {
+ public void testGroupsCaching() throws Exception {
Groups groups = new Groups(conf);
groups.cacheGroupsAdd(Arrays.asList(myGroups));
groups.refresh();
@@ -117,4 +122,45 @@ public class TestGroupsCaching {
FakeGroupMapping.clearBlackList();
assertTrue(groups.getGroups("user1").size() == 2);
}
+
+ public static class FakeunPrivilegedGroupMapping extends FakeGroupMapping {
+ private static boolean invoked = false;
+ @Override
+ public List<String> getGroups(String user) throws IOException {
+ invoked = true;
+ return super.getGroups(user);
+ }
+ }
+
+ /*
+ * Group lookup should not happen for static users
+ */
+ @Test
+ public void testGroupLookupForStaticUsers() throws Exception {
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ FakeunPrivilegedGroupMapping.class, ShellBasedUnixGroupsMapping.class);
+ conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, "me=;user1=group1;user2=group1,group2");
+ Groups groups = new Groups(conf);
+ List<String> userGroups = groups.getGroups("me");
+ assertTrue("non-empty groups for static user", userGroups.isEmpty());
+ assertFalse("group lookup done for static user",
+ FakeunPrivilegedGroupMapping.invoked);
+
+ List<String> expected = new ArrayList<String>();
+ expected.add("group1");
+
+ FakeunPrivilegedGroupMapping.invoked = false;
+ userGroups = groups.getGroups("user1");
+ assertTrue("groups not correct", expected.equals(userGroups));
+ assertFalse("group lookup done for unprivileged user",
+ FakeunPrivilegedGroupMapping.invoked);
+
+ expected.add("group2");
+ FakeunPrivilegedGroupMapping.invoked = false;
+ userGroups = groups.getGroups("user2");
+ assertTrue("groups not correct", expected.equals(userGroups));
+ assertFalse("group lookup done for unprivileged user",
+ FakeunPrivilegedGroupMapping.invoked);
+
+ }
}