You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2021/05/24 09:45:08 UTC
[hadoop] branch branch-3.3.1 updated: Revert "HADOOP-17669.
Backport HADOOP-17079, HADOOP-17505 to branch-3.3 (#2959)"
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3.1 by this push:
new baf385e Revert "HADOOP-17669. Backport HADOOP-17079, HADOOP-17505 to branch-3.3 (#2959)"
baf385e is described below
commit baf385e77e0c998f8bcdde9d0d7f7d35449d9e4a
Author: Wei-Chiu Chuang <we...@apache.org>
AuthorDate: Mon May 24 17:37:31 2021 +0800
Revert "HADOOP-17669. Backport HADOOP-17079, HADOOP-17505 to branch-3.3 (#2959)"
This reverts commit 4ffe5eb1ddb4aa260134005335a003ed6d270685.
---
.../main/java/org/apache/hadoop/fs/FileSystem.java | 2 +-
.../java/org/apache/hadoop/io/SecureIOUtils.java | 2 +-
.../hadoop/security/CompositeGroupsMapping.java | 24 -----
.../security/GroupMappingServiceProvider.java | 14 ---
.../java/org/apache/hadoop/security/Groups.java | 97 +++++++-----------
.../hadoop/security/JniBasedUnixGroupsMapping.java | 17 +--
.../JniBasedUnixGroupsMappingWithFallback.java | 6 --
...BasedUnixGroupsNetgroupMappingWithFallback.java | 6 --
.../apache/hadoop/security/LdapGroupsMapping.java | 114 ++++++++++-----------
.../apache/hadoop/security/NullGroupsMapping.java | 15 ---
.../security/RuleBasedLdapGroupsMapping.java | 17 +--
.../security/ShellBasedUnixGroupsMapping.java | 51 ++++-----
.../hadoop/security/UserGroupInformation.java | 52 +++-------
.../security/authorize/AccessControlList.java | 6 +-
.../org/apache/hadoop/http/TestHttpServer.java | 9 --
.../hadoop/security/TestCompositeGroupMapping.java | 42 ++------
.../apache/hadoop/security/TestGroupsCaching.java | 23 ++---
.../security/TestRuleBasedLdapGroupsMapping.java | 10 +-
.../apache/hadoop/fs/http/server/HttpFSServer.java | 3 +-
.../java/org/apache/hadoop/lib/service/Groups.java | 3 -
.../hadoop/lib/service/security/GroupsService.java | 10 --
.../hadoop/fs/http/server/TestHttpFSServer.java | 7 --
.../lib/service/security/DummyGroupMapping.java | 15 ---
.../federation/router/RouterPermissionChecker.java | 5 +-
.../federation/store/records/MountTable.java | 2 +-
...tRouterRefreshSuperUserGroupsConfiguration.java | 3 -
.../federation/router/TestRouterUserMappings.java | 19 ----
.../hadoop/hdfs/server/datanode/DataNode.java | 3 +-
.../hdfs/server/namenode/FSPermissionChecker.java | 2 +-
.../hadoop/security/TestRefreshUserMappings.java | 15 ---
.../mapreduce/v2/hs/server/TestHSAdminServer.java | 15 ---
.../v2/hs/webapp/TestHsWebServicesAcls.java | 6 --
.../resources/NetworkTagMappingJsonManager.java | 2 +-
.../runtime/JavaSandboxLinuxContainerRuntime.java | 8 +-
.../placement/PrimaryGroupPlacementRule.java | 10 +-
.../SecondaryGroupExistingPlacementRule.java | 12 +--
.../placement/UserGroupMappingPlacementRule.java | 17 ++-
.../server/resourcemanager/TestRMAdminService.java | 5 -
.../scheduler/fair/PeriodGroupsMapping.java | 10 +-
.../scheduler/fair/PrimaryGroupMapping.java | 7 --
.../scheduler/fair/SimpleGroupsMapping.java | 8 --
41 files changed, 188 insertions(+), 506 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index df92ee0..528f6c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2747,7 +2747,7 @@ public abstract class FileSystem extends Configured
if (perm.getUserAction().implies(mode)) {
return;
}
- } else if (ugi.getGroupsSet().contains(stat.getGroup())) {
+ } else if (ugi.getGroups().contains(stat.getGroup())) {
if (perm.getGroupAction().implies(mode)) {
return;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
index e6bd81a..2f1eecd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
@@ -272,7 +272,7 @@ public class SecureIOUtils {
UserGroupInformation.createRemoteUser(expectedOwner);
final String adminsGroupString = "Administrators";
success = owner.equals(adminsGroupString)
- && ugi.getGroupsSet().contains(adminsGroupString);
+ && ugi.getGroups().contains(adminsGroupString);
} else {
success = false;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java
index 807628a..b762df2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.security;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -109,29 +108,6 @@ public class CompositeGroupsMapping
}
@Override
- public synchronized Set<String> getGroupsSet(String user) throws IOException {
- Set<String> groupSet = new HashSet<String>();
-
- Set<String> groups = null;
- for (GroupMappingServiceProvider provider : providersList) {
- try {
- groups = provider.getGroupsSet(user);
- } catch (Exception e) {
- LOG.warn("Unable to get groups for user {} via {} because: {}",
- user, provider.getClass().getSimpleName(), e.toString());
- LOG.debug("Stacktrace: ", e);
- }
- if (groups != null && !groups.isEmpty()) {
- groupSet.addAll(groups);
- if (!combined) {
- break;
- }
- }
- }
- return groupSet;
- }
-
- @Override
public synchronized Configuration getConf() {
return conf;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java
index 3a9073b..8b90f5b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java
@@ -18,9 +18,7 @@
package org.apache.hadoop.security;
import java.io.IOException;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -54,16 +52,4 @@ public interface GroupMappingServiceProvider {
* @throws IOException
*/
public void cacheGroupsAdd(List<String> groups) throws IOException;
-
- /**
- * Get all various group memberships of a given user.
- * Returns EMPTY set in case of non-existing user
- * @param user User's name
- * @return set of group memberships of user
- * @throws IOException
- */
- default Set<String> getGroupsSet(String user) throws IOException {
- //Override to form the set directly to avoid another conversion
- return new LinkedHashSet<>(getGroups(user));
- }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index 406d0d0..5ff5778 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -26,6 +26,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
@@ -77,8 +78,8 @@ public class Groups {
private final GroupMappingServiceProvider impl;
- private final LoadingCache<String, Set<String>> cache;
- private final AtomicReference<Map<String, Set<String>>> staticMapRef =
+ private final LoadingCache<String, List<String>> cache;
+ private final AtomicReference<Map<String, List<String>>> staticMapRef =
new AtomicReference<>();
private final long cacheTimeout;
private final long negativeCacheTimeout;
@@ -167,7 +168,8 @@ public class Groups {
CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT);
Collection<String> mappings = StringUtils.getStringCollection(
staticMapping, ";");
- Map<String, Set<String>> staticUserToGroupsMap = new HashMap<>();
+ Map<String, List<String>> staticUserToGroupsMap =
+ new HashMap<String, List<String>>();
for (String users : mappings) {
Collection<String> userToGroups = StringUtils.getStringCollection(users,
"=");
@@ -179,10 +181,10 @@ public class Groups {
String[] userToGroupsArray = userToGroups.toArray(new String[userToGroups
.size()]);
String user = userToGroupsArray[0];
- Set<String> groups = Collections.emptySet();
+ List<String> groups = Collections.emptyList();
if (userToGroupsArray.length == 2) {
- groups = new LinkedHashSet(StringUtils
- .getStringCollection(userToGroupsArray[1]));
+ groups = (List<String>) StringUtils
+ .getStringCollection(userToGroupsArray[1]);
}
staticUserToGroupsMap.put(user, groups);
}
@@ -201,47 +203,15 @@ public class Groups {
/**
* Get the group memberships of a given user.
* If the user's group is not cached, this method may block.
- * Note this method can be expensive as it involves Set->List conversion.
- * For user with large group membership (i.e., > 1000 groups), we recommend
- * using getGroupSet to avoid the conversion and fast membership look up via
- * contains().
* @param user User's name
- * @return the group memberships of the user as list
+ * @return the group memberships of the user
* @throws IOException if user does not exist
- * @deprecated Use {@link #getGroupsSet(String user)} instead.
*/
- @Deprecated
public List<String> getGroups(final String user) throws IOException {
- return Collections.unmodifiableList(new ArrayList<>(
- getGroupInternal(user)));
- }
-
- /**
- * Get the group memberships of a given user.
- * If the user's group is not cached, this method may block.
- * This provide better performance when user has large group membership via
- * 1) avoid set->list->set conversion for the caller UGI/PermissionCheck
- * 2) fast lookup using contains() via Set instead of List
- * @param user User's name
- * @return the group memberships of the user as set
- * @throws IOException if user does not exist
- */
- public Set<String> getGroupsSet(final String user) throws IOException {
- return Collections.unmodifiableSet(getGroupInternal(user));
- }
-
- /**
- * Get the group memberships of a given user.
- * If the user's group is not cached, this method may block.
- * @param user User's name
- * @return the group memberships of the user as Set
- * @throws IOException if user does not exist
- */
- private Set<String> getGroupInternal(final String user) throws IOException {
// No need to lookup for groups of static users
- Map<String, Set<String>> staticUserToGroupsMap = staticMapRef.get();
+ Map<String, List<String>> staticUserToGroupsMap = staticMapRef.get();
if (staticUserToGroupsMap != null) {
- Set<String> staticMapping = staticUserToGroupsMap.get(user);
+ List<String> staticMapping = staticUserToGroupsMap.get(user);
if (staticMapping != null) {
return staticMapping;
}
@@ -297,7 +267,7 @@ public class Groups {
/**
* Deals with loading data into the cache.
*/
- private class GroupCacheLoader extends CacheLoader<String, Set<String>> {
+ private class GroupCacheLoader extends CacheLoader<String, List<String>> {
private ListeningExecutorService executorService;
@@ -338,7 +308,7 @@ public class Groups {
* @throws IOException to prevent caching negative entries
*/
@Override
- public Set<String> load(String user) throws Exception {
+ public List<String> load(String user) throws Exception {
LOG.debug("GroupCacheLoader - load.");
TraceScope scope = null;
Tracer tracer = Tracer.curThreadTracer();
@@ -346,9 +316,9 @@ public class Groups {
scope = tracer.newScope("Groups#fetchGroupList");
scope.addKVAnnotation("user", user);
}
- Set<String> groups = null;
+ List<String> groups = null;
try {
- groups = fetchGroupSet(user);
+ groups = fetchGroupList(user);
} finally {
if (scope != null) {
scope.close();
@@ -364,7 +334,9 @@ public class Groups {
throw noGroupsForUser(user);
}
- return groups;
+ // return immutable de-duped list
+ return Collections.unmodifiableList(
+ new ArrayList<>(new LinkedHashSet<>(groups)));
}
/**
@@ -373,8 +345,8 @@ public class Groups {
* implementation, otherwise is arranges for the cache to be updated later
*/
@Override
- public ListenableFuture<Set<String>> reload(final String key,
- Set<String> oldValue)
+ public ListenableFuture<List<String>> reload(final String key,
+ List<String> oldValue)
throws Exception {
LOG.debug("GroupCacheLoader - reload (async).");
if (!reloadGroupsInBackground) {
@@ -382,16 +354,19 @@ public class Groups {
}
backgroundRefreshQueued.incrementAndGet();
- ListenableFuture<Set<String>> listenableFuture =
- executorService.submit(() -> {
- backgroundRefreshQueued.decrementAndGet();
- backgroundRefreshRunning.incrementAndGet();
- Set<String> results = load(key);
- return results;
+ ListenableFuture<List<String>> listenableFuture =
+ executorService.submit(new Callable<List<String>>() {
+ @Override
+ public List<String> call() throws Exception {
+ backgroundRefreshQueued.decrementAndGet();
+ backgroundRefreshRunning.incrementAndGet();
+ List<String> results = load(key);
+ return results;
+ }
});
- Futures.addCallback(listenableFuture, new FutureCallback<Set<String>>() {
+ Futures.addCallback(listenableFuture, new FutureCallback<List<String>>() {
@Override
- public void onSuccess(Set<String> result) {
+ public void onSuccess(List<String> result) {
backgroundRefreshSuccess.incrementAndGet();
backgroundRefreshRunning.decrementAndGet();
}
@@ -405,12 +380,11 @@ public class Groups {
}
/**
- * Queries impl for groups belonging to the user.
- * This could involve I/O and take awhile.
+ * Queries impl for groups belonging to the user. This could involve I/O and take awhile.
*/
- private Set<String> fetchGroupSet(String user) throws IOException {
+ private List<String> fetchGroupList(String user) throws IOException {
long startMs = timer.monotonicNow();
- Set<String> groups = impl.getGroupsSet(user);
+ List<String> groupList = impl.getGroups(user);
long endMs = timer.monotonicNow();
long deltaMs = endMs - startMs ;
UserGroupInformation.metrics.addGetGroups(deltaMs);
@@ -418,7 +392,8 @@ public class Groups {
LOG.warn("Potential performance problem: getGroups(user=" + user +") " +
"took " + deltaMs + " milliseconds.");
}
- return groups;
+
+ return groupList;
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java
index 6c24427..a0f6142 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java
@@ -20,11 +20,8 @@ package org.apache.hadoop.security;
import java.io.IOException;
import java.util.Arrays;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Set;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -78,18 +75,6 @@ public class JniBasedUnixGroupsMapping implements GroupMappingServiceProvider {
@Override
public List<String> getGroups(String user) throws IOException {
- return Arrays.asList(getGroupsInternal(user));
- }
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- String[] groups = getGroupsInternal(user);
- Set<String> result = new LinkedHashSet(groups.length);
- CollectionUtils.addAll(result, groups);
- return result;
- }
-
- private String[] getGroupsInternal(String user) throws IOException {
String[] groups = new String[0];
try {
groups = getGroupsForUser(user);
@@ -100,7 +85,7 @@ public class JniBasedUnixGroupsMapping implements GroupMappingServiceProvider {
LOG.info("Error getting groups for " + user + ": " + e.getMessage());
}
}
- return groups;
+ return Arrays.asList(groups);
}
@Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java
index cc47df1..f164430 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.security;
import java.io.IOException;
import java.util.List;
-import java.util.Set;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.PerformanceAdvisory;
@@ -62,9 +61,4 @@ public class JniBasedUnixGroupsMappingWithFallback implements
impl.cacheGroupsAdd(groups);
}
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return impl.getGroupsSet(user);
- }
-
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMappingWithFallback.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMappingWithFallback.java
index 3d4bd58..fcc47cb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMappingWithFallback.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMappingWithFallback.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.security;
import java.io.IOException;
import java.util.List;
-import java.util.Set;
import org.apache.hadoop.util.NativeCodeLoader;
import org.slf4j.Logger;
@@ -61,9 +60,4 @@ public class JniBasedUnixGroupsNetgroupMappingWithFallback implements
impl.cacheGroupsAdd(groups);
}
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return impl.getGroupsSet(user);
- }
-
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index 63ad723..c51821f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -33,7 +33,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Hashtable;
import java.util.Iterator;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.HashSet;
import java.util.Collection;
@@ -303,12 +302,12 @@ public class LdapGroupsMapping
}
private DirContext ctx;
- private volatile Configuration conf;
+ private Configuration conf;
- private volatile Iterator<String> ldapUrls;
+ private Iterator<String> ldapUrls;
private String currentLdapUrl;
- private volatile boolean useSsl;
+ private boolean useSsl;
private String keystore;
private String keystorePass;
private String truststore;
@@ -321,21 +320,21 @@ public class LdapGroupsMapping
private Iterator<BindUserInfo> bindUsers;
private BindUserInfo currentBindUser;
- private volatile String userbaseDN;
+ private String userbaseDN;
private String groupbaseDN;
private String groupSearchFilter;
- private volatile String userSearchFilter;
- private volatile String memberOfAttr;
+ private String userSearchFilter;
+ private String memberOfAttr;
private String groupMemberAttr;
- private volatile String groupNameAttr;
- private volatile int groupHierarchyLevels;
- private volatile String posixUidAttr;
- private volatile String posixGidAttr;
+ private String groupNameAttr;
+ private int groupHierarchyLevels;
+ private String posixUidAttr;
+ private String posixGidAttr;
private boolean isPosix;
- private volatile boolean useOneQuery;
+ private boolean useOneQuery;
private int numAttempts;
- private volatile int numAttemptsBeforeFailover;
- private volatile String ldapCtxFactoryClassName;
+ private int numAttemptsBeforeFailover;
+ private String ldapCtxFactoryClassName;
/**
* Returns list of groups for a user.
@@ -349,7 +348,38 @@ public class LdapGroupsMapping
*/
@Override
public synchronized List<String> getGroups(String user) {
- return new ArrayList<>(getGroupsSet(user));
+ /*
+ * Normal garbage collection takes care of removing Context instances when
+ * they are no longer in use. Connections used by Context instances being
+ * garbage collected will be closed automatically. So in case connection is
+ * closed and gets CommunicationException, retry some times with new new
+ * DirContext/connection.
+ */
+
+ // Tracks the number of attempts made using the same LDAP server
+ int atemptsBeforeFailover = 1;
+
+ for (int attempt = 1; attempt <= numAttempts; attempt++,
+ atemptsBeforeFailover++) {
+ try {
+ return doGetGroups(user, groupHierarchyLevels);
+ } catch (AuthenticationException e) {
+ switchBindUser(e);
+ } catch (NamingException e) {
+ LOG.warn("Failed to get groups for user {} (attempt={}/{}) using {}. " +
+ "Exception: ", user, attempt, numAttempts, currentLdapUrl, e);
+ LOG.trace("TRACE", e);
+
+ if (failover(atemptsBeforeFailover, numAttemptsBeforeFailover)) {
+ atemptsBeforeFailover = 0;
+ }
+ }
+
+ // Reset ctx so that new DirContext can be created with new connection
+ this.ctx = null;
+ }
+
+ return Collections.emptyList();
}
/**
@@ -428,10 +458,10 @@ public class LdapGroupsMapping
* @return a list of strings representing group names of the user.
* @throws NamingException if unable to find group names
*/
- private Set<String> lookupGroup(SearchResult result, DirContext c,
+ private List<String> lookupGroup(SearchResult result, DirContext c,
int goUpHierarchy)
throws NamingException {
- Set<String> groups = new LinkedHashSet<>();
+ List<String> groups = new ArrayList<>();
Set<String> groupDNs = new HashSet<>();
NamingEnumeration<SearchResult> groupResults;
@@ -454,7 +484,11 @@ public class LdapGroupsMapping
getGroupNames(groupResult, groups, groupDNs, goUpHierarchy > 0);
}
if (goUpHierarchy > 0 && !isPosix) {
- goUpGroupHierarchy(groupDNs, goUpHierarchy, groups);
+ // convert groups to a set to ensure uniqueness
+ Set<String> groupset = new HashSet<>(groups);
+ goUpGroupHierarchy(groupDNs, goUpHierarchy, groupset);
+ // convert set back to list for compatibility
+ groups = new ArrayList<>(groupset);
}
}
return groups;
@@ -473,7 +507,7 @@ public class LdapGroupsMapping
* return an empty string array.
* @throws NamingException if unable to get group names
*/
- Set<String> doGetGroups(String user, int goUpHierarchy)
+ List<String> doGetGroups(String user, int goUpHierarchy)
throws NamingException {
DirContext c = getDirContext();
@@ -484,11 +518,11 @@ public class LdapGroupsMapping
if (!results.hasMoreElements()) {
LOG.debug("doGetGroups({}) returned no groups because the " +
"user is not found.", user);
- return Collections.emptySet();
+ return new ArrayList<>();
}
SearchResult result = results.nextElement();
- Set<String> groups = Collections.emptySet();
+ List<String> groups = null;
if (useOneQuery) {
try {
/**
@@ -502,7 +536,7 @@ public class LdapGroupsMapping
memberOfAttr + "' attribute." +
"Returned user object: " + result.toString());
}
- groups = new LinkedHashSet<>();
+ groups = new ArrayList<>();
NamingEnumeration groupEnumeration = groupDNAttr.getAll();
while (groupEnumeration.hasMore()) {
String groupDN = groupEnumeration.next().toString();
@@ -689,42 +723,6 @@ public class LdapGroupsMapping
}
@Override
- public Set<String> getGroupsSet(String user) {
- /*
- * Normal garbage collection takes care of removing Context instances when
- * they are no longer in use. Connections used by Context instances being
- * garbage collected will be closed automatically. So in case connection is
- * closed and gets CommunicationException, retry some times with new new
- * DirContext/connection.
- */
-
- // Tracks the number of attempts made using the same LDAP server
- int atemptsBeforeFailover = 1;
-
- for (int attempt = 1; attempt <= numAttempts; attempt++,
- atemptsBeforeFailover++) {
- try {
- return doGetGroups(user, groupHierarchyLevels);
- } catch (AuthenticationException e) {
- switchBindUser(e);
- } catch (NamingException e) {
- LOG.warn("Failed to get groups for user {} (attempt={}/{}) using {}. " +
- "Exception: ", user, attempt, numAttempts, currentLdapUrl, e);
- LOG.trace("TRACE", e);
-
- if (failover(atemptsBeforeFailover, numAttemptsBeforeFailover)) {
- atemptsBeforeFailover = 0;
- }
- }
-
- // Reset ctx so that new DirContext can be created with new connection
- this.ctx = null;
- }
-
- return Collections.emptySet();
- }
-
- @Override
public synchronized Configuration getConf() {
return conf;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NullGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NullGroupsMapping.java
index 9592ecc..f3d048d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NullGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NullGroupsMapping.java
@@ -15,10 +15,8 @@
*/
package org.apache.hadoop.security;
-import java.io.IOException;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
/**
* This class provides groups mapping for {@link UserGroupInformation} when the
@@ -34,19 +32,6 @@ public class NullGroupsMapping implements GroupMappingServiceProvider {
}
/**
- * Get all various group memberships of a given user.
- * Returns EMPTY set in case of non-existing user
- *
- * @param user User's name
- * @return set of group memberships of user
- * @throws IOException
- */
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return Collections.emptySet();
- }
-
- /**
* Returns an empty list.
* @param user ignored
* @return an empty list
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/RuleBasedLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/RuleBasedLdapGroupsMapping.java
index 5fadcc3..6af28f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/RuleBasedLdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/RuleBasedLdapGroupsMapping.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.security;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -24,9 +25,7 @@ import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Set;
import java.util.stream.Collectors;
/**
@@ -89,18 +88,4 @@ public class RuleBasedLdapGroupsMapping extends LdapGroupsMapping {
}
}
- public synchronized Set<String> getGroupsSet(String user) {
- Set<String> groups = super.getGroupsSet(user);
- switch (rule) {
- case TO_UPPER:
- return groups.stream().map(StringUtils::toUpperCase).collect(
- Collectors.toCollection(LinkedHashSet::new));
- case TO_LOWER:
- return groups.stream().map(StringUtils::toLowerCase).collect(
- Collectors.toCollection(LinkedHashSet::new));
- case NONE:
- default:
- return groups;
- }
- }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
index ca9f474..96e4402 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
@@ -18,11 +18,8 @@
package org.apache.hadoop.security;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedHashSet;
+import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
import java.util.StringTokenizer;
import java.util.concurrent.TimeUnit;
@@ -56,7 +53,7 @@ public class ShellBasedUnixGroupsMapping extends Configured
private long timeout = CommonConfigurationKeys.
HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT;
- private static final Set<String> EMPTY_GROUPS_SET = Collections.emptySet();
+ private static final List<String> EMPTY_GROUPS = new LinkedList<>();
@Override
public void setConf(Configuration conf) {
@@ -97,7 +94,7 @@ public class ShellBasedUnixGroupsMapping extends Configured
*/
@Override
public List<String> getGroups(String userName) throws IOException {
- return new ArrayList(getUnixGroups(userName));
+ return getUnixGroups(userName);
}
/**
@@ -118,11 +115,6 @@ public class ShellBasedUnixGroupsMapping extends Configured
// does nothing in this provider of user to groups mapping
}
- @Override
- public Set<String> getGroupsSet(String userName) throws IOException {
- return getUnixGroups(userName);
- }
-
/**
* Create a ShellCommandExecutor object using the user's name.
*
@@ -200,33 +192,44 @@ public class ShellBasedUnixGroupsMapping extends Configured
* group is returned first.
* @throws IOException if encounter any error when running the command
*/
- private Set<String> getUnixGroups(String user) throws IOException {
+ private List<String> getUnixGroups(String user) throws IOException {
ShellCommandExecutor executor = createGroupExecutor(user);
- Set<String> groups;
+ List<String> groups;
try {
executor.execute();
groups = resolveFullGroupNames(executor.getOutput());
} catch (ExitCodeException e) {
if (handleExecutorTimeout(executor, user)) {
- return EMPTY_GROUPS_SET;
+ return EMPTY_GROUPS;
} else {
try {
groups = resolvePartialGroupNames(user, e.getMessage(),
executor.getOutput());
} catch (PartialGroupNameException pge) {
LOG.warn("unable to return groups for user {}", user, pge);
- return EMPTY_GROUPS_SET;
+ return EMPTY_GROUPS;
}
}
} catch (IOException ioe) {
if (handleExecutorTimeout(executor, user)) {
- return EMPTY_GROUPS_SET;
+ return EMPTY_GROUPS;
} else {
// If its not an executor timeout, we should let the caller handle it
throw ioe;
}
}
+
+ // remove duplicated primary group
+ if (!Shell.WINDOWS) {
+ for (int i = 1; i < groups.size(); i++) {
+ if (groups.get(i).equals(groups.get(0))) {
+ groups.remove(i);
+ break;
+ }
+ }
+ }
+
return groups;
}
@@ -239,13 +242,13 @@ public class ShellBasedUnixGroupsMapping extends Configured
* @return a linked list of group names
* @throws PartialGroupNameException
*/
- private Set<String> parsePartialGroupNames(String groupNames,
+ private List<String> parsePartialGroupNames(String groupNames,
String groupIDs) throws PartialGroupNameException {
StringTokenizer nameTokenizer =
new StringTokenizer(groupNames, Shell.TOKEN_SEPARATOR_REGEX);
StringTokenizer idTokenizer =
new StringTokenizer(groupIDs, Shell.TOKEN_SEPARATOR_REGEX);
- Set<String> groups = new LinkedHashSet<>();
+ List<String> groups = new LinkedList<String>();
while (nameTokenizer.hasMoreTokens()) {
// check for unresolvable group names.
if (!idTokenizer.hasMoreTokens()) {
@@ -274,10 +277,10 @@ public class ShellBasedUnixGroupsMapping extends Configured
* @param userName the user's name
* @param errMessage error message from the shell command
* @param groupNames the incomplete list of group names
- * @return a set of resolved group names
+ * @return a list of resolved group names
* @throws PartialGroupNameException if the resolution fails or times out
*/
- private Set<String> resolvePartialGroupNames(String userName,
+ private List<String> resolvePartialGroupNames(String userName,
String errMessage, String groupNames) throws PartialGroupNameException {
// Exception may indicate that some group names are not resolvable.
// Shell-based implementation should tolerate unresolvable groups names,
@@ -319,16 +322,16 @@ public class ShellBasedUnixGroupsMapping extends Configured
}
/**
- * Split group names into a set.
+ * Split group names into a linked list.
*
* @param groupNames a string representing the user's group names
- * @return a set of group names
+ * @return a linked list of group names
*/
@VisibleForTesting
- protected Set<String> resolveFullGroupNames(String groupNames) {
+ protected List<String> resolveFullGroupNames(String groupNames) {
StringTokenizer tokenizer =
new StringTokenizer(groupNames, Shell.TOKEN_SEPARATOR_REGEX);
- Set<String> groups = new LinkedHashSet<>();
+ List<String> groups = new LinkedList<String>();
while (tokenizer.hasMoreTokens()) {
groups.add(tokenizer.nextToken());
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 4ee177c..67d1518 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -40,6 +40,7 @@ import java.security.PrivilegedAction;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
@@ -1515,8 +1516,8 @@ public class UserGroupInformation {
* map that has the translation of usernames to groups.
*/
private static class TestingGroups extends Groups {
- private final Map<String, Set<String>> userToGroupsMapping =
- new HashMap<>();
+ private final Map<String, List<String>> userToGroupsMapping =
+ new HashMap<String,List<String>>();
private Groups underlyingImplementation;
private TestingGroups(Groups underlyingImplementation) {
@@ -1526,22 +1527,17 @@ public class UserGroupInformation {
@Override
public List<String> getGroups(String user) throws IOException {
- return new ArrayList<>(getGroupsSet(user));
- }
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- Set<String> result = userToGroupsMapping.get(user);
+ List<String> result = userToGroupsMapping.get(user);
+
if (result == null) {
- result = underlyingImplementation.getGroupsSet(user);
+ result = underlyingImplementation.getGroups(user);
}
+
return result;
}
private void setUserGroups(String user, String[] groups) {
- Set<String> groupsSet = new LinkedHashSet<>();
- Collections.addAll(groupsSet, groups);
- userToGroupsMapping.put(user, groupsSet);
+ userToGroupsMapping.put(user, Arrays.asList(groups));
}
}
@@ -1600,11 +1596,11 @@ public class UserGroupInformation {
}
public String getPrimaryGroupName() throws IOException {
- Set<String> groupsSet = getGroupsSet();
- if (groupsSet.isEmpty()) {
+ List<String> groups = getGroups();
+ if (groups.isEmpty()) {
throw new IOException("There is no primary group for UGI " + this);
}
- return groupsSet.iterator().next();
+ return groups.get(0);
}
/**
@@ -1717,24 +1713,21 @@ public class UserGroupInformation {
}
/**
- * Get the group names for this user. {@link #getGroupsSet()} is less
+ * Get the group names for this user. {@link #getGroups()} is less
* expensive alternative when checking for a contained element.
* @return the list of users with the primary group first. If the command
* fails, it returns an empty list.
*/
public String[] getGroupNames() {
- Collection<String> groupsSet = getGroupsSet();
- return groupsSet.toArray(new String[groupsSet.size()]);
+ List<String> groups = getGroups();
+ return groups.toArray(new String[groups.size()]);
}
/**
- * Get the group names for this user. {@link #getGroupsSet()} is less
- * expensive alternative when checking for a contained element.
+ * Get the group names for this user.
* @return the list of users with the primary group first. If the command
* fails, it returns an empty list.
- * @deprecated Use {@link #getGroupsSet()} instead.
*/
- @Deprecated
public List<String> getGroups() {
ensureInitialized();
try {
@@ -1746,21 +1739,6 @@ public class UserGroupInformation {
}
/**
- * Get the groups names for the user as a Set.
- * @return the set of users with the primary group first. If the command
- * fails, it returns an empty set.
- */
- public Set<String> getGroupsSet() {
- ensureInitialized();
- try {
- return groups.getGroupsSet(getShortUserName());
- } catch (IOException ie) {
- LOG.debug("Failed to get groups for user {}", getShortUserName(), ie);
- return Collections.emptySet();
- }
- }
-
- /**
* Return the username.
*/
@Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
index e86d918..8af47d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
@@ -24,7 +24,6 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -232,9 +231,8 @@ public class AccessControlList implements Writable {
if (allAllowed || users.contains(ugi.getShortUserName())) {
return true;
} else if (!groups.isEmpty()) {
- Set<String> ugiGroups = ugi.getGroupsSet();
- for (String group : groups) {
- if (ugiGroups.contains(group)) {
+ for (String group : ugi.getGroups()) {
+ if (groups.contains(group)) {
return true;
}
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
index ad9617d..e0c87e9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
@@ -62,10 +62,8 @@ import java.net.URL;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.CountDownLatch;
@@ -412,13 +410,6 @@ public class TestHttpServer extends HttpServerFunctionalTest {
public List<String> getGroups(String user) throws IOException {
return mapping.get(user);
}
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- Set<String> result = new HashSet();
- result.addAll(mapping.get(user));
- return result;
- }
}
/**
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java
index 1803fb1..0a2d42c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java
@@ -22,9 +22,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -89,22 +87,13 @@ public class TestCompositeGroupMapping {
public void cacheGroupsAdd(List<String> groups) throws IOException {
}
-
+
protected List<String> toList(String group) {
if (group != null) {
return Arrays.asList(new String[] {group});
}
return new ArrayList<String>();
}
-
- protected Set<String> toSet(String group) {
- if (group != null) {
- Set<String> result = new HashSet<>();
- result.add(group);
- return result;
- }
- return new HashSet<String>();
- }
protected void checkTestConf(String expectedValue) {
String configValue = getConf().get(PROVIDER_SPECIFIC_CONF_KEY);
@@ -117,49 +106,32 @@ public class TestCompositeGroupMapping {
private static class UserProvider extends GroupMappingProviderBase {
@Override
public List<String> getGroups(String user) throws IOException {
- return toList(getGroupInternal(user));
- }
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return toSet(getGroupInternal(user));
- }
-
- private String getGroupInternal(String user) throws IOException {
checkTestConf(PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER);
-
+
String group = null;
if (user.equals(john.name)) {
group = john.group;
} else if (user.equals(jack.name)) {
group = jack.group;
}
- return group;
+
+ return toList(group);
}
}
private static class ClusterProvider extends GroupMappingProviderBase {
@Override
public List<String> getGroups(String user) throws IOException {
- return toList(getGroupsInternal(user));
- }
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return toSet(getGroupsInternal(user));
- }
-
- private String getGroupsInternal(String user) throws IOException {
checkTestConf(PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER);
-
+
String group = null;
if (user.equals(hdfs.name)) {
group = hdfs.group;
} else if (user.equals(jack.name)) { // jack has another group from clusterProvider
group = jack.group2;
}
- return group;
-
+
+ return toList(group);
}
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
index 8778869..ebff93d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
@@ -21,9 +21,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@@ -75,7 +75,7 @@ public class TestGroupsCaching {
private static volatile CountDownLatch latch = null;
@Override
- public Set<String> getGroupsSet(String user) throws IOException {
+ public List<String> getGroups(String user) throws IOException {
TESTLOG.info("Getting groups for " + user);
delayIfNecessary();
@@ -86,14 +86,9 @@ public class TestGroupsCaching {
}
if (blackList.contains(user)) {
- return Collections.emptySet();
+ return new LinkedList<String>();
}
- return new LinkedHashSet<>(allGroups);
- }
-
- @Override
- public List<String> getGroups(String user) throws IOException {
- return new ArrayList<>(getGroupsSet(user));
+ return new LinkedList<String>(allGroups);
}
/**
@@ -134,7 +129,7 @@ public class TestGroupsCaching {
TESTLOG.info("Resetting FakeGroupMapping");
blackList.clear();
allGroups.clear();
- resetRequestCount();
+ requestCount = 0;
getGroupsDelayMs = 0;
throwException = false;
latch = null;
@@ -202,12 +197,6 @@ public class TestGroupsCaching {
throw new IOException("For test");
}
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- requestCount++;
- throw new IOException("For test");
- }
-
public static int getRequestCount() {
return requestCount;
}
@@ -561,7 +550,7 @@ public class TestGroupsCaching {
FakeGroupMapping.clearBlackList();
// We make an initial request to populate the cache
- List<String> g1 = groups.getGroups("me");
+ groups.getGroups("me");
// add another group
groups.cacheGroupsAdd(Arrays.asList("grp3"));
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestRuleBasedLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestRuleBasedLdapGroupsMapping.java
index 8862fd7..cd04ae0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestRuleBasedLdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestRuleBasedLdapGroupsMapping.java
@@ -24,9 +24,7 @@ import org.mockito.Mockito;
import javax.naming.NamingException;
import java.util.ArrayList;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Set;
import static org.apache.hadoop.security.RuleBasedLdapGroupsMapping
.CONVERSION_RULE_KEY;
@@ -42,7 +40,7 @@ public class TestRuleBasedLdapGroupsMapping {
public void testGetGroupsToUpper() throws NamingException {
RuleBasedLdapGroupsMapping groupsMapping = Mockito.spy(
new RuleBasedLdapGroupsMapping());
- Set<String> groups = new LinkedHashSet<>();
+ List<String> groups = new ArrayList<>();
groups.add("group1");
groups.add("group2");
Mockito.doReturn(groups).when((LdapGroupsMapping) groupsMapping)
@@ -63,7 +61,7 @@ public class TestRuleBasedLdapGroupsMapping {
public void testGetGroupsToLower() throws NamingException {
RuleBasedLdapGroupsMapping groupsMapping = Mockito.spy(
new RuleBasedLdapGroupsMapping());
- Set<String> groups = new LinkedHashSet<>();
+ List<String> groups = new ArrayList<>();
groups.add("GROUP1");
groups.add("GROUP2");
Mockito.doReturn(groups).when((LdapGroupsMapping) groupsMapping)
@@ -84,7 +82,7 @@ public class TestRuleBasedLdapGroupsMapping {
public void testGetGroupsInvalidRule() throws NamingException {
RuleBasedLdapGroupsMapping groupsMapping = Mockito.spy(
new RuleBasedLdapGroupsMapping());
- Set<String> groups = new LinkedHashSet<>();
+ List<String> groups = new ArrayList<>();
groups.add("group1");
groups.add("GROUP2");
Mockito.doReturn(groups).when((LdapGroupsMapping) groupsMapping)
@@ -95,7 +93,7 @@ public class TestRuleBasedLdapGroupsMapping {
conf.set(CONVERSION_RULE_KEY, "none");
groupsMapping.setConf(conf);
- Assert.assertEquals(groups, groupsMapping.getGroupsSet("admin"));
+ Assert.assertEquals(groups, groupsMapping.getGroups("admin"));
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
index 9159922..5965f70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
@@ -96,7 +96,6 @@ import java.text.MessageFormat;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
/**
* Main class of HttpFSServer server.
@@ -325,7 +324,7 @@ public class HttpFSServer {
case INSTRUMENTATION: {
enforceRootPath(op.value(), path);
Groups groups = HttpFSServerWebApp.get().get(Groups.class);
- Set<String> userGroups = groups.getGroupsSet(user.getShortUserName());
+ List<String> userGroups = groups.getGroups(user.getShortUserName());
if (!userGroups.contains(HttpFSServerWebApp.get().getAdminGroup())) {
throw new AccessControlException(
"User not in HttpFSServer admin group");
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/Groups.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/Groups.java
index 2cc942f..90733f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/Groups.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/Groups.java
@@ -22,13 +22,10 @@ import org.apache.hadoop.classification.InterfaceAudience;
import java.io.IOException;
import java.util.List;
-import java.util.Set;
@InterfaceAudience.Private
public interface Groups {
public List<String> getGroups(String user) throws IOException;
- Set<String> getGroupsSet(String user) throws IOException;
-
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/GroupsService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/GroupsService.java
index 8de0630..560a3cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/GroupsService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/GroupsService.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.lib.util.ConfigurationUtils;
import java.io.IOException;
import java.util.List;
-import java.util.Set;
@InterfaceAudience.Private
public class GroupsService extends BaseService implements Groups {
@@ -51,18 +50,9 @@ public class GroupsService extends BaseService implements Groups {
return Groups.class;
}
- /**
- * @deprecated use {@link #getGroupsSet(String user)}
- */
- @Deprecated
@Override
public List<String> getGroups(String user) throws IOException {
return hGroups.getGroups(user);
}
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return hGroups.getGroupsSet(user);
- }
-
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
index a06bce3..2f0ef9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
@@ -61,11 +61,9 @@ import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@@ -191,11 +189,6 @@ public class TestHttpFSServer extends HFSTestCase {
return Arrays.asList(HadoopUsersConfTestHelper.getHadoopUserGroups(user));
}
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return new HashSet<>(getGroups(user));
- }
-
}
private Configuration createHttpFSConf(boolean addDelegationTokenAuthHandler,
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java
index 80a94b1..9ef786d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java
@@ -21,9 +21,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.security.GroupMappingServiceProvider;
import org.apache.hadoop.test.HadoopUsersConfTestHelper;
@@ -49,17 +47,4 @@ public class DummyGroupMapping implements GroupMappingServiceProvider {
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
}
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- if (user.equals("root")) {
- return Sets.newHashSet("admin");
- } else if (user.equals("nobody")) {
- return Sets.newHashSet("nobody");
- } else {
- String[] groups = HadoopUsersConfTestHelper.getHadoopUserGroups(user);
- return (groups != null) ? Sets.newHashSet(groups) :
- Collections.emptySet();
- }
- }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java
index 23a3c6e..3d8ef21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs.server.federation.router;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -124,7 +126,8 @@ public class RouterPermissionChecker extends FSPermissionChecker {
}
// Is the user a member of the super group?
- if (ugi.getGroupsSet().contains(superGroup)) {
+ List<String> groups = Arrays.asList(ugi.getGroupNames());
+ if (groups.contains(superGroup)) {
return;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
index 907a405..5d7d5c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
@@ -149,7 +149,7 @@ public abstract class MountTable extends BaseRecord {
// Set permission fields
UserGroupInformation ugi = NameNode.getRemoteUser();
record.setOwnerName(ugi.getShortUserName());
- String group = ugi.getGroupsSet().isEmpty() ? ugi.getShortUserName()
+ String group = ugi.getGroups().isEmpty() ? ugi.getShortUserName()
: ugi.getPrimaryGroupName();
record.setGroupName(group);
record.setMode(new FsPermission(
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java
index 62fcf31..fb88882 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java
@@ -45,7 +45,6 @@ import java.net.URL;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.LinkedHashSet;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
@@ -136,8 +135,6 @@ public class TestRouterRefreshSuperUserGroupsConfiguration {
when(ugi.getRealUser()).thenReturn(impersonator);
when(ugi.getUserName()).thenReturn("victim");
when(ugi.getGroups()).thenReturn(Arrays.asList("groupVictim"));
- when(ugi.getGroupsSet()).thenReturn(new LinkedHashSet<>(Arrays.asList(
- "groupVictim")));
// Exception should be thrown before applying config
LambdaTestUtils.intercept(
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java
index ba8c463..dc7ebbf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.federation.router;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
@@ -57,9 +56,7 @@ import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.ArrayList;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertArrayEquals;
@@ -114,16 +111,6 @@ public class TestRouterUserMappings {
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
}
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- LOG.info("Getting groups in MockUnixGroupsMapping");
- String g1 = user + (10 * i + 1);
- String g2 = user + (10 * i + 2);
- Set<String> s = Sets.newHashSet(g1, g2);
- i++;
- return s;
- }
}
@Before
@@ -204,10 +191,6 @@ public class TestRouterUserMappings {
final List<String> groupNames2 = new ArrayList<>();
groupNames2.add("gr3");
groupNames2.add("gr4");
- final Set<String> groupNamesSet1 = new LinkedHashSet<>();
- groupNamesSet1.addAll(groupNames1);
- final Set<String> groupNamesSet2 = new LinkedHashSet<>();
- groupNamesSet2.addAll(groupNames2);
//keys in conf
String userKeyGroups = DefaultImpersonationProvider.getTestProvider().
@@ -239,8 +222,6 @@ public class TestRouterUserMappings {
// set groups for users
when(ugi1.getGroups()).thenReturn(groupNames1);
when(ugi2.getGroups()).thenReturn(groupNames2);
- when(ugi1.getGroupsSet()).thenReturn(groupNamesSet1);
- when(ugi2.getGroupsSet()).thenReturn(groupNamesSet2);
// check before refresh
LambdaTestUtils.intercept(AuthorizationException.class,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 23f0a97..079dda4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1082,7 +1082,8 @@ public class DataNode extends ReconfigurableBase
}
// Is the user a member of the super group?
- if (callerUgi.getGroupsSet().contains(supergroup)) {
+ List<String> groups = Arrays.asList(callerUgi.getGroupNames());
+ if (groups.contains(supergroup)) {
return;
}
// Not a superuser.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index a83ec51..a539bf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -103,7 +103,7 @@ public class FSPermissionChecker implements AccessControlEnforcer {
this.fsOwner = fsOwner;
this.supergroup = supergroup;
this.callerUgi = callerUgi;
- this.groups = callerUgi.getGroupsSet();
+ this.groups = callerUgi.getGroups();
user = callerUgi.getShortUserName();
isSuper = user.equals(fsOwner) || groups.contains(supergroup);
this.attributeProvider = attributeProvider;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
index 890c6fb..2d7410a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
@@ -34,11 +34,8 @@ import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.ArrayList;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Set;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -87,16 +84,6 @@ public class TestRefreshUserMappings {
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
}
-
- @Override
- public Set<String> getGroupsSet(String user) {
- LOG.info("Getting groups in MockUnixGroupsMapping");
- String g1 = user + (10 * i + 1);
- String g2 = user + (10 * i + 2);
- Set<String> s = Sets.newHashSet(g1, g2);
- i++;
- return s;
- }
}
@Before
@@ -209,8 +196,6 @@ public class TestRefreshUserMappings {
// set groups for users
when(ugi1.getGroups()).thenReturn(groupNames1);
when(ugi2.getGroups()).thenReturn(groupNames2);
- when(ugi1.getGroupsSet()).thenReturn(new LinkedHashSet<>(groupNames1));
- when(ugi2.getGroupsSet()).thenReturn(new LinkedHashSet<>(groupNames2));
// check before
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
index b961a23..1eb1d1c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
@@ -26,9 +26,7 @@ import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Set;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
@@ -58,7 +56,6 @@ import static org.mockito.Mockito.verify;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
-import org.mockito.internal.util.collections.Sets;
@RunWith(Parameterized.class)
public class TestHSAdminServer {
@@ -94,15 +91,6 @@ public class TestHSAdminServer {
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
}
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- Set<String> result = new LinkedHashSet<>();
- result.add(user + (10 * i + 1));
- result.add(user + (10 * i +2));
- i++;
- return result;
- }
}
@Parameters
@@ -201,9 +189,6 @@ public class TestHSAdminServer {
when(superUser.getUserName()).thenReturn("superuser");
when(ugi.getGroups())
.thenReturn(Arrays.asList(new String[] { "group3" }));
- when(ugi.getGroupsSet())
- .thenReturn(Sets.newSet("group3"));
-
when(ugi.getUserName()).thenReturn("regularUser");
// Set super user groups not to include groups of regularUser
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
index 8d4f635..960993e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
@@ -28,7 +28,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -277,11 +276,6 @@ public class TestHsWebServicesAcls {
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
}
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return Collections.emptySet();
- }
}
private static class MockJobForAcls implements Job {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java
index 9fd293c..e59454b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java
@@ -86,7 +86,7 @@ public class NetworkTagMappingJsonManager implements NetworkTagMappingManager {
container.getUser());
List<Group> groups = this.networkTagMapping.getGroups();
for(Group group : groups) {
- if (userUGI.getGroupsSet().contains(group.getGroupName())) {
+ if (userUGI.getGroups().contains(group.getGroupName())) {
return group.getNetworkTagID();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
index 0a25d10..b4ea66d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
@@ -303,9 +303,9 @@ public class JavaSandboxLinuxContainerRuntime
private static List<String> getGroupPolicyFiles(Configuration conf,
String user) throws ContainerExecutionException {
Groups groups = Groups.getUserToGroupsMappingService(conf);
- Set<String> userGroups;
+ List<String> userGroups;
try {
- userGroups = groups.getGroupsSet(user);
+ userGroups = groups.getGroups(user);
} catch (IOException e) {
throw new ContainerExecutionException("Container user does not exist");
}
@@ -330,11 +330,11 @@ public class JavaSandboxLinuxContainerRuntime
String whitelistGroup = configuration.get(
YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
Groups groups = Groups.getUserToGroupsMappingService(configuration);
- Set<String> userGroups;
+ List<String> userGroups;
boolean isWhitelisted = false;
try {
- userGroups = groups.getGroupsSet(username);
+ userGroups = groups.getGroups(username);
} catch (IOException e) {
throw new ContainerExecutionException("Container user does not exist");
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java
index 948194f..73e5cd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java
@@ -30,7 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.Set;
+import java.util.List;
import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT;
import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot;
@@ -62,19 +62,19 @@ public class PrimaryGroupPlacementRule extends FSPlacementRule {
// All users should have at least one group the primary group. If no groups
// are returned then there is a real issue.
- final Set<String> groupSet;
+ final List<String> groupList;
try {
- groupSet = groupProvider.getGroupsSet(user);
+ groupList = groupProvider.getGroups(user);
} catch (IOException ioe) {
throw new YarnException("Group resolution failed", ioe);
}
- if (groupSet.isEmpty()) {
+ if (groupList.isEmpty()) {
LOG.error("Group placement rule failed: No groups returned for user {}",
user);
throw new YarnException("No groups returned for user " + user);
}
- String cleanGroup = cleanName(groupSet.iterator().next());
+ String cleanGroup = cleanName(groupList.get(0));
String queueName;
PlacementRule parentRule = getParentRule();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java
index 8e6ccb3..9acdbcc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java
@@ -30,8 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.Iterator;
-import java.util.Set;
+import java.util.List;
import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT;
import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot;
@@ -66,9 +65,9 @@ public class SecondaryGroupExistingPlacementRule extends FSPlacementRule {
// All users should have at least one group the primary group. If no groups
// are returned then there is a real issue.
- final Set<String> groupSet;
+ final List<String> groupList;
try {
- groupSet = groupProvider.getGroupsSet(user);
+ groupList = groupProvider.getGroups(user);
} catch (IOException ioe) {
throw new YarnException("Group resolution failed", ioe);
}
@@ -91,9 +90,8 @@ public class SecondaryGroupExistingPlacementRule extends FSPlacementRule {
parentQueue);
}
// now check the groups inside the parent
- Iterator<String> it = groupSet.iterator();
- while (it.hasNext()) {
- String group = cleanName(it.next());
+ for (int i = 1; i < groupList.size(); i++) {
+ String group = cleanName(groupList.get(i));
String queueName =
parentQueue == null ? assureRoot(group) : parentQueue + DOT + group;
if (configuredQueue(queueName)) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index c793c76..8eb912b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -20,9 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.placement;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
-import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -76,21 +74,18 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
}
private String getPrimaryGroup(String user) throws IOException {
- return groups.getGroupsSet(user).iterator().next();
+ return groups.getGroups(user).get(0);
}
private String getSecondaryGroup(String user) throws IOException {
- Set<String> groupsSet = groups.getGroupsSet(user);
+ List<String> groupsList = groups.getGroups(user);
String secondaryGroup = null;
// Traverse all secondary groups (as there could be more than one
// and position is not guaranteed) and ensure there is queue with
// the same name
- Iterator<String> it = groupsSet.iterator();
- it.next();
- while (it.hasNext()) {
- String group = it.next();
- if (this.queueManager.getQueue(group) != null) {
- secondaryGroup = group;
+ for (int i = 1; i < groupsList.size(); i++) {
+ if (this.queueManager.getQueue(groupsList.get(i)) != null) {
+ secondaryGroup = groupsList.get(i);
break;
}
}
@@ -185,7 +180,7 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
}
}
if (mapping.getType().equals(MappingType.GROUP)) {
- for (String userGroups : groups.getGroupsSet(user)) {
+ for (String userGroups : groups.getGroups(user)) {
if (userGroups.equals(mapping.getSource())) {
if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
if (LOG.isDebugEnabled()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index bc540b0..699d975 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -1459,11 +1459,6 @@ public class TestRMAdminService {
// Do nothing
}
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return ImmutableSet.copyOf(group);
- }
-
public static void updateGroups() {
group.clear();
group.add("test_group_D");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PeriodGroupsMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PeriodGroupsMapping.java
index cb24799..9586381 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PeriodGroupsMapping.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PeriodGroupsMapping.java
@@ -18,20 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hadoop.security.GroupMappingServiceProvider;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
-import java.util.Set;
public class PeriodGroupsMapping implements GroupMappingServiceProvider {
@Override
public List<String> getGroups(String user) {
- return Arrays.asList(user + ".group", user + "subgroup1",
- user + "subgroup2");
+ return Arrays.asList(user + ".group", user + "subgroup1", user + "subgroup2");
}
@Override
@@ -44,9 +41,4 @@ public class PeriodGroupsMapping implements GroupMappingServiceProvider {
throw new UnsupportedOperationException();
}
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return ImmutableSet.of(user + ".group", user + "subgroup1",
- user + "subgroup2");
- }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java
index a34ca8b..11415b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java
@@ -22,9 +22,7 @@ import org.apache.hadoop.security.GroupMappingServiceProvider;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
-import java.util.Set;
/**
* Group Mapping class used for test cases. Returns only primary group of the
@@ -46,9 +44,4 @@ public class PrimaryGroupMapping implements GroupMappingServiceProvider {
public void cacheGroupsAdd(List<String> groups) throws IOException {
throw new UnsupportedOperationException();
}
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return Collections.singleton(user + "group");
- }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
index 129df0e..9c916e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
@@ -21,9 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
-import java.util.Set;
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hadoop.security.GroupMappingServiceProvider;
public class SimpleGroupsMapping implements GroupMappingServiceProvider {
@@ -47,10 +45,4 @@ public class SimpleGroupsMapping implements GroupMappingServiceProvider {
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
}
-
- @Override
- public Set<String> getGroupsSet(String user) throws IOException {
- return ImmutableSet.of(user + "group", user + "subgroup1",
- user + "subgroup2");
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org