You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ve...@apache.org on 2015/12/01 23:46:16 UTC
incubator-ranger git commit: Ranger-742: Made code changes to
complete user search before performing group search. Also,
added check to retrieve groups from user's memberof attribute only when group
search is not enabled.
Repository: incubator-ranger
Updated Branches:
refs/heads/master 7ac6a02a3 -> ec2ea9213
Ranger-742: Made code changes to complete user search before performing group search. Also, added check to retrieve groups from user's memberof attribute only when group search is not enabled.
Signed-off-by: Velmurugan Periasamy <ve...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/ec2ea921
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/ec2ea921
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/ec2ea921
Branch: refs/heads/master
Commit: ec2ea92135cfae32378c377a1fad73039f3e1f3f
Parents: 7ac6a02
Author: Sailaja Polavarapu <sp...@hortonworks.com>
Authored: Mon Nov 30 13:18:59 2015 -0800
Committer: Velmurugan Periasamy <ve...@apache.org>
Committed: Tue Dec 1 14:46:01 2015 -0800
----------------------------------------------------------------------
.../process/LdapUserGroupBuilder.java | 388 ++++++++++---------
1 file changed, 210 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ec2ea921/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
index bab9e84..bb5fad5 100644
--- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
@@ -21,8 +21,11 @@
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
@@ -95,6 +98,7 @@ public class LdapUserGroupBuilder implements UserGroupSource {
Mapper userNameRegExInst = null;
Mapper groupNameRegExInst = null;
+ private List<UserInfo> userGroupMap;
public static void main(String[] args) throws Throwable {
LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder();
@@ -293,61 +297,62 @@ public class LdapUserGroupBuilder implements UserGroupSource {
@Override
public void updateSink(UserGroupSink sink) throws Throwable {
LOG.info("LDAPUserGroupBuilder updateSink started");
- NamingEnumeration<SearchResult> userSearchResultEnum = null;
- NamingEnumeration<SearchResult> groupSearchResultEnum = null;
+ userGroupMap = new ArrayList<UserInfo>();
+ NamingEnumeration<SearchResult> userSearchResultEnum = null;
+ NamingEnumeration<SearchResult> groupSearchResultEnum = null;
try {
createLdapContext();
- int total;
- // Activate paged results
- byte[] cookie = null;
- if (pagedResultsEnabled) {
- ldapContext.setRequestControls(new Control[]{
- new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) });
- }
+ int total;
+ // Activate paged results
+ byte[] cookie = null;
+ if (pagedResultsEnabled) {
+ ldapContext.setRequestControls(new Control[]{
+ new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) });
+ }
int counter = 0;
do {
userSearchResultEnum = ldapContext
- .search(userSearchBase, extendedUserSearchFilter,
- userSearchControls);
+ .search(userSearchBase, extendedUserSearchFilter,
+ userSearchControls);
while (userSearchResultEnum.hasMore()) {
// searchResults contains all the user entries
final SearchResult userEntry = userSearchResultEnum.next();
- if (userEntry == null) {
- if (LOG.isInfoEnabled()) {
- LOG.info("userEntry null, skipping sync for the entry");
- }
- continue;
- }
-
- Attributes attributes = userEntry.getAttributes();
- if (attributes == null) {
- if (LOG.isInfoEnabled()) {
- LOG.info("attributes missing for entry " + userEntry.getNameInNamespace() +
- ", skipping sync");
- }
- continue;
- }
-
- Attribute userNameAttr = attributes.get(userNameAttribute);
- if (userNameAttr == null) {
- if (LOG.isInfoEnabled()) {
- LOG.info(userNameAttribute + " missing for entry " + userEntry.getNameInNamespace() +
- ", skipping sync");
- }
- continue;
- }
+ if (userEntry == null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("userEntry null, skipping sync for the entry");
+ }
+ continue;
+ }
+
+ Attributes attributes = userEntry.getAttributes();
+ if (attributes == null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("attributes missing for entry " + userEntry.getNameInNamespace() +
+ ", skipping sync");
+ }
+ continue;
+ }
+
+ Attribute userNameAttr = attributes.get(userNameAttribute);
+ if (userNameAttr == null) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(userNameAttribute + " missing for entry " + userEntry.getNameInNamespace() +
+ ", skipping sync");
+ }
+ continue;
+ }
String userName = (String) userNameAttr.get();
- if (userName == null || userName.trim().isEmpty()) {
- if (LOG.isInfoEnabled()) {
- LOG.info(userNameAttribute + " empty for entry " + userEntry.getNameInNamespace() +
- ", skipping sync");
- }
- continue;
- }
+ if (userName == null || userName.trim().isEmpty()) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(userNameAttribute + " empty for entry " + userEntry.getNameInNamespace() +
+ ", skipping sync");
+ }
+ continue;
+ }
if (userNameCaseConversionFlag) {
if (userNameLowerCaseFlag) {
@@ -357,73 +362,53 @@ public class LdapUserGroupBuilder implements UserGroupSource {
userName = userName.toUpperCase() ;
}
}
-
+
if (userNameRegExInst != null) {
- userName = userNameRegExInst.transform(userName);
+ userName = userNameRegExInst.transform(userName);
}
- Set<String> groups = new HashSet<String>();
-
- for (String useGroupNameAttribute : userGroupNameAttributeSet) {
- Attribute userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute);
- if (userGroupfAttribute != null) {
- NamingEnumeration<?> groupEnum = userGroupfAttribute.getAll();
- while (groupEnum.hasMore()) {
- String gName = getShortGroupName((String) groupEnum
- .next());
- if (groupNameCaseConversionFlag) {
- if (groupNameLowerCaseFlag) {
- gName = gName.toLowerCase();
- } else {
- gName = gName.toUpperCase();
- }
- }
- if (groupNameRegExInst != null) {
- gName = groupNameRegExInst.transform(gName);
- }
- groups.add(gName);
- }
- }
- }
-
- if (groupSearchEnabled && groupUserMapSyncEnabled) {
- LOG.info("groupSearch and groupUserMapSync are enabled, would search for groups and compute memberships");
- groupSearchResultEnum = ldapContext
- .search(groupSearchBase, extendedGroupSearchFilter,
- new Object[]{userEntry.getNameInNamespace()},
- groupSearchControls);
- Set<String> computedGroups = new HashSet<String>();
- while (groupSearchResultEnum.hasMore()) {
- final SearchResult groupEntry = groupSearchResultEnum.next();
- if (groupEntry != null) {
- String gName = (String) groupEntry.getAttributes()
- .get(groupNameAttribute).get();
- if (groupNameCaseConversionFlag) {
- if (groupNameLowerCaseFlag) {
- gName = gName.toLowerCase();
- } else {
- gName = gName.toUpperCase();
- }
- }
- if (groupNameRegExInst != null) {
- gName = groupNameRegExInst.transform(gName);
- }
- computedGroups.add(gName);
- }
- }
- if (LOG.isInfoEnabled()) {
- LOG.info("computed groups for user: " + userName +", groups: " + computedGroups);
- }
- groups.addAll(computedGroups);
- }
-
- List<String> groupList = new ArrayList<String>(groups);
+ UserInfo userInfo = new UserInfo(userName, userEntry.getNameInNamespace());
+ Set<String> groups = new HashSet<String>();
+
+ // Get all the groups from the group name attribute of the user only when group search is not enabled.
+ if (!groupSearchEnabled) {
+ for (String useGroupNameAttribute : userGroupNameAttributeSet) {
+ Attribute userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute);
+ if (userGroupfAttribute != null) {
+ NamingEnumeration<?> groupEnum = userGroupfAttribute.getAll();
+ while (groupEnum.hasMore()) {
+ String gName = getShortGroupName((String) groupEnum
+ .next());
+ if (groupNameCaseConversionFlag) {
+ if (groupNameLowerCaseFlag) {
+ gName = gName.toLowerCase();
+ } else {
+ gName = gName.toUpperCase();
+ }
+ }
+ if (groupNameRegExInst != null) {
+ gName = groupNameRegExInst.transform(gName);
+ }
+ groups.add(gName);
+ }
+ }
+ }
+ }
+
+ userInfo.addGroups(groups);
+ //populate the userGroupMap with username, userInfo.
+ //userInfo contains details of user that will be later used for
+ //group search to compute group membership as well as to call sink.addOrUpdateUser()
+ userGroupMap.add(userInfo);
+
+ //List<String> groupList = new ArrayList<String>(groups);
+ List<String> groupList = userInfo.getGroups();
counter++;
if (counter <= 2000) {
if (LOG.isInfoEnabled()) {
LOG.info("Updating user count: " + counter
- + ", userName: " + userName + ", groupList: "
- + groupList);
+ + ", userName: " + userName + ", groupList: "
+ + groupList);
}
if ( counter == 2000 ) {
LOG.info("===> 2000 user records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <===");
@@ -439,90 +424,112 @@ public class LdapUserGroupBuilder implements UserGroupSource {
}
}
}
- try {
- sink.addOrUpdateUser(userName, groupList);
- } catch (Throwable t) {
- LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage()
- + ", for user: " + userName
- + ", groups: " + groupList);
- }
+
}
-
+
// Examine the paged results control response
- Control[] controls = ldapContext.getResponseControls();
- if (controls != null) {
- for (int i = 0; i < controls.length; i++) {
- if (controls[i] instanceof PagedResultsResponseControl) {
- PagedResultsResponseControl prrc =
- (PagedResultsResponseControl)controls[i];
- total = prrc.getResultSize();
- if (total != 0) {
- LOG.debug("END-OF-PAGE total : " + total);
- } else {
- LOG.debug("END-OF-PAGE total : unknown");
- }
- cookie = prrc.getCookie();
- }
- }
- } else {
- LOG.debug("No controls were sent from the server");
- }
- // Re-activate paged results
- if (pagedResultsEnabled) {
- ldapContext.setRequestControls(new Control[]{
- new PagedResultsControl(PAGE_SIZE, cookie, Control.CRITICAL) });
- }
+ Control[] controls = ldapContext.getResponseControls();
+ if (controls != null) {
+ for (int i = 0; i < controls.length; i++) {
+ if (controls[i] instanceof PagedResultsResponseControl) {
+ PagedResultsResponseControl prrc =
+ (PagedResultsResponseControl)controls[i];
+ total = prrc.getResultSize();
+ if (total != 0) {
+ LOG.debug("END-OF-PAGE total : " + total);
+ } else {
+ LOG.debug("END-OF-PAGE total : unknown");
+ }
+ cookie = prrc.getCookie();
+ }
+ }
+ } else {
+ LOG.debug("No controls were sent from the server");
+ }
+ // Re-activate paged results
+ if (pagedResultsEnabled) {
+ ldapContext.setRequestControls(new Control[]{
+ new PagedResultsControl(PAGE_SIZE, cookie, Control.CRITICAL) });
+ }
} while (cookie != null);
LOG.info("LDAPUserGroupBuilder.updateSink() completed with user count: "
+ counter);
- if (groupSearchEnabled && !groupUserMapSyncEnabled) {
- if (LOG.isInfoEnabled()) {
- LOG.info("groupSearch enabled and groupUserMapSync not enabled, "
- + "would search for groups, would not compute memberships");
- }
- Set <String> groupNames = new HashSet<String>();
- groupSearchResultEnum = ldapContext
- .search(groupSearchBase, extendedAllGroupsSearchFilter,
- groupSearchControls);
-
- while (groupSearchResultEnum.hasMore()) {
- final SearchResult groupEntry = groupSearchResultEnum.next();
- if (groupEntry.getAttributes().get(groupNameAttribute) == null) {
- continue;
- }
- String gName = (String) groupEntry.getAttributes()
- .get(groupNameAttribute).get();
- if (groupNameCaseConversionFlag) {
- if (groupNameLowerCaseFlag) {
- gName = gName.toLowerCase();
- } else {
- gName = gName.toUpperCase();
- }
- }
- if (groupNameRegExInst != null) {
- gName = groupNameRegExInst.transform(gName);
- }
- groupNames.add(gName);
- }
- if (LOG.isInfoEnabled()) {
- LOG.info("found groups from ldap source: " + groupNames);
- }
-
- // TODO: push groupNames to ranger
- // POST http://<IP>:6080/service/xusers/secure/groups create group
- // PUT http://<IP>:6080/service/xusers/secure/groups/{id} update group
- // sink.addOrUpdateUser(groupNames);
-
- }
+
} finally {
- if (userSearchResultEnum != null) {
- userSearchResultEnum.close();
- }
- if (groupSearchResultEnum != null) {
- groupSearchResultEnum.close();
- }
+ if (userSearchResultEnum != null) {
+ userSearchResultEnum.close();
+ }
+ if (groupSearchResultEnum != null) {
+ groupSearchResultEnum.close();
+ }
+ closeLdapContext();
+ }
+ // Perform group search
+ getUserGroups(sink);
+ }
+
+ private void getUserGroups(UserGroupSink sink) throws Throwable {
+ NamingEnumeration<SearchResult> groupSearchResultEnum = null;
+ LOG.debug("Total No. of users saved = " + userGroupMap.size());
+ if (groupSearchEnabled && groupUserMapSyncEnabled) {
+ LOG.info("groupSearch and groupUserMapSync are enabled, would search for groups and compute memberships");
+ createLdapContext();
+ }
+
+ Iterator<UserInfo> userInfoIterator = userGroupMap.iterator();
+ while(userInfoIterator.hasNext()) {
+ UserInfo userInfo = userInfoIterator.next();
+ String userName = userInfo.getUserName();
+ if (groupSearchEnabled && groupUserMapSyncEnabled) {
+ //LOG.info("groupSearch and groupUserMapSync are enabled, would search for groups and compute memberships");
+ try {
+
+ groupSearchResultEnum = ldapContext
+ .search(groupSearchBase, extendedGroupSearchFilter,
+ new Object[]{userInfo.getUserFullName()},
+ groupSearchControls);
+ Set<String> computedGroups = new HashSet<String>();
+ while (groupSearchResultEnum.hasMore()) {
+ final SearchResult groupEntry = groupSearchResultEnum.next();
+ if (groupEntry != null) {
+ String gName = (String) groupEntry.getAttributes()
+ .get(groupNameAttribute).get();
+ if (groupNameCaseConversionFlag) {
+ if (groupNameLowerCaseFlag) {
+ gName = gName.toLowerCase();
+ } else {
+ gName = gName.toUpperCase();
+ }
+ }
+ if (groupNameRegExInst != null) {
+ gName = groupNameRegExInst.transform(gName);
+ }
+ computedGroups.add(gName);
+ }
+ }
+ if (LOG.isInfoEnabled()) {
+ LOG.info("computed groups for user: " + userName +", groups: " + computedGroups);
+ }
+ userInfo.addGroups(computedGroups);
+
+ } finally {
+ if (groupSearchResultEnum != null) {
+ groupSearchResultEnum.close();
+ }
+ }
+ }
+ List<String> groupList = userInfo.getGroups();
+ try {
+ sink.addOrUpdateUser(userName, groupList);
+ } catch (Throwable t) {
+ LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage()
+ + ", for user: " + userName
+ + ", groups: " + groupList);
+ }
+ }
+ if (groupSearchEnabled && groupUserMapSyncEnabled) {
closeLdapContext();
}
}
@@ -544,3 +551,28 @@ public class LdapUserGroupBuilder implements UserGroupSource {
}
}
+
+class UserInfo {
+ private String userName;
+ private String userFullName;
+ private Set<String> groupList;
+
+ public UserInfo(String userName, String userFullName) {
+ this.userName = userName;
+ this.userFullName = userFullName;
+ this.groupList = new HashSet<String>();
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+ public String getUserFullName() {
+ return userFullName;
+ }
+ public void addGroups(Set<String> groups) {
+ groupList.addAll(groups);
+ }
+ public List<String> getGroups() {
+ return (new ArrayList<String>(groupList));
+ }
+}