You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by dh...@apache.org on 2026/05/22 14:56:24 UTC
(ranger) branch ds_RANGER-5427_adsync created (now b19478a6d)
This is an automated email from the ASF dual-hosted git repository.
dhavalshah9131 pushed a change to branch ds_RANGER-5427_adsync
in repository https://gitbox.apache.org/repos/asf/ranger.git
at b19478a6d RANGER-5427 : AD Groups with 1500+ Users Fail to Sync into Ranger Admin via RangerUserSync
This branch includes the following new commits:
new b19478a6d RANGER-5427 : AD Groups with 1500+ Users Fail to Sync into Ranger Admin via RangerUserSync
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(ranger) 01/01: RANGER-5427 : AD Groups with 1500+ Users Fail to Sync into Ranger Admin via RangerUserSync
Posted by dh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dhavalshah9131 pushed a commit to branch ds_RANGER-5427_adsync
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit b19478a6d05ee9840af5a272b3abec2d3f9ab228
Author: Dhaval Shah <dh...@gmail.com>
AuthorDate: Fri May 22 20:24:11 2026 +0530
RANGER-5427 : AD Groups with 1500+ Users Fail to Sync into Ranger Admin via RangerUserSync
---
.../ugsyncutil/util/UgsyncCommonConstants.java | 6 +-
.../ldapusersync/process/LdapUserGroupBuilder.java | 392 ++++++++++++---------
.../unixusersync/config/UserGroupSyncConfig.java | 15 +
.../process/PolicyMgrUserGroupBuilder.java | 3 +-
.../process/TestLdapUserGroupBuilder.java | 272 +++++++++++++-
5 files changed, 517 insertions(+), 171 deletions(-)
diff --git a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java
index 6f8905a35..c86e2dba3 100644
--- a/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java
+++ b/ugsync-util/src/main/java/org/apache/ranger/ugsyncutil/util/UgsyncCommonConstants.java
@@ -22,10 +22,14 @@
public class UgsyncCommonConstants {
public enum CaseConversion { NONE, TO_LOWER, TO_UPPER }
- public static final String ORIGINAL_NAME = "original_name";
+ public static final String ORIGINAL_NAME = "original_name";
public static final String FULL_NAME = "full_name";
public static final String SYNC_SOURCE = "sync_source";
public static final String LDAP_URL = "ldap_url";
+ public static final String PRINCIPAL = "ranger.usersync.kerberos.principal";
+ public static final String AUTHENTICATION_TYPE = "hadoop.security.authentication";
+ public static final String KEYTAB = "ranger.usersync.kerberos.keytab";
+ public static final String NAME_RULE = "hadoop.security.auth_to_local";
public static final String UGSYNC_NONE_CASE_CONVERSION_VALUE = "none";
public static final String UGSYNC_LOWER_CASE_CONVERSION_VALUE = "lower";
diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
index 7f1463ce7..afcd7946b 100644
--- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
@@ -21,6 +21,7 @@
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.security.SecureClientLogin;
import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable;
import org.apache.hadoop.thirdparty.com.google.common.collect.Table;
import org.apache.ranger.ugsyncutil.model.LdapSyncSourceInfo;
@@ -49,7 +50,11 @@
import javax.naming.ldap.Rdn;
import javax.naming.ldap.StartTlsRequest;
import javax.naming.ldap.StartTlsResponse;
+import javax.security.auth.Subject;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.security.PrivilegedAction;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
@@ -70,6 +75,7 @@ public class LdapUserGroupBuilder implements UserGroupSource {
private static final String DATE_FORMAT = "yyyyMMddHHmmss";
private static final String MEMBER_OF_ATTR = "memberof=";
private static final String GROUP_NAME_ATTRIBUTE = "cn=";
+ public static String localHostname = "unknown";
private static final int PAGE_SIZE = 500;
/* for AD uSNChanged */
@@ -98,6 +104,9 @@ public class LdapUserGroupBuilder implements UserGroupSource {
private String ldapBindDn;
private String ldapBindPassword;
private String ldapAuthenticationMechanism;
+ private String principal;
+ private String keytab;
+ private String nameRules;
private String ldapReferral;
private String searchBase;
private String userNameAttribute;
@@ -280,7 +289,7 @@ public void updateSink(UserGroupSink sink) throws Throwable {
}
}
- private void createLdapContext() throws Throwable {
+ private void initializeLdapContext() throws Throwable {
Properties env = new Properties();
env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory");
@@ -318,7 +327,16 @@ private void createLdapContext() throws Throwable {
}
}
- ldapContext = new InitialLdapContext(env, null);
+ env.put(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism);
+ env.put(Context.SECURITY_PRINCIPAL, ldapBindDn);
+ env.put(Context.SECURITY_CREDENTIALS, ldapBindPassword);
+ env.put(Context.REFERRAL, ldapReferral);
+
+ try {
+ ldapContext = new InitialLdapContext(env, null);
+ } catch (NamingException e) {
+ throw e;
+ }
if (!ldapUrl.startsWith("ldaps")) {
if (config.isStartTlsEnabled()) {
@@ -333,11 +351,22 @@ private void createLdapContext() throws Throwable {
LOG.info("Starting TLS session...");
}
}
+ }
- ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapBindDn);
- ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS, ldapBindPassword);
- ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism);
- ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral);
+ private void createLdapContext() throws Throwable {
+ if (ldapAuthenticationMechanism.equalsIgnoreCase("GSSAPI")) {
+ Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules);
+ Subject.doAs(sub, (PrivilegedAction<Void>) () -> {
+ try {
+ initializeLdapContext();
+ } catch (Throwable e) {
+ LOG.error("Failed to create LDAP context: ", e);
+ }
+ return null;
+ });
+ } else {
+ initializeLdapContext();
+ }
}
private void setConfig() throws Throwable {
@@ -351,6 +380,13 @@ private void setConfig() throws Throwable {
ldapBindDn = config.getLdapBindDn();
ldapBindPassword = config.getLdapBindPassword();
ldapAuthenticationMechanism = config.getLdapAuthenticationMechanism();
+ try {
+ principal = SecureClientLogin.getPrincipal(config.getProperty(UgsyncCommonConstants.PRINCIPAL, ""), localHostname);
+ } catch (IOException ignored) {
+ // do nothing
+ }
+ keytab = config.getProperty(UgsyncCommonConstants.KEYTAB, "");
+ nameRules = config.getProperty(UgsyncCommonConstants.NAME_RULE, "DEFAULT");
ldapReferral = config.getContextReferral();
searchBase = config.getSearchBase();
userSearchBase = config.getUserSearchBase().split(";");
@@ -649,28 +685,7 @@ private long getUsers(boolean computeDeletes) throws Throwable {
}
}
- // Examine the paged results control response
- Control[] controls = ldapContext.getResponseControls();
-
- if (controls != null) {
- for (Control control : controls) {
- if (control instanceof PagedResultsResponseControl) {
- PagedResultsResponseControl prrc = (PagedResultsResponseControl) control;
-
- total = prrc.getResultSize();
-
- if (total != 0) {
- LOG.debug("END-OF-PAGE total : {}", total);
- } else {
- LOG.debug("END-OF-PAGE total : unknown");
- }
-
- cookie = prrc.getCookie();
- }
- }
- } else {
- LOG.debug("No controls were sent from the server");
- }
+ cookie = handlePagedResults();
// Re-activate paged results
if (pagedResultsEnabled) {
@@ -708,6 +723,7 @@ private long getGroups(boolean computeDeletes) throws Throwable {
NamingEnumeration<SearchResult> groupSearchResultEnum = null;
DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
long highestdeltaSyncGroupTime = deltaSyncGroupTime;
+ Map<String, Set<String>> largeGroups = new HashMap<>();
try {
createLdapContext();
@@ -741,15 +757,16 @@ private long getGroups(boolean computeDeletes) throws Throwable {
LOG.info("extendedAllGroupsSearchFilter = {}", extendedAllGroupsSearchFilter);
- for (String s : groupSearchBase) {
+ for (int ou = 0; ou < groupSearchBase.length; ou++) {
byte[] cookie = null;
int counter = 0;
+ Set<String> largeGroupNames = new HashSet<>();
try {
int paged = 0;
do {
- groupSearchResultEnum = ldapContext.search(s, extendedAllGroupsSearchFilter, groupSearchControls);
+ groupSearchResultEnum = ldapContext.search(groupSearchBase[ou], extendedAllGroupsSearchFilter, groupSearchControls);
while (groupSearchResultEnum.hasMore()) {
final SearchResult groupEntry = groupSearchResultEnum.next();
@@ -823,71 +840,22 @@ private long getGroups(boolean computeDeletes) throws Throwable {
int userCount = 0;
if (groupMemberAttr == null || groupMemberAttr.size() <= 0) {
- try {
- LOG.info("No members available for {}", gName);
- } catch (Exception e) {
- throw new RuntimeException(e);
+ if (config.isLargeGroupSyncEnabled()) {
+ groupMemberAttr = attributes.get("member;range=0-1499");
}
-
- sourceGroupUsers.put(groupFullName, new HashSet<>());
- continue;
- }
-
- NamingEnumeration<?> userEnum = groupMemberAttr.getAll();
-
- while (userEnum.hasMore()) {
- String originalUserFullName = (String) userEnum.next();
-
- if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) {
+ if (groupMemberAttr == null || groupMemberAttr.size() <= 0) {
+ LOG.info("No members available for {}", gName);
sourceGroupUsers.put(groupFullName, new HashSet<>());
continue;
+ } else {
+ largeGroupNames.add(gName);
}
-
- userCount++;
-
- if (!userSearchEnabled) {
- Map<String, String> userAttrMap = new HashMap<>();
- String userName = getShortName(originalUserFullName);
-
- userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
- userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName);
- userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
- userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
-
- sourceUsers.put(originalUserFullName, userAttrMap);
-
- LOG.debug("As usersearch is disabled, adding user {} from group member attribute for group {}", userName, gName);
- }
-
- groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName);
}
+ userCount = processGroupMembers(groupMemberAttr, groupFullName);
LOG.info("No. of members in the group {} = {}", gName, userCount);
}
-
- // Examine the paged results control response
- Control[] controls = ldapContext.getResponseControls();
-
- if (controls != null) {
- for (Control control : controls) {
- if (control instanceof PagedResultsResponseControl) {
- PagedResultsResponseControl prrc = (PagedResultsResponseControl) control;
-
- total = prrc.getResultSize();
-
- if (total != 0) {
- LOG.debug("END-OF-PAGE total: {}", total);
- } else {
- LOG.debug("END-OF-PAGE total: unknown");
- }
-
- cookie = prrc.getCookie();
- }
- }
- } else {
- LOG.debug("No controls were sent from the server");
- }
-
+ cookie = handlePagedResults();
// Re-activate paged results
if (pagedResultsEnabled) {
LOG.debug("Fetched paged results round: {}", ++paged);
@@ -901,6 +869,10 @@ private long getGroups(boolean computeDeletes) throws Throwable {
LOG.error("LdapUserGroupBuilder.getGroups() failed with exception: ", t);
LOG.info("LdapUserGroupBuilder.getGroups() group count: {}", counter);
}
+ if (config.isLargeGroupSyncEnabled() && !largeGroupNames.isEmpty()) {
+ LOG.info("Large groups found: {}", largeGroupNames);
+ largeGroups.put(groupSearchBase[ou], largeGroupNames);
+ }
}
} finally {
if (groupSearchResultEnum != null) {
@@ -908,6 +880,10 @@ private long getGroups(boolean computeDeletes) throws Throwable {
}
closeLdapContext();
+ if (config.isLargeGroupSyncEnabled() && MapUtils.isNotEmpty(largeGroups)) {
+ LOG.info("Large groups found: {}", largeGroups);
+ getRemainingGroupMembers(largeGroups);
+ }
}
if (groupHierarchyLevels > 0) {
@@ -1056,59 +1032,10 @@ private void goUpGroupHierarchyLdap(Set<String> groupDNs, int groupHierarchyLeve
}
sourceGroups.put(groupFullName, groupAttrMap);
-
- NamingEnumeration<?> userEnum = groupMemberAttr.getAll();
-
- while (userEnum.hasMore()) {
- String originalUserFullName = (String) userEnum.next();
-
- if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) {
- continue;
- }
-
- userCount++;
-
- if (!userSearchEnabled && !sourceGroups.containsKey(originalUserFullName)) {
- Map<String, String> userAttrMap = new HashMap<>();
- String userName = getShortName(originalUserFullName);
-
- userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
- userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName);
- userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
- userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
-
- sourceUsers.put(originalUserFullName, userAttrMap);
- }
-
- groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName);
- }
-
+ userCount = processGroupMembers(groupMemberAttr, groupFullName);
LOG.info("No. of members in the group {} = {}", gName, userCount);
}
-
- // Examine the paged results control response
- Control[] controls = ldapContext.getResponseControls();
-
- if (controls != null) {
- for (Control control : controls) {
- if (control instanceof PagedResultsResponseControl) {
- PagedResultsResponseControl prrc = (PagedResultsResponseControl) control;
-
- total = prrc.getResultSize();
-
- if (total != 0) {
- LOG.debug("END-OF-PAGE total : {}", total);
- } else {
- LOG.debug("END-OF-PAGE total : unknown");
- }
-
- cookie = prrc.getCookie();
- }
- }
- } else {
- LOG.debug("No controls were sent from the server");
- }
-
+ cookie = handlePagedResults();
// Re-activate paged results
if (pagedResultsEnabled) {
ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, cookie, Control.CRITICAL)});
@@ -1149,7 +1076,7 @@ private void addToAttrMap(Map<String, String> userAttrMap, String attrName, Attr
} catch (ClassCastException e) {
LOG.error("{} type is not set properly {}", attrName, e.getMessage());
}
- } else if (attrType.equals("String")) {
+ } else if (attrType.equalsIgnoreCase("String")) {
userAttrMap.put(attrName, (String) attr.get());
} else {
LOG.warn("Attribute Type {} not supported for {}", attrType, attrName);
@@ -1208,6 +1135,160 @@ private static String getShortName(String longName) {
return shortName;
}
+ private byte[] handlePagedResults() throws Throwable {
+ Control[] controls = ldapContext.getResponseControls();
+
+ if (controls != null) {
+ for (int i = 0; i < controls.length; i++) {
+ if (controls[i] instanceof PagedResultsResponseControl) {
+ PagedResultsResponseControl prrc = (PagedResultsResponseControl) controls[i];
+ LOG.debug("END-OF-PAGE total: {}", prrc.getResultSize());
+ return prrc.getCookie();
+ }
+ }
+ } else {
+ LOG.debug("No controls were sent from the server");
+ }
+ return null;
+ }
+
+ private int processGroupMembers(Attribute groupMemberAttr, String groupFullName) throws Throwable {
+ NamingEnumeration<?> userEnum = groupMemberAttr.getAll();
+ int userCount = 0;
+ while (userEnum.hasMore()) {
+ String originalUserFullName = (String) userEnum.next();
+
+ if (originalUserFullName == null || originalUserFullName.trim().isEmpty()) {
+ sourceGroupUsers.put(groupFullName, new HashSet<>());
+ continue;
+ }
+
+ userCount++;
+
+ if (!userSearchEnabled) {
+ Map<String, String> userAttrMap = new HashMap<>();
+ String userName = getShortName(originalUserFullName);
+
+ userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
+ userAttrMap.put(UgsyncCommonConstants.FULL_NAME, originalUserFullName);
+ userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
+ userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
+
+ sourceUsers.put(originalUserFullName, userAttrMap);
+
+ LOG.debug("As usersearch is disabled, adding user {} from group member attribute for group {}", userName, groupFullName);
+ }
+
+ groupUserTable.put(groupFullName, originalUserFullName, originalUserFullName);
+ }
+ return userCount;
+ }
+
+ private void getRemainingGroupMembers(Map<String, Set<String>> largeGroups) throws Throwable {
+ NamingEnumeration<SearchResult> groupSearchResultEnum = null;
+
+ try {
+ createLdapContext();
+
+ int counter = 0;
+
+ // Loop through each OU
+ for (String ou : largeGroups.keySet()) {
+ Set<String> groupNames = largeGroups.get(ou);
+
+ // Process each group individually
+ for (String groupName : groupNames) {
+ LOG.info("Processing large group: {}", groupName);
+
+ // Build filter for THIS group only
+ String groupFilter = "(&(objectclass=" + groupObjectClass + ")";
+
+ // Add custom filter if present
+ if (groupSearchFilter != null && !groupSearchFilter.trim().isEmpty()) {
+ String customFilter = groupSearchFilter.trim();
+ if (!customFilter.startsWith("(")) {
+ customFilter = "(" + customFilter + ")";
+ }
+ groupFilter += customFilter;
+ }
+
+ // Add the specific group name
+ groupFilter += "(CN=" + groupName + "))";
+
+ LOG.info("Group filter for {}: {}", groupName, groupFilter);
+
+ try {
+ int rangeStep = 1500;
+ int start = 1500;
+ boolean lastRange = false;
+
+ // Loop until we get all members for THIS group
+ while (!lastRange) {
+ // Build the range attribute name
+ String rangeAttr = "member;range=" + start + "-" + (start + rangeStep - 1);
+ String[] attributes = {rangeAttr};
+
+ LOG.info("Fetching range for {}: {}", groupName, rangeAttr);
+
+ // Prepare search controls
+ SearchControls controls = new SearchControls();
+ controls.setSearchScope(SearchControls.SUBTREE_SCOPE);
+ controls.setReturningAttributes(attributes);
+
+ // Search for this specific group
+ groupSearchResultEnum = ldapContext.search(ou, groupFilter, controls);
+
+ while (groupSearchResultEnum.hasMore()) {
+ final SearchResult result = groupSearchResultEnum.next();
+ Attributes attrs = result.getAttributes();
+ String groupFullName = result.getNameInNamespace();
+
+ LOG.info("Processing range for group: {}", groupFullName);
+
+ NamingEnumeration<? extends Attribute> attrEnum = attrs.getAll();
+
+ while (attrEnum.hasMore()) {
+ Attribute attr = attrEnum.next();
+ String attrID = attr.getID();
+
+ LOG.info("Attribute ID: {}", attrID);
+
+ // Check if this is the last range for THIS group
+ if (attrID.endsWith("*")) {
+ lastRange = true;
+ LOG.info("Last range reached for group: {}", groupName);
+ }
+
+ int userCount = processGroupMembers(attr, groupFullName);
+ LOG.info("No. of members in the group {} for range {} = {}", groupFullName, rangeAttr, userCount);
+ }
+
+ counter++;
+ }
+
+ // Move to next range
+ start += rangeStep;
+ }
+
+ LOG.info("Completed processing group: {} with total iterations: {}", groupName, counter);
+ } catch (Exception e) {
+ LOG.error("Error processing group {}: {}", groupName, e.getMessage(), e);
+ }
+ }
+ }
+
+ LOG.info("Completed processing all large groups");
+ } catch (Throwable t) {
+ LOG.error("LdapUserGroupBuilder.getRemainingGroupMembers() failed with exception:", t);
+ throw t;
+ } finally {
+ if (groupSearchResultEnum != null) {
+ groupSearchResultEnum.close();
+ }
+ closeLdapContext();
+ }
+ }
+
private String getFirstRDN(String name) {
if (StringUtils.isEmpty(name)) {
return null;
@@ -1333,30 +1414,7 @@ private String getDNForMemberOf(String searchFilter) throws Throwable {
}
}
}
-
- // Examine the paged results control response
- Control[] controls = ldapContext.getResponseControls();
-
- if (controls != null) {
- for (Control control : controls) {
- if (control instanceof PagedResultsResponseControl) {
- PagedResultsResponseControl prrc = (PagedResultsResponseControl) control;
-
- total = prrc.getResultSize();
-
- if (total != 0) {
- LOG.debug("END-OF-PAGE total : {}", total);
- } else {
- LOG.debug("END-OF-PAGE total : unknown");
- }
-
- cookie = prrc.getCookie();
- }
- }
- } else {
- LOG.debug("No controls were sent from the server");
- }
-
+ cookie = handlePagedResults();
// Re-activate paged results
if (pagedResultsEnabled) {
LOG.debug("Fetched paged results round: {}", ++paged);
@@ -1384,4 +1442,12 @@ private String getDNForMemberOf(String searchFilter) throws Throwable {
return computedSearchFilter;
}
+
+ static {
+ try {
+ localHostname = java.net.InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ localHostname = "unknown";
+ }
+ }
}
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
index 08ca72500..eb501ea12 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
@@ -137,6 +137,8 @@ public class UserGroupSyncConfig {
private static final int DEFAULT_LGSYNC_GROUP_HIERARCHY_LEVELS = 0;
private static final int DEFAULT_LGSYNC_PAGED_RESULTS_SIZE = 500;
private static final boolean DEFAULT_LGSYNC_LDAP_DELTASYNC_ENABLED = false;
+ private static final String LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED = "ranger.usersync.ldap.largegroupsync";
+ private static final boolean DEFAULT_LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED = false;
private static final boolean DEFAULT_LGSYNC_LDAP_STARTTLS_ENABLED = false;
private static final boolean DEFAULT_LGSYNC_PAGED_RESULTS_ENABLED = true;
private static final boolean DEFAULT_LGSYNC_GROUP_SEARCH_ENABLED = true;
@@ -1389,6 +1391,19 @@ public boolean isSyncSourceValidationEnabled() {
return isSyncSourceValidationEnabled;
}
+ public boolean isLargeGroupSyncEnabled() {
+ boolean largeGroupSyncEnabled;
+ String val = prop.getProperty(LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED);
+
+ if (val == null || val.trim().isEmpty()) {
+ largeGroupSyncEnabled = DEFAULT_LGSYNC_LDAP_LARGEGROUPSYNC_ENABLED;
+ } else {
+ largeGroupSyncEnabled = Boolean.valueOf(val);
+ }
+
+ return largeGroupSyncEnabled;
+ }
+
private void init() {
XMLUtils.loadConfig(DEFAULT_CONFIG_FILE, prop);
XMLUtils.loadConfig(CORE_SITE_CONFIG_FILE, prop);
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
index ad8170efd..b42700cd8 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
@@ -81,7 +81,6 @@ public class PolicyMgrUserGroupBuilder extends AbstractUserGroupSource implement
public static final String PM_UPDATE_USERS_ROLES_URI = "/service/xusers/users/roleassignments";
/* ******************* */
- private static final String AUTHENTICATION_TYPE = "hadoop.security.authentication";
private static final String AUTH_KERBEROS = "kerberos";
private static final String KERBEROS_PRINCIPAL = "ranger.usersync.kerberos.principal";
private static final String KERBEROS_KEYTAB = "ranger.usersync.kerberos.keytab";
@@ -205,7 +204,7 @@ public synchronized void init() throws Throwable {
String keyStoreType = config.getSSLKeyStoreType();
String trustStoreType = config.getSSLTrustStoreType();
- authenticationType = config.getProperty(AUTHENTICATION_TYPE, "simple");
+ authenticationType = config.getProperty(UgsyncCommonConstants.AUTHENTICATION_TYPE, "simple");
try {
principal = SecureClientLogin.getPrincipal(config.getProperty(KERBEROS_PRINCIPAL, ""), localHostname);
diff --git a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java
index 0b9624390..4c2af85af 100644
--- a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java
+++ b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java
@@ -31,15 +31,19 @@
import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo;
import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
import org.apache.ranger.usergroupsync.UserGroupSink;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.MockedConstruction;
+import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import javax.naming.NamingEnumeration;
+import javax.naming.NamingException;
import javax.naming.directory.Attribute;
import javax.naming.directory.Attributes;
import javax.naming.directory.BasicAttribute;
@@ -87,7 +91,8 @@
@CreateLdapConnectionPool(maxActive = 1, maxWait = 5000)
@ApplyLdifFiles("ADSchema.ldif")
public class TestLdapUserGroupBuilder extends AbstractLdapTestUnit {
- private static final String STR_EPOCH = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(0));
+ private static final String STR_EPOCH = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(0));
+ private MockedStatic<UserGroupSyncConfig> mockedConfig;
@Test
public void testA_init_and_isChanged() throws Exception {
@@ -530,6 +535,7 @@ public void testW_getGroups_processes_groups_and_memberships() throws Throwable
configureMinimalLdapConfig();
UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
cfg.setProperty("ranger.usersync.ldap.group.searchfilter", "cn=Group*");
+ cfg.setProperty("ranger.usersync.ldap.largegroupsync", "true");
LdapUserGroupBuilder b = new LdapUserGroupBuilder();
b.init();
@@ -667,7 +673,7 @@ public void testX_goUpGroupHierarchyLdap_fetches_nested_groups_and_members() thr
configureMinimalLdapConfig();
UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
cfg.setProperty("ranger.usersync.ldap.group.searchfilter", "cn=Nested*");
-
+ cfg.setProperty("ranger.usersync.ldap.groupattributelist", "");
LdapUserGroupBuilder b = new LdapUserGroupBuilder();
b.init();
@@ -677,15 +683,18 @@ public void testX_goUpGroupHierarchyLdap_fetches_nested_groups_and_members() thr
fUserEnabled.setBoolean(b, false);
// Prepare internal structures used by goUpGroupHierarchyLdap
- Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
- Field srcUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceUsers");
- Field srcGroupsF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroups");
+ Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceUsers");
+ Field srcGroupsF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroups");
+ Field srcGroupUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers");
gutF.setAccessible(true);
srcUsersF.setAccessible(true);
srcGroupsF.setAccessible(true);
+ srcGroupUsersF.setAccessible(true);
gutF.set(b, HashBasedTable.create());
srcUsersF.set(b, new HashMap<>());
srcGroupsF.set(b, new HashMap<>());
+ srcGroupUsersF.set(b, new HashMap<>());
// Parent group DN used to build filter
final String parentGroupDn = "CN=Parent,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com";
@@ -867,6 +876,259 @@ public SearchResult nextElement() {
}
}
+ @BeforeEach
+ void setupActiveState() {
+ // Mock the static method to always return true for active state
+ mockedConfig = Mockito.mockStatic(UserGroupSyncConfig.class, Mockito.CALLS_REAL_METHODS);
+ mockedConfig.when(UserGroupSyncConfig::isUgsyncServiceActive).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDownActiveState() {
+ if (mockedConfig != null) {
+ mockedConfig.close();
+ }
+ }
+
+ @Test
+ void testZ_largeGroupSyncEnabled_detectsInitialRange() throws Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+ UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
+ // FIX: Use the correct property name
+ cfg.setProperty("ranger.usersync.ldap.largegroupsync", "true");
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Verify that large group sync is enabled in config
+ assertTrue(cfg.isLargeGroupSyncEnabled(), "Large group sync should be enabled");
+
+ // Verify the processGroupMembers method exists
+ Method processGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers", Attribute.class, String.class);
+ assertNotNull(processGroupMembers, "processGroupMembers method should exist");
+
+ // Verify getRemainingGroupMembers method exists (new method for large groups)
+ Method getRemainingGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("getRemainingGroupMembers", Map.class);
+ assertNotNull(getRemainingGroupMembers, "getRemainingGroupMembers method should exist");
+
+ // Verify that group search controls include the ranged attribute when large group sync is enabled
+ Field groupSearchControlsF = LdapUserGroupBuilder.class.getDeclaredField("groupSearchControls");
+ groupSearchControlsF.setAccessible(true);
+ SearchControls controls = (SearchControls) groupSearchControlsF.get(b);
+ String[] returningAttrs = controls.getReturningAttributes();
+
+ // Check if "member;range=0-1499" is in the returning attributes
+ boolean hasRangeAttr = false;
+ for (String attr : returningAttrs) {
+ if (attr.equals("member;range=0-1499")) {
+ hasRangeAttr = true;
+ break;
+ }
+ }
+ assertFalse(hasRangeAttr, "Should NOT include member;range=0-1499 in initial search - it prevents AD from returning regular member attribute for small groups");
+ }
+
+ @Test
+ void testZA_getRemainingGroupMembers_handles_ranged_attributes() throws Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+ UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
+ cfg.setProperty("ranger.usersync.ldap.largegroupsync", "true");
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Prepare internal structures
+ Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceUsers");
+ Field srcGroupsF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroups");
+ gutF.setAccessible(true);
+ srcUsersF.setAccessible(true);
+ srcGroupsF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+ srcUsersF.set(b, new HashMap<>());
+ srcGroupsF.set(b, new HashMap<>());
+
+ Method getRemainingMembers = LdapUserGroupBuilder.class.getDeclaredMethod("getRemainingGroupMembers", Map.class);
+ getRemainingMembers.setAccessible(true);
+
+ // This will throw because it tries to create LDAP context
+ // Test that it throws Throwable (which is expected without a real LDAP server)
+ Map<String, Set<String>> emptyMap = new HashMap<>();
+ assertThrows(Throwable.class, () -> getRemainingMembers.invoke(b, emptyMap));
+ }
+
+ @Test
+ void testZB_processGroupMembers_handles_empty_members() throws Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Prepare internal structures
+ Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcGroupUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers");
+ gutF.setAccessible(true);
+ srcGroupUsersF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+ srcGroupUsersF.set(b, new HashMap<>());
+
+ Method processGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers", Attribute.class, String.class);
+ processGroupMembers.setAccessible(true);
+
+ final String groupDn = "CN=TestGroup,OU=Groups,DC=example,DC=com";
+
+ // Create attribute with mix of empty and valid members
+ BasicAttribute members = new BasicAttribute("member");
+ members.add(""); // empty member - causes sourceGroupUsers.put and continue
+ members.add("uid=validuser,ou=people,dc=example,dc=com");
+
+ int count = (Integer) processGroupMembers.invoke(b, members, groupDn);
+
+ // Only the valid member is counted (empty member triggers continue before userCount++)
+ assertEquals(1, count);
+
+ @SuppressWarnings("unchecked")
+ Table<String, String, String> gut = (Table<String, String, String>) gutF.get(b);
+
+ // Only valid user should be in the table
+ assertFalse(gut.contains(groupDn, ""));
+ assertTrue(gut.contains(groupDn, "uid=validuser,ou=people,dc=example,dc=com"));
+
+ // sourceGroupUsers should have been set (from the empty member check)
+ @SuppressWarnings("unchecked")
+ Map<String, Set<String>> srcGroupUsers = (Map<String, Set<String>>) srcGroupUsersF.get(b);
+ assertTrue(srcGroupUsers.containsKey(groupDn));
+ }
+
+ @Test
+ void testZC_largeGroupSync_disabled_skips_remaining_members() throws Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+ UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
+ cfg.setProperty("ranger.usersync.ldap.group.searchfilter", "cn=LargeGroup*");
+ cfg.setProperty("ranger.usersync.large.group.sync.enabled", "false");
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Verify that large group sync is disabled
+ assertFalse(cfg.isLargeGroupSyncEnabled());
+
+ // Test that with disabled large group sync, processGroupMembers still works normally
+ Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcGroupUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers");
+ gutF.setAccessible(true);
+ srcGroupUsersF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+ srcGroupUsersF.set(b, new HashMap<>());
+
+ Method processGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers", Attribute.class, String.class);
+ processGroupMembers.setAccessible(true);
+
+ final String groupDn = "CN=NormalGroup,OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com";
+ BasicAttribute members = new BasicAttribute("member");
+ members.add("uid=user1,ou=people,dc=example,dc=com");
+ members.add("uid=user2,ou=people,dc=example,dc=com");
+
+ int count = (Integer) processGroupMembers.invoke(b, members, groupDn);
+ assertEquals(2, count);
+ }
+
+ @Test
+ void testZD_getRemainingGroupMembers_handles_exceptions() throws Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Prepare internal structures
+ Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ gutF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+
+ try (MockedConstruction<InitialLdapContext> mocked = Mockito.mockConstruction(InitialLdapContext.class, (mock, ctx) -> {
+ Mockito.when(mock.search(Mockito.anyString(), Mockito.anyString(), Mockito.any(SearchControls.class)))
+ .thenThrow(new NamingException("Simulated LDAP error"));
+ Mockito.when(mock.getResponseControls()).thenReturn(null);
+ })) {
+ Map<String, Set<String>> largeGroups = new HashMap<>();
+ Set<String> groupNames = new HashSet<>();
+ groupNames.add("FailingGroup");
+ largeGroups.put("OU=Groups,DC=ranger,DC=qe,DC=hortonworks,DC=com", groupNames);
+
+ Method getRemainingMembers = LdapUserGroupBuilder.class.getDeclaredMethod("getRemainingGroupMembers", Map.class);
+ getRemainingMembers.setAccessible(true);
+
+ // Should not throw, but log error and continue
+ assertDoesNotThrow(() -> getRemainingMembers.invoke(b, largeGroups));
+ }
+ }
+
+ @Test
+ void testZE_processGroupMembers_handles_members_correctly() throws Throwable {
+ resetConfig();
+ configureMinimalLdapConfig();
+
+ LdapUserGroupBuilder b = new LdapUserGroupBuilder();
+ b.init();
+
+ // Disable user search so members are added to sourceUsers directly
+ Field fUserEnabled = LdapUserGroupBuilder.class.getDeclaredField("userSearchEnabled");
+ Field currentSyncSourceF = LdapUserGroupBuilder.class.getDeclaredField("currentSyncSource");
+ fUserEnabled.setAccessible(true);
+ currentSyncSourceF.setAccessible(true);
+ fUserEnabled.setBoolean(b, false);
+
+ // Ensure currentSyncSource is set
+ if (currentSyncSourceF.get(b) == null) {
+ currentSyncSourceF.set(b, "LDAP");
+ }
+
+ // Prepare internal structures
+ Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable");
+ Field srcUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceUsers");
+ Field srcGroupUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroupUsers");
+ gutF.setAccessible(true);
+ srcUsersF.setAccessible(true);
+ srcGroupUsersF.setAccessible(true);
+ gutF.set(b, HashBasedTable.create());
+ srcUsersF.set(b, new HashMap<>());
+ srcGroupUsersF.set(b, new HashMap<>());
+
+ Method processGroupMembers = LdapUserGroupBuilder.class.getDeclaredMethod("processGroupMembers", Attribute.class, String.class);
+ processGroupMembers.setAccessible(true);
+
+ final String groupDn = "CN=TestGroup,OU=Groups,DC=example,DC=com";
+
+ // Test with a mix of valid and edge case members
+ BasicAttribute members = new BasicAttribute("member");
+ members.add("uid=user1,ou=people,dc=example,dc=com");
+ members.add("uid=user2,ou=people,dc=example,dc=com");
+ members.add("uid=user3,ou=people,dc=example,dc=com");
+
+ int count = (Integer) processGroupMembers.invoke(b, members, groupDn);
+ assertEquals(3, count);
+
+ @SuppressWarnings("unchecked")
+ Table<String, String, String> gut = (Table<String, String, String>) gutF.get(b);
+
+ // All valid users should be in the table
+ assertTrue(gut.contains(groupDn, "uid=user1,ou=people,dc=example,dc=com"));
+ assertTrue(gut.contains(groupDn, "uid=user2,ou=people,dc=example,dc=com"));
+ assertTrue(gut.contains(groupDn, "uid=user3,ou=people,dc=example,dc=com"));
+
+ // When userSearchEnabled is false, users should also be added to sourceUsers
+ @SuppressWarnings("unchecked")
+ Map<String, Map<String, String>> srcUsers = (Map<String, Map<String, String>>) srcUsersF.get(b);
+ assertTrue(srcUsers.containsKey("uid=user1,ou=people,dc=example,dc=com"));
+ assertTrue(srcUsers.containsKey("uid=user2,ou=people,dc=example,dc=com"));
+ assertTrue(srcUsers.containsKey("uid=user3,ou=people,dc=example,dc=com"));
+ }
+
private static void configureMinimalLdapConfig() throws Exception {
UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance();
cfg.setProperty("ranger.usersync.ldap.url", "ldap://127.0.0.1:389");