You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ranger.apache.org by "kumaab (via GitHub)" <gi...@apache.org> on 2024/12/10 02:57:32 UTC
[PR] RANGER-5064: checkstyle compliance updates - usersync modules [ranger]
kumaab opened a new pull request, #440:
URL: https://github.com/apache/ranger/pull/440
## What changes were proposed in this pull request?
Updated all `usersync modules` sources to comply with the check-style requirements.
## How was this patch tested?
`mvn clean verify` on local machine.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@ranger.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] RANGER-5064: checkstyle compliance updates - usersync modules [ranger]
Posted by "kumaab (via GitHub)" <gi...@apache.org>.
kumaab closed pull request #440: RANGER-5064: checkstyle compliance updates - usersync modules
URL: https://github.com/apache/ranger/pull/440
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@ranger.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] RANGER-5064: checkstyle compliance updates - usersync modules [ranger]
Posted by "kumaab (via GitHub)" <gi...@apache.org>.
kumaab commented on PR #440:
URL: https://github.com/apache/ranger/pull/440#issuecomment-2573867494
Closing the PR, as this is merged in master.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@ranger.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] RANGER-5064: checkstyle compliance updates - usersync modules [ranger]
Posted by "mneethiraj (via GitHub)" <gi...@apache.org>.
mneethiraj commented on code in PR #440:
URL: https://github.com/apache/ranger/pull/440#discussion_r1879004080
##########
ugsync-util/pom.xml:
##########
@@ -26,6 +26,9 @@
<packaging>jar</packaging>
<name>User Group Synchronizer Util</name>
<description>User Group Synchronizer Util</description>
+ <properties>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
Review Comment:
`<checkstyle.skip>false</checkstyle.skip>` would be needed as well.
##########
ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java:
##########
@@ -30,89 +42,70 @@
import java.security.KeyStore;
import java.security.SecureRandom;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-
-import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
-import org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CustomSSLSocketFactory extends SSLSocketFactory{
- private static final Logger LOG = LoggerFactory.getLogger(CustomSSLSocketFactory.class);
- private SSLSocketFactory sockFactory;
- private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
+public class CustomSSLSocketFactory extends SSLSocketFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(CustomSSLSocketFactory.class);
+ private SSLSocketFactory sockFactory;
+ private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
public CustomSSLSocketFactory() {
- SSLContext sslContext = null;
- String keyStoreFile = config.getSSLKeyStorePath();
- String keyStoreFilepwd = config.getSSLKeyStorePathPassword();
- String trustStoreFile = config.getSSLTrustStorePath();
- String trustStoreFilepwd = config.getSSLTrustStorePathPassword();
- String keyStoreType = config.getSSLKeyStoreType();
- String trustStoreType = config.getSSLTrustStoreType();
- try {
-
- KeyManager[] kmList = null;
- TrustManager[] tmList = null;
-
- if (keyStoreFile != null && keyStoreFilepwd != null) {
-
- KeyStore keyStore = KeyStore.getInstance(keyStoreType);
- InputStream in = null;
- try {
- in = getFileInputStream(keyStoreFile);
- if (in == null) {
- LOG.error("Unable to obtain keystore from file [" + keyStoreFile + "]");
- return;
- }
- keyStore.load(in, keyStoreFilepwd.toCharArray());
- KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray());
- kmList = keyManagerFactory.getKeyManagers();
- }
- finally {
- if (in != null) {
- in.close();
- }
- }
-
- }
-
- if (trustStoreFile != null && trustStoreFilepwd != null) {
-
- KeyStore trustStore = KeyStore.getInstance(trustStoreType);
- InputStream in = null;
- try {
- in = getFileInputStream(trustStoreFile);
- if (in == null) {
- LOG.error("Unable to obtain keystore from file [" + trustStoreFile + "]");
- return;
- }
- trustStore.load(in, trustStoreFilepwd.toCharArray());
- TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
- trustManagerFactory.init(trustStore);
- tmList = trustManagerFactory.getTrustManagers();
- }
- finally {
- if (in != null) {
- in.close();
- }
- }
- }
-
- sslContext = SSLContext.getInstance("TLSv1.2");
-
- sslContext.init(kmList, tmList, new SecureRandom());
- sockFactory = sslContext.getSocketFactory();
- }
- catch(Throwable t) {
- throw new RuntimeException("Unable to create SSLConext for communication to policy manager", t);
- }
+ SSLContext sslContext = null;
+ String keyStoreFile = config.getSSLKeyStorePath();
+ String keyStoreFilepwd = config.getSSLKeyStorePathPassword();
+ String trustStoreFile = config.getSSLTrustStorePath();
+ String trustStoreFilepwd = config.getSSLTrustStorePathPassword();
+ String keyStoreType = config.getSSLKeyStoreType();
+ String trustStoreType = config.getSSLTrustStoreType();
+ try {
+ KeyManager[] kmList = null;
+ TrustManager[] tmList = null;
+
+ if (keyStoreFile != null && keyStoreFilepwd != null) {
+ KeyStore keyStore = KeyStore.getInstance(keyStoreType);
+ InputStream in = null;
+ try {
Review Comment:
Consider replacing finally with try-with-resource: `try (InputStream in getFileInputStream(keyStoreFile)) {`.
Same at line 85 as well.
##########
ugsync/src/main/java/org/apache/ranger/ldapusersync/process/CustomSSLSocketFactory.java:
##########
@@ -30,89 +42,70 @@
import java.security.KeyStore;
import java.security.SecureRandom;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-
-import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
-import org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class CustomSSLSocketFactory extends SSLSocketFactory{
- private static final Logger LOG = LoggerFactory.getLogger(CustomSSLSocketFactory.class);
- private SSLSocketFactory sockFactory;
- private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
+public class CustomSSLSocketFactory extends SSLSocketFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(CustomSSLSocketFactory.class);
+ private SSLSocketFactory sockFactory;
Review Comment:
I suggest separating static and non-static members with a blank line.
##########
ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java:
##########
@@ -52,1240 +50,1161 @@
import javax.naming.ldap.StartTlsRequest;
import javax.naming.ldap.StartTlsResponse;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Table;
-import org.apache.ranger.ugsyncutil.util.UgsyncCommonConstants;
-import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
-import org.apache.ranger.ugsyncutil.model.LdapSyncSourceInfo;
-import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo;
-import org.apache.ranger.usergroupsync.UserGroupSink;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.ranger.usergroupsync.UserGroupSource;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
public class LdapUserGroupBuilder implements UserGroupSource {
-
- private static final Logger LOG = LoggerFactory.getLogger(LdapUserGroupBuilder.class);
-
- private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
-
- private static final String DATA_TYPE_BYTEARRAY = "byte[]";
- private static final String DATE_FORMAT = "yyyyMMddHHmmss";
- private static final int PAGE_SIZE = 500;
- private static final String MEMBER_OF_ATTR = "memberof=";
- private static final String GROUP_NAME_ATTRIBUTE = "cn=";
- private static long deltaSyncUserTime = 0; // Used for AD uSNChanged
- private static long deltaSyncGroupTime = 0; // Used for AD uSNChanged
- private String deltaSyncUserTimeStamp; // Used for OpenLdap modifyTimestamp
- private String deltaSyncGroupTimeStamp; // Used for OpenLdap modifyTimestamp
-
- private String ldapUrl;
- private String ldapBindDn;
- private String ldapBindPassword;
- private String ldapAuthenticationMechanism;
- private String ldapReferral;
- private String searchBase;
-
- private String[] userSearchBase;
- private String userNameAttribute;
- private String userCloudIdAttribute;
- private int userSearchScope;
- private String userObjectClass;
- private String userSearchFilter;
- private Set<String> groupNameSet;
- private String extendedUserSearchFilter;
- private SearchControls userSearchControls;
- private Set<String> userGroupNameAttributeSet;
- private Set<String> otherUserAttributes;
-
- private boolean pagedResultsEnabled = true;
- private int pagedResultsSize = PAGE_SIZE;
-
- private boolean groupSearchFirstEnabled = true;
- private boolean userSearchEnabled = true;
- private boolean groupSearchEnabled = true;
- private String[] groupSearchBase;
- private int groupSearchScope;
- private String groupObjectClass;
- private String groupSearchFilter;
- private String extendedGroupSearchFilter;
- private String extendedAllGroupsSearchFilter;
- private SearchControls groupSearchControls;
- private String groupMemberAttributeName;
- private String groupNameAttribute;
- private String groupCloudIdAttribute;
- private Set<String> otherGroupAttributes;
- private int groupHierarchyLevels;
- private int deleteCycles;
- private String currentSyncSource;
-
- private LdapContext ldapContext;
- StartTlsResponse tls;
-
- private Table<String, String, String> groupUserTable;
- UgsyncAuditInfo ugsyncAuditInfo;
- LdapSyncSourceInfo ldapSyncSourceInfo;
-
- private Map<String, Map<String, String>> sourceUsers; // key is user DN and value is map of user attributes containing original name, DN, etc...
- private Map<String, Map<String, String>> sourceGroups; // key is group DN and value is map of group attributes containing original name, DN, etc...
- private Map<String, Set<String>> sourceGroupUsers; // key is group DN and value is set of user DNs (members)
-
- public static void main(String[] args) throws Throwable {
- LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder();
- ugBuilder.init();
- }
-
- @Override
- public void init() throws Throwable{
- deltaSyncUserTime = 0;
- deltaSyncGroupTime = 0;
- deleteCycles = 1;
- DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
- deltaSyncGroupTimeStamp = dateFormat.format(new Date(0));
- setConfig();
- ugsyncAuditInfo = new UgsyncAuditInfo();
- ldapSyncSourceInfo = new LdapSyncSourceInfo();
- ldapSyncSourceInfo.setLdapUrl(ldapUrl);
- ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled()));
- ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
- ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
- ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
- ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
- ugsyncAuditInfo.setSyncSource(currentSyncSource);
- ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
- }
-
- private void createLdapContext() throws Throwable {
- Properties env = new Properties();
- env.put(Context.INITIAL_CONTEXT_FACTORY,
- "com.sun.jndi.ldap.LdapCtxFactory");
- env.put(Context.PROVIDER_URL, ldapUrl);
- if (ldapUrl.startsWith("ldaps") && (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty())) {
- env.put("java.naming.ldap.factory.socket", "org.apache.ranger.ldapusersync.process.CustomSSLSocketFactory");
- }
-
- if (StringUtils.isNotEmpty(userCloudIdAttribute)) {
- if (config.getUserCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary", userCloudIdAttribute);
- }
- }
-
- if (StringUtils.isNotEmpty(groupCloudIdAttribute)) {
- if (config.getGroupCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary", groupCloudIdAttribute);
- }
- }
-
- for (String otherUserAttribute : otherUserAttributes) {
- String attrType = config.getOtherUserAttributeDataType(otherUserAttribute);
- if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary", otherUserAttribute);
- }
- }
-
- for (String otherGroupAttribute : otherGroupAttributes) {
- String attrType = config.getOtherGroupAttributeDataType(otherGroupAttribute);
- if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary", otherGroupAttribute);
- }
- }
- ldapContext = new InitialLdapContext(env, null);
- if (!ldapUrl.startsWith("ldaps")) {
- if (config.isStartTlsEnabled()) {
- tls = (StartTlsResponse) ldapContext.extendedOperation(new StartTlsRequest());
- if (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty()) {
- tls.negotiate(CustomSSLSocketFactory.getDefault());
- } else {
- tls.negotiate();
- }
- LOG.info("Starting TLS session...");
- }
- }
-
- ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapBindDn);
- ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS, ldapBindPassword);
- ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism);
- ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral);
- }
-
- private void setConfig() throws Throwable {
- LOG.info("LdapUserGroupBuilder initialization started");
-
- currentSyncSource = config.getCurrentSyncSource();
- groupSearchFirstEnabled = true;
- userSearchEnabled = config.isUserSearchEnabled();
- groupSearchEnabled = config.isGroupSearchEnabled();
- ldapUrl = config.getLdapUrl();
- ldapBindDn = config.getLdapBindDn();
- ldapBindPassword = config.getLdapBindPassword();
- //ldapBindPassword = "admin-password";
- ldapAuthenticationMechanism = config.getLdapAuthenticationMechanism();
- ldapReferral = config.getContextReferral();
- searchBase = config.getSearchBase();
-
- userSearchBase = config.getUserSearchBase().split(";");
- userSearchScope = config.getUserSearchScope();
- userObjectClass = config.getUserObjectClass();
- userSearchFilter = config.getUserSearchFilter();
- userNameAttribute = config.getUserNameAttribute();
- userCloudIdAttribute = config.getUserCloudIdAttribute();
-
- Set<String> userSearchAttributes = new HashSet<String>();
- userSearchAttributes.add(userNameAttribute);
- userGroupNameAttributeSet = config.getUserGroupNameAttributeSet();
- for (String useGroupNameAttribute : userGroupNameAttributeSet) {
- userSearchAttributes.add(useGroupNameAttribute);
- }
- userSearchAttributes.add(userCloudIdAttribute);
- otherUserAttributes = config.getOtherUserAttributes();
- for (String otherUserAttribute : otherUserAttributes) {
- userSearchAttributes.add(otherUserAttribute);
- }
- userSearchAttributes.add("uSNChanged");
- userSearchAttributes.add("modifytimestamp");
- userSearchControls = new SearchControls();
- userSearchControls.setSearchScope(userSearchScope);
- userSearchControls.setReturningAttributes(userSearchAttributes.toArray(
- new String[userSearchAttributes.size()]));
-
- pagedResultsEnabled = config.isPagedResultsEnabled();
- pagedResultsSize = config.getPagedResultsSize();
-
- groupSearchBase = config.getGroupSearchBase().split(";");
- groupSearchScope = config.getGroupSearchScope();
- groupObjectClass = config.getGroupObjectClass();
- groupSearchFilter = config.getGroupSearchFilter();
- groupMemberAttributeName = config.getUserGroupMemberAttributeName();
- groupNameAttribute = config.getGroupNameAttribute();
- groupCloudIdAttribute = config.getGroupCloudIdAttribute();
- groupHierarchyLevels = config.getGroupHierarchyLevels();
-
- extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter + "(|(" + groupMemberAttributeName + "={0})(" + groupMemberAttributeName + "={1})))";
-
- groupSearchControls = new SearchControls();
- groupSearchControls.setSearchScope(groupSearchScope);
-
- Set<String> groupSearchAttributes = new HashSet<String>();
- groupSearchAttributes.add(groupNameAttribute);
- groupSearchAttributes.add(groupCloudIdAttribute);
- groupSearchAttributes.add(groupMemberAttributeName);
- groupSearchAttributes.add("uSNChanged");
- groupSearchAttributes.add("modifytimestamp");
- otherGroupAttributes = config.getOtherGroupAttributes();
- for (String otherGroupAttribute : otherGroupAttributes) {
- groupSearchAttributes.add(otherGroupAttribute);
- }
- groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(
- new String[groupSearchAttributes.size()]));
-
- if (StringUtils.isEmpty(userSearchFilter)) {
- groupNameSet = config.getGroupNameSet();
- String computedSearchFilter = "";
- for (String groupName : groupNameSet) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("groupName = " + groupName);
- }
- if (!groupName.startsWith(MEMBER_OF_ATTR) && !groupName.startsWith(GROUP_NAME_ATTRIBUTE)){
- LOG.info("Ignoring unsupported format for " + groupName );
- continue;
- }
- String searchFilter = groupName;
- if (groupName.startsWith(MEMBER_OF_ATTR)) {
- searchFilter = groupName.substring(MEMBER_OF_ATTR.length());
- }
- searchFilter = getFirstRDN(searchFilter);
- computedSearchFilter += getDNForMemberOf(searchFilter);
- }
- if (StringUtils.isNotEmpty(computedSearchFilter)) {
- computedSearchFilter = "(|" + computedSearchFilter + ")";
- }
- LOG.info("Final computedSearchFilter = " + computedSearchFilter);
- userSearchFilter = computedSearchFilter;
- }
-
- LOG.info("LdapUserGroupBuilder initialization completed with -- "
- + "ldapUrl: " + ldapUrl
- + ", ldapBindDn: " + ldapBindDn
- + ", ldapBindPassword: ***** "
- + ", ldapAuthenticationMechanism: " + ldapAuthenticationMechanism
- + ", searchBase: " + searchBase
- + ", userSearchBase: " + Arrays.toString(userSearchBase)
- + ", userSearchScope: " + userSearchScope
- + ", userObjectClass: " + userObjectClass
- + ", userSearchFilter: " + userSearchFilter
- + ", extendedUserSearchFilter: " + extendedUserSearchFilter
- + ", userNameAttribute: " + userNameAttribute
- + ", userSearchAttributes: " + userSearchAttributes
- + ", userGroupNameAttributeSet: " + userGroupNameAttributeSet
- + ", otherUserAttributes: " + otherUserAttributes
- + ", pagedResultsEnabled: " + pagedResultsEnabled
- + ", pagedResultsSize: " + pagedResultsSize
- + ", groupSearchEnabled: " + groupSearchEnabled
- + ", groupSearchBase: " + Arrays.toString(groupSearchBase)
- + ", groupSearchScope: " + groupSearchScope
- + ", groupObjectClass: " + groupObjectClass
- + ", groupSearchFilter: " + groupSearchFilter
- + ", extendedGroupSearchFilter: " + extendedGroupSearchFilter
- + ", extendedAllGroupsSearchFilter: " + extendedAllGroupsSearchFilter
- + ", groupMemberAttributeName: " + groupMemberAttributeName
- + ", groupNameAttribute: " + groupNameAttribute
- + ", groupSearchAttributes: " + groupSearchAttributes
- + ", groupSearchFirstEnabled: " + groupSearchFirstEnabled
- + ", userSearchEnabled: " + userSearchEnabled
- + ", ldapReferral: " + ldapReferral
- );
- }
-
- private void closeLdapContext() throws Throwable {
- if (tls != null) {
- tls.close();
- }
- if (ldapContext != null) {
- ldapContext.close();
- }
- }
-
- @Override
- public boolean isChanged() {
- // we do not want to get the full ldap dit and check whether anything has changed
- return true;
- }
-
- @Override
- public void updateSink(UserGroupSink sink) throws Throwable {
- LOG.info("LdapUserGroupBuilder updateSink started");
+ private static final Logger LOG = LoggerFactory.getLogger(LdapUserGroupBuilder.class);
+ private static final String DATA_TYPE_BYTEARRAY = "byte[]";
+ private static final String DATE_FORMAT = "yyyyMMddHHmmss";
+ private static final String MEMBER_OF_ATTR = "memberof=";
+ private static final String GROUP_NAME_ATTRIBUTE = "cn=";
+ private static final int PAGE_SIZE = 500;
+ /* for AD uSNChanged */
+ private static long deltaSyncUserTime;
+ private static long deltaSyncGroupTime;
+ /* ***************** */
+ private boolean pagedResultsEnabled = true;
+ private boolean groupSearchFirstEnabled = true;
+ private boolean userSearchEnabled = true;
+ private boolean groupSearchEnabled = true;
+ private int pagedResultsSize = PAGE_SIZE;
+ private int groupHierarchyLevels;
+ private int deleteCycles;
+ private int userSearchScope;
+ private int groupSearchScope;
+
+ private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
+ /* for OpenLdap modifyTimestamp */
+ private String deltaSyncUserTimeStamp;
+ private String deltaSyncGroupTimeStamp;
+ /* ******************************** */
+ private String ldapUrl;
+ private String ldapBindDn;
+ private String ldapBindPassword;
+ private String ldapAuthenticationMechanism;
+ private String ldapReferral;
+ private String searchBase;
+ private String userNameAttribute;
+ private String userCloudIdAttribute;
+ private String userObjectClass;
+ private String userSearchFilter;
+ private String extendedUserSearchFilter;
+ private String groupObjectClass;
+ private String groupSearchFilter;
+ private String extendedGroupSearchFilter;
+ private String extendedAllGroupsSearchFilter;
+ private String groupMemberAttributeName;
+ private String groupNameAttribute;
+ private String groupCloudIdAttribute;
+ private String currentSyncSource;
+ private String[] userSearchBase;
+ private String[] groupSearchBase;
+ private Set<String> groupNameSet;
+ private Set<String> userGroupNameAttributeSet;
+ private Set<String> otherUserAttributes;
+ private Set<String> otherGroupAttributes;
+ private LdapContext ldapContext;
+ private SearchControls userSearchControls;
+ private SearchControls groupSearchControls;
+ private Table<String, String, String> groupUserTable;
+ /* { key = user DN, value = map of user attributes {original name, DN, etc.}} */
+ private Map<String, Map<String, String>> sourceUsers;
+ /* { key = group DN, value = map of group attributes {original name, DN, etc.}} */
+ private Map<String, Map<String, String>> sourceGroups;
+ /* { key = group DN, value = set of user DNs (members) } */
+ private Map<String, Set<String>> sourceGroupUsers;
+
+ StartTlsResponse tls;
+ UgsyncAuditInfo ugsyncAuditInfo;
+ LdapSyncSourceInfo ldapSyncSourceInfo;
+
+ public static void main(String[] args) throws Throwable {
+ LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder();
+ ugBuilder.init();
+ }
+
+ @Override
+ public void init() throws Throwable {
+ deltaSyncUserTime = 0;
+ deltaSyncGroupTime = 0;
+ deleteCycles = 1;
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
+ deltaSyncGroupTimeStamp = dateFormat.format(new Date(0));
+ setConfig();
+ ugsyncAuditInfo = new UgsyncAuditInfo();
+ ldapSyncSourceInfo = new LdapSyncSourceInfo();
+ ldapSyncSourceInfo.setLdapUrl(ldapUrl);
+ ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled()));
+ ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
+ ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
+ ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
+ ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
+ ugsyncAuditInfo.setSyncSource(currentSyncSource);
+ ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
+ }
+
+ @Override
+ public boolean isChanged() {
+ // we do not want to get the full ldap dit and check whether anything has changed
+ return true;
+ }
+
+ @Override
+ public void updateSink(UserGroupSink sink) throws Throwable {
+ LOG.info("LdapUserGroupBuilder updateSink started");
boolean computeDeletes = false;
- groupUserTable = HashBasedTable.create();
- sourceGroups = new HashMap<>();
- sourceUsers = new HashMap<>();
- sourceGroupUsers = new HashMap<>();
- long highestdeltaSyncUserTime = 0;
- long highestdeltaSyncGroupTime = 0;
-
- if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) {
- deleteCycles = 1;
- computeDeletes = true;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Compute deleted users/groups is enabled for this sync cycle");
- }
- }
- if (config.isUserSyncDeletesEnabled()) {
- deleteCycles++;
- }
- if (groupSearchEnabled) {
- highestdeltaSyncGroupTime = getGroups(computeDeletes);
- }
- if (userSearchEnabled) {
- LOG.info("Performing user search to retrieve users from AD/LDAP");
- highestdeltaSyncUserTime = getUsers(computeDeletes);
- }
-
- if (groupHierarchyLevels > 0) {
- LOG.info("Going through group hierarchy for nested group evaluation");
+ groupUserTable = HashBasedTable.create();
+ sourceGroups = new HashMap<>();
+ sourceUsers = new HashMap<>();
+ sourceGroupUsers = new HashMap<>();
+ long highestdeltaSyncUserTime = 0;
+ long highestdeltaSyncGroupTime = 0;
+
+ if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) {
+ deleteCycles = 1;
+ computeDeletes = true;
+ LOG.debug("Compute deleted users/groups is enabled for this sync cycle");
+ }
+ if (config.isUserSyncDeletesEnabled()) {
+ deleteCycles++;
+ }
+ if (groupSearchEnabled) {
+ highestdeltaSyncGroupTime = getGroups(computeDeletes);
+ }
+ if (userSearchEnabled) {
+ LOG.info("Performing user search to retrieve users from AD/LDAP");
+ highestdeltaSyncUserTime = getUsers(computeDeletes);
+ }
+
+ if (groupHierarchyLevels > 0) {
+ LOG.info("Going through group hierarchy for nested group evaluation");
Set<String> groupFullNames = sourceGroups.keySet();
- for(String group : groupFullNames) {
- Set<String> nextLevelGroups = groupUserTable.column(group).keySet();
- goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels-1, group);
- }
- LOG.info("Completed group hierarchy computation");
- }
-
- Iterator<String> groupUserTableIterator = groupUserTable.rowKeySet().iterator();
- while (groupUserTableIterator.hasNext()) {
- String groupName = groupUserTableIterator.next();
- Map<String,String> groupUsersMap = groupUserTable.row(groupName);
- Set<String> userSet = new HashSet<String>();
- for(Map.Entry<String, String> entry : groupUsersMap.entrySet()){
- if (sourceUsers.containsKey(entry.getValue())) {
- userSet.add(entry.getValue());
- }
- }
- sourceGroupUsers.put(groupName, userSet);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Users = " + sourceUsers.keySet());
- LOG.debug("Groups = " + sourceGroups.keySet());
- LOG.debug("GroupUsers = " + sourceGroupUsers.keySet());
- }
-
- try {
- sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes);
- DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- LOG.info("deltaSyncUserTime = " + deltaSyncUserTime + " and highestdeltaSyncUserTime = " + highestdeltaSyncUserTime);
- if (deltaSyncUserTime < highestdeltaSyncUserTime) {
- // Incrementing highestdeltaSyncUserTime (for AD) in order to avoid search record repetition for next sync cycle.
- deltaSyncUserTime = highestdeltaSyncUserTime + 1;
- // Incrementing the highest timestamp value (for Openldap) with 1sec in order to avoid search record repetition for next sync cycle.
- deltaSyncUserTimeStamp = dateFormat.format(new Date(highestdeltaSyncUserTime + 60l));
- }
-
- LOG.info("deltaSyncGroupTime = " + deltaSyncGroupTime + " and highestdeltaSyncGroupTime = " + highestdeltaSyncGroupTime);
- // Update deltaSyncUserTime/deltaSyncUserTimeStamp here so that in case of failures, we get updates in next cycle
- if (deltaSyncGroupTime < highestdeltaSyncGroupTime) {
- // Incrementing highestdeltaSyncGroupTime (for AD) in order to avoid search record repetition for next sync cycle.
- deltaSyncGroupTime = highestdeltaSyncGroupTime+1;
- // Incrementing the highest timestamp value (for OpenLdap) with 1min in order to avoid search record repetition for next sync cycle.
- deltaSyncGroupTimeStamp = dateFormat.format(new Date(highestdeltaSyncGroupTime + 60l));
- }
- } catch (Throwable t) {
- LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t);
- }
-
- ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
- ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
-
- try {
- sink.postUserGroupAuditInfo(ugsyncAuditInfo);
- } catch (Throwable t) {
- LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage());
- }
- }
-
- private long getUsers(boolean computeDeletes) throws Throwable {
- NamingEnumeration<SearchResult> userSearchResultEnum = null;
- NamingEnumeration<SearchResult> groupSearchResultEnum = null;
- long highestdeltaSyncUserTime;
- try {
- createLdapContext();
- int total;
- // Activate paged results
- if (pagedResultsEnabled) {
- ldapContext.setRequestControls(new Control[]{
- new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) });
- }
- DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- if (groupUserTable.rowKeySet().size() != 0 || !config.isDeltaSyncEnabled() || (computeDeletes)) {
- // Fix RANGER-1957: Perform full sync when there are updates to the groups or when incremental sync is not enabled
- deltaSyncUserTime = 0;
- deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
- }
-
- extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))";
-
- if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) {
- String customFilter = userSearchFilter.trim();
- if (!customFilter.startsWith("(")) {
- customFilter = "(" + customFilter + ")";
- }
-
- extendedUserSearchFilter = "(&" + extendedUserSearchFilter + customFilter + ")";
- } else {
- extendedUserSearchFilter = "(&" + extendedUserSearchFilter + ")";
- }
- LOG.info("extendedUserSearchFilter = " + extendedUserSearchFilter);
-
- highestdeltaSyncUserTime = deltaSyncUserTime;
-
- // When multiple OUs are configured, go through each OU as the user search base to search for users.
- for (int ou=0; ou<userSearchBase.length; ou++) {
- byte[] cookie = null;
- int counter = 0;
- try {
- int paged = 0;
- do {
- userSearchResultEnum = ldapContext
- .search(userSearchBase[ou], extendedUserSearchFilter,
- userSearchControls);
-
- while (userSearchResultEnum.hasMore()) {
- // searchResults contains all the user entries
- final SearchResult userEntry = userSearchResultEnum.next();
-
- if (userEntry == null) {
- LOG.info("userEntry null, skipping sync for the entry");
- continue;
- }
-
- Attributes attributes = userEntry.getAttributes();
- if (attributes == null) {
- LOG.info("attributes missing for entry " + userEntry.getNameInNamespace() +
- ", skipping sync");
- continue;
- }
-
- Attribute userNameAttr = attributes.get(userNameAttribute);
- if (userNameAttr == null) {
- LOG.info(userNameAttribute + " missing for entry " + userEntry.getNameInNamespace() +
- ", skipping sync");
- continue;
- }
-
- String userFullName = (userEntry.getNameInNamespace());
- String userName = (String) userNameAttr.get();
-
- if (userName == null || userName.trim().isEmpty()) {
- LOG.info(userNameAttribute + " empty for entry " + userEntry.getNameInNamespace() +
- ", skipping sync");
- continue;
- }
-
- Attribute timeStampAttr = attributes.get("uSNChanged");
- if (timeStampAttr != null) {
- String uSNChangedVal = (String) timeStampAttr.get();
- long currentDeltaSyncTime = Long.parseLong(uSNChangedVal);
- LOG.info("uSNChangedVal = " + uSNChangedVal + "and currentDeltaSyncTime = " + currentDeltaSyncTime);
- if (currentDeltaSyncTime > highestdeltaSyncUserTime) {
- highestdeltaSyncUserTime = currentDeltaSyncTime;
- }
- } else {
- timeStampAttr = attributes.get("modifytimestamp");
- if (timeStampAttr != null) {
- String timeStampVal = (String) timeStampAttr.get();
- Date parseDate = dateFormat.parse(timeStampVal);
- long currentDeltaSyncTime = parseDate.getTime();
- LOG.info("timeStampVal = " + timeStampVal + "and currentDeltaSyncTime = " + currentDeltaSyncTime);
- if (currentDeltaSyncTime > highestdeltaSyncUserTime) {
- highestdeltaSyncUserTime = currentDeltaSyncTime;
- deltaSyncUserTimeStamp = timeStampVal;
- }
- }
- }
-
- // Get all the groups from the group name attribute of the user only when group search is not enabled.
- if (!groupSearchEnabled) {
- for (String useGroupNameAttribute : userGroupNameAttributeSet) {
- Attribute userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute);
- if (userGroupfAttribute != null) {
- NamingEnumeration<?> groupEnum = userGroupfAttribute.getAll();
- while (groupEnum.hasMore()) {
- String groupDN = (String) groupEnum.next();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding " + groupDN + " to " + userName);
- }
- Map<String, String> groupAttrMap = new HashMap<>();
- String groupName = getShortName(groupDN);
- groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName);
- groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupDN);
- groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
- groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
- sourceGroups.put(groupDN, groupAttrMap);
- if (LOG.isDebugEnabled()) {
- LOG.debug("As groupsearch is disabled, adding group " + groupName + " from user memberof attribute for user " + userName);
- }
- groupUserTable.put(groupDN, userFullName, userFullName);
- }
- }
- }
- }
-
- Map<String, String> userAttrMap = new HashMap<>();
- userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
- userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userFullName);
- userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
- userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
- Attribute userCloudIdAttr = attributes.get(userCloudIdAttribute);
- if (userCloudIdAttr != null) {
- addToAttrMap(userAttrMap, "cloud_id", userCloudIdAttr, config.getUserCloudIdAttributeDataType());
- }
- for (String otherUserAttribute : otherUserAttributes) {
- if (attributes.get(otherUserAttribute) != null) {
- String attrType = config.getOtherUserAttributeDataType(otherUserAttribute);
- addToAttrMap(userAttrMap, otherUserAttribute, attributes.get(otherUserAttribute), attrType);
- }
- }
-
- sourceUsers.put(userFullName, userAttrMap);
- if ((groupUserTable.containsColumn(userFullName) || groupUserTable.containsColumn(userName))) {
- //Update the username in the groupUserTable with the one from username attribute.
- Map<String, String> userMap = groupUserTable.column(userFullName);
- if (MapUtils.isEmpty(userMap)) {
- userMap = groupUserTable.column(userName);
- }
- for (Map.Entry<String, String> entry : userMap.entrySet()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Updating groupUserTable " + entry.getValue() + " with: " + userName + " for " + entry.getKey());
- }
- groupUserTable.put(entry.getKey(), userFullName, userFullName);
- }
- }
- counter++;
-
- if (counter <= 2000) {
- LOG.info("Updating user count: " + counter + ", userName: " + userName);
- if ( counter == 2000 ) {
- LOG.info("===> 2000 user records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <===");
+ for (String group : groupFullNames) {
+ Set<String> nextLevelGroups = groupUserTable.column(group).keySet();
+ goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels - 1, group);
+ }
+ LOG.info("Completed group hierarchy computation");
+ }
+
+ Iterator<String> groupUserTableIterator = groupUserTable.rowKeySet().iterator();
+ while (groupUserTableIterator.hasNext()) {
+ String groupName = groupUserTableIterator.next();
+ Map<String, String> groupUsersMap = groupUserTable.row(groupName);
+ Set<String> userSet = new HashSet<String>();
+ for (Map.Entry<String, String> entry : groupUsersMap.entrySet()) {
+ if (sourceUsers.containsKey(entry.getValue())) {
+ userSet.add(entry.getValue());
+ }
+ }
+ sourceGroupUsers.put(groupName, userSet);
+ }
+
+ LOG.debug("Users = {}", sourceUsers.keySet());
+ LOG.debug("Groups = {}", sourceGroups.keySet());
+ LOG.debug("GroupUsers = {}", sourceGroupUsers.keySet());
+
+ try {
+ sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes);
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ LOG.info("deltaSyncUserTime = {} and highestdeltaSyncUserTime = {}", deltaSyncUserTime, highestdeltaSyncUserTime);
+ if (deltaSyncUserTime < highestdeltaSyncUserTime) {
+ // Incrementing highestdeltaSyncUserTime (for AD) in order to avoid search record repetition for next sync cycle.
+ deltaSyncUserTime = highestdeltaSyncUserTime + 1;
+ // Incrementing the highest timestamp value (for Openldap) with 1sec in order to avoid search record repetition for next sync cycle.
+ deltaSyncUserTimeStamp = dateFormat.format(new Date(highestdeltaSyncUserTime + 60L));
+ }
+
+ LOG.info("deltaSyncGroupTime = {} and highestdeltaSyncGroupTime = {} ", deltaSyncGroupTime, highestdeltaSyncGroupTime);
+ // Update deltaSyncUserTime/deltaSyncUserTimeStamp here so that in case of failures, we get updates in next cycle
+ if (deltaSyncGroupTime < highestdeltaSyncGroupTime) {
+ // Incrementing highestdeltaSyncGroupTime (for AD) in order to avoid search record repetition for next sync cycle.
+ deltaSyncGroupTime = highestdeltaSyncGroupTime + 1;
+ // Incrementing the highest timestamp value (for OpenLdap) with 1min in order to avoid search record repetition for next sync cycle.
+ deltaSyncGroupTimeStamp = dateFormat.format(new Date(highestdeltaSyncGroupTime + 60L));
+ }
+ } catch (Throwable t) {
+ LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t);
+ }
+
+ ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
+ ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
+
+ try {
+ sink.postUserGroupAuditInfo(ugsyncAuditInfo);
+ } catch (Throwable t) {
+ LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage());
Review Comment:
Review log messages to replace string concat with parameterization.
##########
ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java:
##########
@@ -52,1240 +50,1161 @@
import javax.naming.ldap.StartTlsRequest;
import javax.naming.ldap.StartTlsResponse;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.thirdparty.com.google.common.collect.HashBasedTable;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Table;
-import org.apache.ranger.ugsyncutil.util.UgsyncCommonConstants;
-import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
-import org.apache.ranger.ugsyncutil.model.LdapSyncSourceInfo;
-import org.apache.ranger.ugsyncutil.model.UgsyncAuditInfo;
-import org.apache.ranger.usergroupsync.UserGroupSink;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.ranger.usergroupsync.UserGroupSource;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
public class LdapUserGroupBuilder implements UserGroupSource {
-
- private static final Logger LOG = LoggerFactory.getLogger(LdapUserGroupBuilder.class);
-
- private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
-
- private static final String DATA_TYPE_BYTEARRAY = "byte[]";
- private static final String DATE_FORMAT = "yyyyMMddHHmmss";
- private static final int PAGE_SIZE = 500;
- private static final String MEMBER_OF_ATTR = "memberof=";
- private static final String GROUP_NAME_ATTRIBUTE = "cn=";
- private static long deltaSyncUserTime = 0; // Used for AD uSNChanged
- private static long deltaSyncGroupTime = 0; // Used for AD uSNChanged
- private String deltaSyncUserTimeStamp; // Used for OpenLdap modifyTimestamp
- private String deltaSyncGroupTimeStamp; // Used for OpenLdap modifyTimestamp
-
- private String ldapUrl;
- private String ldapBindDn;
- private String ldapBindPassword;
- private String ldapAuthenticationMechanism;
- private String ldapReferral;
- private String searchBase;
-
- private String[] userSearchBase;
- private String userNameAttribute;
- private String userCloudIdAttribute;
- private int userSearchScope;
- private String userObjectClass;
- private String userSearchFilter;
- private Set<String> groupNameSet;
- private String extendedUserSearchFilter;
- private SearchControls userSearchControls;
- private Set<String> userGroupNameAttributeSet;
- private Set<String> otherUserAttributes;
-
- private boolean pagedResultsEnabled = true;
- private int pagedResultsSize = PAGE_SIZE;
-
- private boolean groupSearchFirstEnabled = true;
- private boolean userSearchEnabled = true;
- private boolean groupSearchEnabled = true;
- private String[] groupSearchBase;
- private int groupSearchScope;
- private String groupObjectClass;
- private String groupSearchFilter;
- private String extendedGroupSearchFilter;
- private String extendedAllGroupsSearchFilter;
- private SearchControls groupSearchControls;
- private String groupMemberAttributeName;
- private String groupNameAttribute;
- private String groupCloudIdAttribute;
- private Set<String> otherGroupAttributes;
- private int groupHierarchyLevels;
- private int deleteCycles;
- private String currentSyncSource;
-
- private LdapContext ldapContext;
- StartTlsResponse tls;
-
- private Table<String, String, String> groupUserTable;
- UgsyncAuditInfo ugsyncAuditInfo;
- LdapSyncSourceInfo ldapSyncSourceInfo;
-
- private Map<String, Map<String, String>> sourceUsers; // key is user DN and value is map of user attributes containing original name, DN, etc...
- private Map<String, Map<String, String>> sourceGroups; // key is group DN and value is map of group attributes containing original name, DN, etc...
- private Map<String, Set<String>> sourceGroupUsers; // key is group DN and value is set of user DNs (members)
-
- public static void main(String[] args) throws Throwable {
- LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder();
- ugBuilder.init();
- }
-
- @Override
- public void init() throws Throwable{
- deltaSyncUserTime = 0;
- deltaSyncGroupTime = 0;
- deleteCycles = 1;
- DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
- deltaSyncGroupTimeStamp = dateFormat.format(new Date(0));
- setConfig();
- ugsyncAuditInfo = new UgsyncAuditInfo();
- ldapSyncSourceInfo = new LdapSyncSourceInfo();
- ldapSyncSourceInfo.setLdapUrl(ldapUrl);
- ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled()));
- ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
- ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
- ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
- ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
- ugsyncAuditInfo.setSyncSource(currentSyncSource);
- ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
- }
-
- private void createLdapContext() throws Throwable {
- Properties env = new Properties();
- env.put(Context.INITIAL_CONTEXT_FACTORY,
- "com.sun.jndi.ldap.LdapCtxFactory");
- env.put(Context.PROVIDER_URL, ldapUrl);
- if (ldapUrl.startsWith("ldaps") && (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty())) {
- env.put("java.naming.ldap.factory.socket", "org.apache.ranger.ldapusersync.process.CustomSSLSocketFactory");
- }
-
- if (StringUtils.isNotEmpty(userCloudIdAttribute)) {
- if (config.getUserCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary", userCloudIdAttribute);
- }
- }
-
- if (StringUtils.isNotEmpty(groupCloudIdAttribute)) {
- if (config.getGroupCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary", groupCloudIdAttribute);
- }
- }
-
- for (String otherUserAttribute : otherUserAttributes) {
- String attrType = config.getOtherUserAttributeDataType(otherUserAttribute);
- if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary", otherUserAttribute);
- }
- }
-
- for (String otherGroupAttribute : otherGroupAttributes) {
- String attrType = config.getOtherGroupAttributeDataType(otherGroupAttribute);
- if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
- env.put("java.naming.ldap.attributes.binary", otherGroupAttribute);
- }
- }
- ldapContext = new InitialLdapContext(env, null);
- if (!ldapUrl.startsWith("ldaps")) {
- if (config.isStartTlsEnabled()) {
- tls = (StartTlsResponse) ldapContext.extendedOperation(new StartTlsRequest());
- if (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty()) {
- tls.negotiate(CustomSSLSocketFactory.getDefault());
- } else {
- tls.negotiate();
- }
- LOG.info("Starting TLS session...");
- }
- }
-
- ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapBindDn);
- ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS, ldapBindPassword);
- ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism);
- ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral);
- }
-
- private void setConfig() throws Throwable {
- LOG.info("LdapUserGroupBuilder initialization started");
-
- currentSyncSource = config.getCurrentSyncSource();
- groupSearchFirstEnabled = true;
- userSearchEnabled = config.isUserSearchEnabled();
- groupSearchEnabled = config.isGroupSearchEnabled();
- ldapUrl = config.getLdapUrl();
- ldapBindDn = config.getLdapBindDn();
- ldapBindPassword = config.getLdapBindPassword();
- //ldapBindPassword = "admin-password";
- ldapAuthenticationMechanism = config.getLdapAuthenticationMechanism();
- ldapReferral = config.getContextReferral();
- searchBase = config.getSearchBase();
-
- userSearchBase = config.getUserSearchBase().split(";");
- userSearchScope = config.getUserSearchScope();
- userObjectClass = config.getUserObjectClass();
- userSearchFilter = config.getUserSearchFilter();
- userNameAttribute = config.getUserNameAttribute();
- userCloudIdAttribute = config.getUserCloudIdAttribute();
-
- Set<String> userSearchAttributes = new HashSet<String>();
- userSearchAttributes.add(userNameAttribute);
- userGroupNameAttributeSet = config.getUserGroupNameAttributeSet();
- for (String useGroupNameAttribute : userGroupNameAttributeSet) {
- userSearchAttributes.add(useGroupNameAttribute);
- }
- userSearchAttributes.add(userCloudIdAttribute);
- otherUserAttributes = config.getOtherUserAttributes();
- for (String otherUserAttribute : otherUserAttributes) {
- userSearchAttributes.add(otherUserAttribute);
- }
- userSearchAttributes.add("uSNChanged");
- userSearchAttributes.add("modifytimestamp");
- userSearchControls = new SearchControls();
- userSearchControls.setSearchScope(userSearchScope);
- userSearchControls.setReturningAttributes(userSearchAttributes.toArray(
- new String[userSearchAttributes.size()]));
-
- pagedResultsEnabled = config.isPagedResultsEnabled();
- pagedResultsSize = config.getPagedResultsSize();
-
- groupSearchBase = config.getGroupSearchBase().split(";");
- groupSearchScope = config.getGroupSearchScope();
- groupObjectClass = config.getGroupObjectClass();
- groupSearchFilter = config.getGroupSearchFilter();
- groupMemberAttributeName = config.getUserGroupMemberAttributeName();
- groupNameAttribute = config.getGroupNameAttribute();
- groupCloudIdAttribute = config.getGroupCloudIdAttribute();
- groupHierarchyLevels = config.getGroupHierarchyLevels();
-
- extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter + "(|(" + groupMemberAttributeName + "={0})(" + groupMemberAttributeName + "={1})))";
-
- groupSearchControls = new SearchControls();
- groupSearchControls.setSearchScope(groupSearchScope);
-
- Set<String> groupSearchAttributes = new HashSet<String>();
- groupSearchAttributes.add(groupNameAttribute);
- groupSearchAttributes.add(groupCloudIdAttribute);
- groupSearchAttributes.add(groupMemberAttributeName);
- groupSearchAttributes.add("uSNChanged");
- groupSearchAttributes.add("modifytimestamp");
- otherGroupAttributes = config.getOtherGroupAttributes();
- for (String otherGroupAttribute : otherGroupAttributes) {
- groupSearchAttributes.add(otherGroupAttribute);
- }
- groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(
- new String[groupSearchAttributes.size()]));
-
- if (StringUtils.isEmpty(userSearchFilter)) {
- groupNameSet = config.getGroupNameSet();
- String computedSearchFilter = "";
- for (String groupName : groupNameSet) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("groupName = " + groupName);
- }
- if (!groupName.startsWith(MEMBER_OF_ATTR) && !groupName.startsWith(GROUP_NAME_ATTRIBUTE)){
- LOG.info("Ignoring unsupported format for " + groupName );
- continue;
- }
- String searchFilter = groupName;
- if (groupName.startsWith(MEMBER_OF_ATTR)) {
- searchFilter = groupName.substring(MEMBER_OF_ATTR.length());
- }
- searchFilter = getFirstRDN(searchFilter);
- computedSearchFilter += getDNForMemberOf(searchFilter);
- }
- if (StringUtils.isNotEmpty(computedSearchFilter)) {
- computedSearchFilter = "(|" + computedSearchFilter + ")";
- }
- LOG.info("Final computedSearchFilter = " + computedSearchFilter);
- userSearchFilter = computedSearchFilter;
- }
-
- LOG.info("LdapUserGroupBuilder initialization completed with -- "
- + "ldapUrl: " + ldapUrl
- + ", ldapBindDn: " + ldapBindDn
- + ", ldapBindPassword: ***** "
- + ", ldapAuthenticationMechanism: " + ldapAuthenticationMechanism
- + ", searchBase: " + searchBase
- + ", userSearchBase: " + Arrays.toString(userSearchBase)
- + ", userSearchScope: " + userSearchScope
- + ", userObjectClass: " + userObjectClass
- + ", userSearchFilter: " + userSearchFilter
- + ", extendedUserSearchFilter: " + extendedUserSearchFilter
- + ", userNameAttribute: " + userNameAttribute
- + ", userSearchAttributes: " + userSearchAttributes
- + ", userGroupNameAttributeSet: " + userGroupNameAttributeSet
- + ", otherUserAttributes: " + otherUserAttributes
- + ", pagedResultsEnabled: " + pagedResultsEnabled
- + ", pagedResultsSize: " + pagedResultsSize
- + ", groupSearchEnabled: " + groupSearchEnabled
- + ", groupSearchBase: " + Arrays.toString(groupSearchBase)
- + ", groupSearchScope: " + groupSearchScope
- + ", groupObjectClass: " + groupObjectClass
- + ", groupSearchFilter: " + groupSearchFilter
- + ", extendedGroupSearchFilter: " + extendedGroupSearchFilter
- + ", extendedAllGroupsSearchFilter: " + extendedAllGroupsSearchFilter
- + ", groupMemberAttributeName: " + groupMemberAttributeName
- + ", groupNameAttribute: " + groupNameAttribute
- + ", groupSearchAttributes: " + groupSearchAttributes
- + ", groupSearchFirstEnabled: " + groupSearchFirstEnabled
- + ", userSearchEnabled: " + userSearchEnabled
- + ", ldapReferral: " + ldapReferral
- );
- }
-
- private void closeLdapContext() throws Throwable {
- if (tls != null) {
- tls.close();
- }
- if (ldapContext != null) {
- ldapContext.close();
- }
- }
-
- @Override
- public boolean isChanged() {
- // we do not want to get the full ldap dit and check whether anything has changed
- return true;
- }
-
- @Override
- public void updateSink(UserGroupSink sink) throws Throwable {
- LOG.info("LdapUserGroupBuilder updateSink started");
+ private static final Logger LOG = LoggerFactory.getLogger(LdapUserGroupBuilder.class);
+ private static final String DATA_TYPE_BYTEARRAY = "byte[]";
+ private static final String DATE_FORMAT = "yyyyMMddHHmmss";
+ private static final String MEMBER_OF_ATTR = "memberof=";
+ private static final String GROUP_NAME_ATTRIBUTE = "cn=";
+ private static final int PAGE_SIZE = 500;
+ /* for AD uSNChanged */
+ private static long deltaSyncUserTime;
+ private static long deltaSyncGroupTime;
+ /* ***************** */
+ private boolean pagedResultsEnabled = true;
+ private boolean groupSearchFirstEnabled = true;
+ private boolean userSearchEnabled = true;
+ private boolean groupSearchEnabled = true;
+ private int pagedResultsSize = PAGE_SIZE;
+ private int groupHierarchyLevels;
+ private int deleteCycles;
+ private int userSearchScope;
+ private int groupSearchScope;
+
+ private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
+ /* for OpenLdap modifyTimestamp */
+ private String deltaSyncUserTimeStamp;
+ private String deltaSyncGroupTimeStamp;
+ /* ******************************** */
+ private String ldapUrl;
+ private String ldapBindDn;
+ private String ldapBindPassword;
+ private String ldapAuthenticationMechanism;
+ private String ldapReferral;
+ private String searchBase;
+ private String userNameAttribute;
+ private String userCloudIdAttribute;
+ private String userObjectClass;
+ private String userSearchFilter;
+ private String extendedUserSearchFilter;
+ private String groupObjectClass;
+ private String groupSearchFilter;
+ private String extendedGroupSearchFilter;
+ private String extendedAllGroupsSearchFilter;
+ private String groupMemberAttributeName;
+ private String groupNameAttribute;
+ private String groupCloudIdAttribute;
+ private String currentSyncSource;
+ private String[] userSearchBase;
+ private String[] groupSearchBase;
+ private Set<String> groupNameSet;
+ private Set<String> userGroupNameAttributeSet;
+ private Set<String> otherUserAttributes;
+ private Set<String> otherGroupAttributes;
+ private LdapContext ldapContext;
+ private SearchControls userSearchControls;
+ private SearchControls groupSearchControls;
+ private Table<String, String, String> groupUserTable;
+ /* { key = user DN, value = map of user attributes {original name, DN, etc.}} */
+ private Map<String, Map<String, String>> sourceUsers;
+ /* { key = group DN, value = map of group attributes {original name, DN, etc.}} */
+ private Map<String, Map<String, String>> sourceGroups;
+ /* { key = group DN, value = set of user DNs (members) } */
+ private Map<String, Set<String>> sourceGroupUsers;
+
+ StartTlsResponse tls;
+ UgsyncAuditInfo ugsyncAuditInfo;
+ LdapSyncSourceInfo ldapSyncSourceInfo;
+
+ public static void main(String[] args) throws Throwable {
+ LdapUserGroupBuilder ugBuilder = new LdapUserGroupBuilder();
+ ugBuilder.init();
+ }
+
+ @Override
+ public void init() throws Throwable {
+ deltaSyncUserTime = 0;
+ deltaSyncGroupTime = 0;
+ deleteCycles = 1;
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
+ deltaSyncGroupTimeStamp = dateFormat.format(new Date(0));
+ setConfig();
+ ugsyncAuditInfo = new UgsyncAuditInfo();
+ ldapSyncSourceInfo = new LdapSyncSourceInfo();
+ ldapSyncSourceInfo.setLdapUrl(ldapUrl);
+ ldapSyncSourceInfo.setIncrementalSycn(Boolean.toString(config.isDeltaSyncEnabled()));
+ ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
+ ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
+ ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
+ ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
+ ugsyncAuditInfo.setSyncSource(currentSyncSource);
+ ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
+ }
+
+ @Override
+ public boolean isChanged() {
+ // we do not want to get the full ldap dit and check whether anything has changed
+ return true;
+ }
+
+ @Override
+ public void updateSink(UserGroupSink sink) throws Throwable {
+ LOG.info("LdapUserGroupBuilder updateSink started");
boolean computeDeletes = false;
- groupUserTable = HashBasedTable.create();
- sourceGroups = new HashMap<>();
- sourceUsers = new HashMap<>();
- sourceGroupUsers = new HashMap<>();
- long highestdeltaSyncUserTime = 0;
- long highestdeltaSyncGroupTime = 0;
-
- if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) {
- deleteCycles = 1;
- computeDeletes = true;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Compute deleted users/groups is enabled for this sync cycle");
- }
- }
- if (config.isUserSyncDeletesEnabled()) {
- deleteCycles++;
- }
- if (groupSearchEnabled) {
- highestdeltaSyncGroupTime = getGroups(computeDeletes);
- }
- if (userSearchEnabled) {
- LOG.info("Performing user search to retrieve users from AD/LDAP");
- highestdeltaSyncUserTime = getUsers(computeDeletes);
- }
-
- if (groupHierarchyLevels > 0) {
- LOG.info("Going through group hierarchy for nested group evaluation");
+ groupUserTable = HashBasedTable.create();
+ sourceGroups = new HashMap<>();
+ sourceUsers = new HashMap<>();
+ sourceGroupUsers = new HashMap<>();
+ long highestdeltaSyncUserTime = 0;
+ long highestdeltaSyncGroupTime = 0;
+
+ if (config.isUserSyncDeletesEnabled() && deleteCycles >= config.getUserSyncDeletesFrequency()) {
+ deleteCycles = 1;
+ computeDeletes = true;
+ LOG.debug("Compute deleted users/groups is enabled for this sync cycle");
+ }
+ if (config.isUserSyncDeletesEnabled()) {
+ deleteCycles++;
+ }
+ if (groupSearchEnabled) {
+ highestdeltaSyncGroupTime = getGroups(computeDeletes);
+ }
+ if (userSearchEnabled) {
+ LOG.info("Performing user search to retrieve users from AD/LDAP");
+ highestdeltaSyncUserTime = getUsers(computeDeletes);
+ }
+
+ if (groupHierarchyLevels > 0) {
+ LOG.info("Going through group hierarchy for nested group evaluation");
Set<String> groupFullNames = sourceGroups.keySet();
- for(String group : groupFullNames) {
- Set<String> nextLevelGroups = groupUserTable.column(group).keySet();
- goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels-1, group);
- }
- LOG.info("Completed group hierarchy computation");
- }
-
- Iterator<String> groupUserTableIterator = groupUserTable.rowKeySet().iterator();
- while (groupUserTableIterator.hasNext()) {
- String groupName = groupUserTableIterator.next();
- Map<String,String> groupUsersMap = groupUserTable.row(groupName);
- Set<String> userSet = new HashSet<String>();
- for(Map.Entry<String, String> entry : groupUsersMap.entrySet()){
- if (sourceUsers.containsKey(entry.getValue())) {
- userSet.add(entry.getValue());
- }
- }
- sourceGroupUsers.put(groupName, userSet);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Users = " + sourceUsers.keySet());
- LOG.debug("Groups = " + sourceGroups.keySet());
- LOG.debug("GroupUsers = " + sourceGroupUsers.keySet());
- }
-
- try {
- sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes);
- DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- LOG.info("deltaSyncUserTime = " + deltaSyncUserTime + " and highestdeltaSyncUserTime = " + highestdeltaSyncUserTime);
- if (deltaSyncUserTime < highestdeltaSyncUserTime) {
- // Incrementing highestdeltaSyncUserTime (for AD) in order to avoid search record repetition for next sync cycle.
- deltaSyncUserTime = highestdeltaSyncUserTime + 1;
- // Incrementing the highest timestamp value (for Openldap) with 1sec in order to avoid search record repetition for next sync cycle.
- deltaSyncUserTimeStamp = dateFormat.format(new Date(highestdeltaSyncUserTime + 60l));
- }
-
- LOG.info("deltaSyncGroupTime = " + deltaSyncGroupTime + " and highestdeltaSyncGroupTime = " + highestdeltaSyncGroupTime);
- // Update deltaSyncUserTime/deltaSyncUserTimeStamp here so that in case of failures, we get updates in next cycle
- if (deltaSyncGroupTime < highestdeltaSyncGroupTime) {
- // Incrementing highestdeltaSyncGroupTime (for AD) in order to avoid search record repetition for next sync cycle.
- deltaSyncGroupTime = highestdeltaSyncGroupTime+1;
- // Incrementing the highest timestamp value (for OpenLdap) with 1min in order to avoid search record repetition for next sync cycle.
- deltaSyncGroupTimeStamp = dateFormat.format(new Date(highestdeltaSyncGroupTime + 60l));
- }
- } catch (Throwable t) {
- LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t);
- }
-
- ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
- ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
-
- try {
- sink.postUserGroupAuditInfo(ugsyncAuditInfo);
- } catch (Throwable t) {
- LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage());
- }
- }
-
- private long getUsers(boolean computeDeletes) throws Throwable {
- NamingEnumeration<SearchResult> userSearchResultEnum = null;
- NamingEnumeration<SearchResult> groupSearchResultEnum = null;
- long highestdeltaSyncUserTime;
- try {
- createLdapContext();
- int total;
- // Activate paged results
- if (pagedResultsEnabled) {
- ldapContext.setRequestControls(new Control[]{
- new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL) });
- }
- DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- if (groupUserTable.rowKeySet().size() != 0 || !config.isDeltaSyncEnabled() || (computeDeletes)) {
- // Fix RANGER-1957: Perform full sync when there are updates to the groups or when incremental sync is not enabled
- deltaSyncUserTime = 0;
- deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
- }
-
- extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))";
-
- if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) {
- String customFilter = userSearchFilter.trim();
- if (!customFilter.startsWith("(")) {
- customFilter = "(" + customFilter + ")";
- }
-
- extendedUserSearchFilter = "(&" + extendedUserSearchFilter + customFilter + ")";
- } else {
- extendedUserSearchFilter = "(&" + extendedUserSearchFilter + ")";
- }
- LOG.info("extendedUserSearchFilter = " + extendedUserSearchFilter);
-
- highestdeltaSyncUserTime = deltaSyncUserTime;
-
- // When multiple OUs are configured, go through each OU as the user search base to search for users.
- for (int ou=0; ou<userSearchBase.length; ou++) {
- byte[] cookie = null;
- int counter = 0;
- try {
- int paged = 0;
- do {
- userSearchResultEnum = ldapContext
- .search(userSearchBase[ou], extendedUserSearchFilter,
- userSearchControls);
-
- while (userSearchResultEnum.hasMore()) {
- // searchResults contains all the user entries
- final SearchResult userEntry = userSearchResultEnum.next();
-
- if (userEntry == null) {
- LOG.info("userEntry null, skipping sync for the entry");
- continue;
- }
-
- Attributes attributes = userEntry.getAttributes();
- if (attributes == null) {
- LOG.info("attributes missing for entry " + userEntry.getNameInNamespace() +
- ", skipping sync");
- continue;
- }
-
- Attribute userNameAttr = attributes.get(userNameAttribute);
- if (userNameAttr == null) {
- LOG.info(userNameAttribute + " missing for entry " + userEntry.getNameInNamespace() +
- ", skipping sync");
- continue;
- }
-
- String userFullName = (userEntry.getNameInNamespace());
- String userName = (String) userNameAttr.get();
-
- if (userName == null || userName.trim().isEmpty()) {
- LOG.info(userNameAttribute + " empty for entry " + userEntry.getNameInNamespace() +
- ", skipping sync");
- continue;
- }
-
- Attribute timeStampAttr = attributes.get("uSNChanged");
- if (timeStampAttr != null) {
- String uSNChangedVal = (String) timeStampAttr.get();
- long currentDeltaSyncTime = Long.parseLong(uSNChangedVal);
- LOG.info("uSNChangedVal = " + uSNChangedVal + "and currentDeltaSyncTime = " + currentDeltaSyncTime);
- if (currentDeltaSyncTime > highestdeltaSyncUserTime) {
- highestdeltaSyncUserTime = currentDeltaSyncTime;
- }
- } else {
- timeStampAttr = attributes.get("modifytimestamp");
- if (timeStampAttr != null) {
- String timeStampVal = (String) timeStampAttr.get();
- Date parseDate = dateFormat.parse(timeStampVal);
- long currentDeltaSyncTime = parseDate.getTime();
- LOG.info("timeStampVal = " + timeStampVal + "and currentDeltaSyncTime = " + currentDeltaSyncTime);
- if (currentDeltaSyncTime > highestdeltaSyncUserTime) {
- highestdeltaSyncUserTime = currentDeltaSyncTime;
- deltaSyncUserTimeStamp = timeStampVal;
- }
- }
- }
-
- // Get all the groups from the group name attribute of the user only when group search is not enabled.
- if (!groupSearchEnabled) {
- for (String useGroupNameAttribute : userGroupNameAttributeSet) {
- Attribute userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute);
- if (userGroupfAttribute != null) {
- NamingEnumeration<?> groupEnum = userGroupfAttribute.getAll();
- while (groupEnum.hasMore()) {
- String groupDN = (String) groupEnum.next();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding " + groupDN + " to " + userName);
- }
- Map<String, String> groupAttrMap = new HashMap<>();
- String groupName = getShortName(groupDN);
- groupAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, groupName);
- groupAttrMap.put(UgsyncCommonConstants.FULL_NAME, groupDN);
- groupAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
- groupAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
- sourceGroups.put(groupDN, groupAttrMap);
- if (LOG.isDebugEnabled()) {
- LOG.debug("As groupsearch is disabled, adding group " + groupName + " from user memberof attribute for user " + userName);
- }
- groupUserTable.put(groupDN, userFullName, userFullName);
- }
- }
- }
- }
-
- Map<String, String> userAttrMap = new HashMap<>();
- userAttrMap.put(UgsyncCommonConstants.ORIGINAL_NAME, userName);
- userAttrMap.put(UgsyncCommonConstants.FULL_NAME, userFullName);
- userAttrMap.put(UgsyncCommonConstants.SYNC_SOURCE, currentSyncSource);
- userAttrMap.put(UgsyncCommonConstants.LDAP_URL, config.getLdapUrl());
- Attribute userCloudIdAttr = attributes.get(userCloudIdAttribute);
- if (userCloudIdAttr != null) {
- addToAttrMap(userAttrMap, "cloud_id", userCloudIdAttr, config.getUserCloudIdAttributeDataType());
- }
- for (String otherUserAttribute : otherUserAttributes) {
- if (attributes.get(otherUserAttribute) != null) {
- String attrType = config.getOtherUserAttributeDataType(otherUserAttribute);
- addToAttrMap(userAttrMap, otherUserAttribute, attributes.get(otherUserAttribute), attrType);
- }
- }
-
- sourceUsers.put(userFullName, userAttrMap);
- if ((groupUserTable.containsColumn(userFullName) || groupUserTable.containsColumn(userName))) {
- //Update the username in the groupUserTable with the one from username attribute.
- Map<String, String> userMap = groupUserTable.column(userFullName);
- if (MapUtils.isEmpty(userMap)) {
- userMap = groupUserTable.column(userName);
- }
- for (Map.Entry<String, String> entry : userMap.entrySet()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Updating groupUserTable " + entry.getValue() + " with: " + userName + " for " + entry.getKey());
- }
- groupUserTable.put(entry.getKey(), userFullName, userFullName);
- }
- }
- counter++;
-
- if (counter <= 2000) {
- LOG.info("Updating user count: " + counter + ", userName: " + userName);
- if ( counter == 2000 ) {
- LOG.info("===> 2000 user records have been synchronized so far. From now on, only a summary progress log will be written for every 100 users. To continue to see detailed log for every user, please enable Trace level logging. <===");
+ for (String group : groupFullNames) {
+ Set<String> nextLevelGroups = groupUserTable.column(group).keySet();
+ goUpGroupHierarchy(nextLevelGroups, groupHierarchyLevels - 1, group);
+ }
+ LOG.info("Completed group hierarchy computation");
+ }
+
+ Iterator<String> groupUserTableIterator = groupUserTable.rowKeySet().iterator();
+ while (groupUserTableIterator.hasNext()) {
+ String groupName = groupUserTableIterator.next();
+ Map<String, String> groupUsersMap = groupUserTable.row(groupName);
+ Set<String> userSet = new HashSet<String>();
+ for (Map.Entry<String, String> entry : groupUsersMap.entrySet()) {
+ if (sourceUsers.containsKey(entry.getValue())) {
+ userSet.add(entry.getValue());
+ }
+ }
+ sourceGroupUsers.put(groupName, userSet);
+ }
+
+ LOG.debug("Users = {}", sourceUsers.keySet());
+ LOG.debug("Groups = {}", sourceGroups.keySet());
+ LOG.debug("GroupUsers = {}", sourceGroupUsers.keySet());
+
+ try {
+ sink.addOrUpdateUsersGroups(sourceGroups, sourceUsers, sourceGroupUsers, computeDeletes);
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ LOG.info("deltaSyncUserTime = {} and highestdeltaSyncUserTime = {}", deltaSyncUserTime, highestdeltaSyncUserTime);
+ if (deltaSyncUserTime < highestdeltaSyncUserTime) {
+ // Incrementing highestdeltaSyncUserTime (for AD) in order to avoid search record repetition for next sync cycle.
+ deltaSyncUserTime = highestdeltaSyncUserTime + 1;
+ // Incrementing the highest timestamp value (for Openldap) with 1sec in order to avoid search record repetition for next sync cycle.
+ deltaSyncUserTimeStamp = dateFormat.format(new Date(highestdeltaSyncUserTime + 60L));
+ }
+
+ LOG.info("deltaSyncGroupTime = {} and highestdeltaSyncGroupTime = {} ", deltaSyncGroupTime, highestdeltaSyncGroupTime);
+ // Update deltaSyncUserTime/deltaSyncUserTimeStamp here so that in case of failures, we get updates in next cycle
+ if (deltaSyncGroupTime < highestdeltaSyncGroupTime) {
+ // Incrementing highestdeltaSyncGroupTime (for AD) in order to avoid search record repetition for next sync cycle.
+ deltaSyncGroupTime = highestdeltaSyncGroupTime + 1;
+ // Incrementing the highest timestamp value (for OpenLdap) with 1min in order to avoid search record repetition for next sync cycle.
+ deltaSyncGroupTimeStamp = dateFormat.format(new Date(highestdeltaSyncGroupTime + 60L));
+ }
+ } catch (Throwable t) {
+ LOG.error("Failed to update ranger admin. Will retry in next sync cycle!!", t);
+ }
+
+ ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
+ ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
+
+ try {
+ sink.postUserGroupAuditInfo(ugsyncAuditInfo);
+ } catch (Throwable t) {
+ LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage());
+ }
+ }
+
+ private void createLdapContext() throws Throwable {
+ Properties env = new Properties();
+ env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory");
+ env.put(Context.PROVIDER_URL, ldapUrl);
+ if (ldapUrl.startsWith("ldaps") && (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty())) {
+ env.put("java.naming.ldap.factory.socket", "org.apache.ranger.ldapusersync.process.CustomSSLSocketFactory");
+ }
+
+ if (StringUtils.isNotEmpty(userCloudIdAttribute)) {
+ if (config.getUserCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
+ env.put("java.naming.ldap.attributes.binary", userCloudIdAttribute);
+ }
+ }
+
+ if (StringUtils.isNotEmpty(groupCloudIdAttribute)) {
+ if (config.getGroupCloudIdAttributeDataType().equals(DATA_TYPE_BYTEARRAY)) {
+ env.put("java.naming.ldap.attributes.binary", groupCloudIdAttribute);
+ }
+ }
+
+ for (String otherUserAttribute : otherUserAttributes) {
+ String attrType = config.getOtherUserAttributeDataType(otherUserAttribute);
+ if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
+ env.put("java.naming.ldap.attributes.binary", otherUserAttribute);
+ }
+ }
+
+ for (String otherGroupAttribute : otherGroupAttributes) {
+ String attrType = config.getOtherGroupAttributeDataType(otherGroupAttribute);
+ if (attrType.equals(DATA_TYPE_BYTEARRAY)) {
+ env.put("java.naming.ldap.attributes.binary", otherGroupAttribute);
+ }
+ }
+ ldapContext = new InitialLdapContext(env, null);
+ if (!ldapUrl.startsWith("ldaps")) {
+ if (config.isStartTlsEnabled()) {
+ tls = (StartTlsResponse) ldapContext.extendedOperation(new StartTlsRequest());
+ if (config.getSSLTrustStorePath() != null && !config.getSSLTrustStorePath().trim().isEmpty()) {
+ tls.negotiate(CustomSSLSocketFactory.getDefault());
+ } else {
+ tls.negotiate();
+ }
+ LOG.info("Starting TLS session...");
+ }
+ }
+
+ ldapContext.addToEnvironment(Context.SECURITY_PRINCIPAL, ldapBindDn);
+ ldapContext.addToEnvironment(Context.SECURITY_CREDENTIALS, ldapBindPassword);
+ ldapContext.addToEnvironment(Context.SECURITY_AUTHENTICATION, ldapAuthenticationMechanism);
+ ldapContext.addToEnvironment(Context.REFERRAL, ldapReferral);
+ }
+
+ private void setConfig() throws Throwable {
+ LOG.info("LdapUserGroupBuilder initialization started");
+ groupSearchFirstEnabled = true;
+ currentSyncSource = config.getCurrentSyncSource();
+ userSearchEnabled = config.isUserSearchEnabled();
+ groupSearchEnabled = config.isGroupSearchEnabled();
+ ldapUrl = config.getLdapUrl();
+ ldapBindDn = config.getLdapBindDn();
+ ldapBindPassword = config.getLdapBindPassword();
+ ldapAuthenticationMechanism = config.getLdapAuthenticationMechanism();
+ ldapReferral = config.getContextReferral();
+ searchBase = config.getSearchBase();
+ userSearchBase = config.getUserSearchBase().split(";");
+ userSearchScope = config.getUserSearchScope();
+ userObjectClass = config.getUserObjectClass();
+ userSearchFilter = config.getUserSearchFilter();
+ userNameAttribute = config.getUserNameAttribute();
+ userCloudIdAttribute = config.getUserCloudIdAttribute();
+
+ Set<String> userSearchAttributes = new HashSet<String>();
+ userSearchAttributes.add(userNameAttribute);
+ userGroupNameAttributeSet = config.getUserGroupNameAttributeSet();
+ for (String useGroupNameAttribute : userGroupNameAttributeSet) {
+ userSearchAttributes.add(useGroupNameAttribute);
+ }
+ userSearchAttributes.add(userCloudIdAttribute);
+ otherUserAttributes = config.getOtherUserAttributes();
+ for (String otherUserAttribute : otherUserAttributes) {
+ userSearchAttributes.add(otherUserAttribute);
+ }
+ userSearchAttributes.add("uSNChanged");
+ userSearchAttributes.add("modifytimestamp");
+ userSearchControls = new SearchControls();
+ userSearchControls.setSearchScope(userSearchScope);
+ userSearchControls.setReturningAttributes(userSearchAttributes.toArray(new String[userSearchAttributes.size()]));
+
+ pagedResultsEnabled = config.isPagedResultsEnabled();
+ pagedResultsSize = config.getPagedResultsSize();
+
+ groupSearchBase = config.getGroupSearchBase().split(";");
+ groupSearchScope = config.getGroupSearchScope();
+ groupObjectClass = config.getGroupObjectClass();
+ groupSearchFilter = config.getGroupSearchFilter();
+ groupMemberAttributeName = config.getUserGroupMemberAttributeName();
+ groupNameAttribute = config.getGroupNameAttribute();
+ groupCloudIdAttribute = config.getGroupCloudIdAttribute();
+ groupHierarchyLevels = config.getGroupHierarchyLevels();
+
+ extendedGroupSearchFilter = "(&" + extendedGroupSearchFilter + "(|(" + groupMemberAttributeName + "={0})(" + groupMemberAttributeName + "={1})))";
+
+ groupSearchControls = new SearchControls();
+ groupSearchControls.setSearchScope(groupSearchScope);
+
+ Set<String> groupSearchAttributes = new HashSet<String>();
+ groupSearchAttributes.add(groupNameAttribute);
+ groupSearchAttributes.add(groupCloudIdAttribute);
+ groupSearchAttributes.add(groupMemberAttributeName);
+ groupSearchAttributes.add("uSNChanged");
+ groupSearchAttributes.add("modifytimestamp");
+ otherGroupAttributes = config.getOtherGroupAttributes();
+ for (String otherGroupAttribute : otherGroupAttributes) {
+ groupSearchAttributes.add(otherGroupAttribute);
+ }
+ groupSearchControls.setReturningAttributes(groupSearchAttributes.toArray(new String[groupSearchAttributes.size()]));
+
+ if (StringUtils.isEmpty(userSearchFilter)) {
+ groupNameSet = config.getGroupNameSet();
+ String computedSearchFilter = "";
+ for (String groupName : groupNameSet) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("groupName = {}", groupName);
+ }
+ if (!groupName.startsWith(MEMBER_OF_ATTR) && !groupName.startsWith(GROUP_NAME_ATTRIBUTE)) {
+ LOG.info("Ignoring unsupported format for {}", groupName);
+ continue;
+ }
+ String searchFilter = groupName;
+ if (groupName.startsWith(MEMBER_OF_ATTR)) {
+ searchFilter = groupName.substring(MEMBER_OF_ATTR.length());
+ }
+ searchFilter = getFirstRDN(searchFilter);
+ computedSearchFilter += getDNForMemberOf(searchFilter);
+ }
+ if (StringUtils.isNotEmpty(computedSearchFilter)) {
+ computedSearchFilter = "(|" + computedSearchFilter + ")";
+ }
+ LOG.info("Final computedSearchFilter = {}", computedSearchFilter);
+ userSearchFilter = computedSearchFilter;
+ }
+
+ LOG.info("LdapUserGroupBuilder initialization completed with -- "
+ + "ldapUrl: " + ldapUrl
+ + ", ldapBindDn: " + ldapBindDn
+ + ", ldapBindPassword: ***** "
+ + ", ldapAuthenticationMechanism: " + ldapAuthenticationMechanism
+ + ", searchBase: " + searchBase
+ + ", userSearchBase: " + Arrays.toString(userSearchBase)
+ + ", userSearchScope: " + userSearchScope
+ + ", userObjectClass: " + userObjectClass
+ + ", userSearchFilter: " + userSearchFilter
+ + ", extendedUserSearchFilter: " + extendedUserSearchFilter
+ + ", userNameAttribute: " + userNameAttribute
+ + ", userSearchAttributes: " + userSearchAttributes
+ + ", userGroupNameAttributeSet: " + userGroupNameAttributeSet
+ + ", otherUserAttributes: " + otherUserAttributes
+ + ", pagedResultsEnabled: " + pagedResultsEnabled
+ + ", pagedResultsSize: " + pagedResultsSize
+ + ", groupSearchEnabled: " + groupSearchEnabled
+ + ", groupSearchBase: " + Arrays.toString(groupSearchBase)
+ + ", groupSearchScope: " + groupSearchScope
+ + ", groupObjectClass: " + groupObjectClass
+ + ", groupSearchFilter: " + groupSearchFilter
+ + ", extendedGroupSearchFilter: " + extendedGroupSearchFilter
+ + ", extendedAllGroupsSearchFilter: " + extendedAllGroupsSearchFilter
+ + ", groupMemberAttributeName: " + groupMemberAttributeName
+ + ", groupNameAttribute: " + groupNameAttribute
+ + ", groupSearchAttributes: " + groupSearchAttributes
+ + ", groupSearchFirstEnabled: " + groupSearchFirstEnabled
+ + ", userSearchEnabled: " + userSearchEnabled
+ + ", ldapReferral: " + ldapReferral);
+ }
+
+ private void closeLdapContext() throws Throwable {
+ if (tls != null) {
+ tls.close();
+ }
+ if (ldapContext != null) {
+ ldapContext.close();
+ }
+ }
+
+ private long getUsers(boolean computeDeletes) throws Throwable {
+ NamingEnumeration<SearchResult> userSearchResultEnum = null;
+ NamingEnumeration<SearchResult> groupSearchResultEnum = null;
+ long highestdeltaSyncUserTime;
+ try {
+ createLdapContext();
+ int total;
+ // Activate paged results
+ if (pagedResultsEnabled) {
+ ldapContext.setRequestControls(new Control[] {new PagedResultsControl(pagedResultsSize, Control.NONCRITICAL)});
+ }
+ DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ if (groupUserTable.rowKeySet().size() != 0 || !config.isDeltaSyncEnabled() || (computeDeletes)) {
+ // Fix RANGER-1957: Perform full sync when there are updates to the groups or when incremental sync is not enabled
+ deltaSyncUserTime = 0;
+ deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
+ }
+
+ extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))";
+
+ if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) {
+ String customFilter = userSearchFilter.trim();
+ if (!customFilter.startsWith("(")) {
+ customFilter = "(" + customFilter + ")";
+ }
+
+ extendedUserSearchFilter = "(&" + extendedUserSearchFilter + customFilter + ")";
+ } else {
+ extendedUserSearchFilter = "(&" + extendedUserSearchFilter + ")";
+ }
+ LOG.info("extendedUserSearchFilter = {}", extendedUserSearchFilter);
+
+ highestdeltaSyncUserTime = deltaSyncUserTime;
+
+ // When multiple OUs are configured, go through each OU as the user search base to search for users.
+ for (String s : userSearchBase) {
+ byte[] cookie = null;
+ int counter = 0;
+ try {
+ int paged = 0;
+ do {
+ userSearchResultEnum = ldapContext.search(s, extendedUserSearchFilter, userSearchControls);
+
+ while (userSearchResultEnum.hasMore()) {
+ // searchResults contains all the user entries
+ final SearchResult userEntry = userSearchResultEnum.next();
+
+ if (userEntry == null) {
+ LOG.info("userEntry null, skipping sync for the entry");
+ continue;
}
- } else {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Updating user count: " + counter
- + ", userName: " + userName);
- } else {
- if ( counter % 100 == 0) {
- LOG.info("Synced " + counter + " users till now");
+
+ Attributes attributes = userEntry.getAttributes();
+ if (attributes == null) {
+ LOG.info("attributes missing for entry {}, skipping sync", userEntry.getNameInNamespace());
+ continue;
+ }
+
+ Attribute userNameAttr = attributes.get(userNameAttribute);
+ if (userNameAttr == null) {
+ LOG.info("{} missing for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace());
+ continue;
+ }
+
+ String userFullName = userEntry.getNameInNamespace();
+ String userName = (String) userNameAttr.get();
+
+ if (userName == null || userName.trim().isEmpty()) {
+ LOG.info("{} empty for entry {}, skipping sync", userNameAttribute, userEntry.getNameInNamespace());
+ continue;
+ }
+
+ Attribute timeStampAttr = attributes.get("uSNChanged");
+ if (timeStampAttr != null) {
+ String uSNChangedVal = (String) timeStampAttr.get();
+ long currentDeltaSyncTime = Long.parseLong(uSNChangedVal);
+ LOG.info("uSNChangedVal = {} and currentDeltaSyncTime = {}", uSNChangedVal, currentDeltaSyncTime);
+ if (currentDeltaSyncTime > highestdeltaSyncUserTime) {
+ highestdeltaSyncUserTime = currentDeltaSyncTime;
+ }
+ } else {
+ timeStampAttr = attributes.get("modifytimestamp");
+ if (timeStampAttr != null) {
+ String timeStampVal = (String) timeStampAttr.get();
+ Date parseDate = dateFormat.parse(timeStampVal);
+ long currentDeltaSyncTime = parseDate.getTime();
+ LOG.info("timeStampVal = {} and currentDeltaSyncTime = {}", timeStampVal, currentDeltaSyncTime);
+ if (currentDeltaSyncTime > highestdeltaSyncUserTime) {
+ highestdeltaSyncUserTime = currentDeltaSyncTime;
+ deltaSyncUserTimeStamp = timeStampVal;
+ }
+ }
+ }
+
+ // Get all the groups from the group name attribute of the user only when group search is not enabled.
+ if (!groupSearchEnabled) {
+ for (String useGroupNameAttribute : userGroupNameAttributeSet) {
+ Attribute userGroupfAttribute = userEntry.getAttributes().get(useGroupNameAttribute);
+ if (userGroupfAttribute != null) {
+ NamingEnumeration<?> groupEnum = userGroupfAttribute.getAll();
+ while (groupEnum.hasMore()) {
+ String groupDN = (String) groupEnum.next();
+ if (LOG.isDebugEnabled()) {
Review Comment:
Consider removing `if (LOG.isDebugEnabled()) {`, unless arguments to log calls are expensive to compute.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@ranger.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org