You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by sp...@apache.org on 2018/04/03 17:39:54 UTC
[1/3] ranger git commit: RANGER-1985: Code changes to support
follow-up tasks found during internal demo
Repository: ranger
Updated Branches:
refs/heads/master cf69754ad -> cafe7aee0
RANGER-1985: Code changes to support follow-up tasks found during internal demo
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/7f61b148
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/7f61b148
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/7f61b148
Branch: refs/heads/master
Commit: 7f61b14810c3d8eb84d0e7ab0695672837130235
Parents: ceaed3f
Author: Sailaja Polavarapu <sp...@hortonworks.com>
Authored: Mon Apr 2 15:37:41 2018 -0700
Committer: Sailaja Polavarapu <sp...@hortonworks.com>
Committed: Mon Apr 2 15:37:41 2018 -0700
----------------------------------------------------------------------
.../optimized/current/ranger_core_db_mysql.sql | 6 +-
...31-create-schema-for-usersync-audit-info.sql | 6 +-
.../optimized/current/ranger_core_db_oracle.sql | 6 +-
...31-create-schema-for-usersync-audit-info.sql | 6 +-
.../current/ranger_core_db_postgres.sql | 6 +-
...31-create-schema-for-usersync-audit-info.sql | 6 +-
.../current/ranger_core_db_sqlanywhere.sql | 6 +-
...31-create-schema-for-usersync-audit-info.sql | 6 +-
.../current/ranger_core_db_sqlserver.sql | 6 +-
...31-create-schema-for-usersync-audit-info.sql | 6 +-
.../apache/ranger/entity/XXUgsyncAuditInfo.java | 63 ++++++++---
.../ranger/service/XUgsyncAuditInfoService.java | 20 ++--
.../service/XUgsyncAuditInfoServiceBase.java | 12 ++-
.../ranger/view/VXFileSyncSourceInfo.java | 20 ++++
.../ranger/view/VXLdapSyncSourceInfo.java | 50 +++++++++
.../apache/ranger/view/VXUgsyncAuditInfo.java | 38 +++++--
.../ranger/view/VXUnixSyncSourceInfo.java | 20 ++++
.../process/LdapDeltaUserGroupBuilder.java | 100 ++++++++++++-----
.../process/LdapPolicyMgrUserGroupBuilder.java | 3 +-
.../process/LdapUserGroupBuilder.java | 16 ++-
.../config/UserGroupSyncConfig.java | 2 +-
.../unixusersync/model/FileSyncSourceInfo.java | 22 +++-
.../unixusersync/model/LdapSyncSourceInfo.java | 51 +++++++++
.../unixusersync/model/UgsyncAuditInfo.java | 44 +++++---
.../unixusersync/model/UnixSyncSourceInfo.java | 22 +++-
.../process/FileSourceUserGroupBuilder.java | 63 ++++++-----
.../process/PolicyMgrUserGroupBuilder.java | 107 ++++++++++++++-----
.../process/UnixUserGroupBuilder.java | 48 +++++----
.../ranger/usergroupsync/UserGroupSync.java | 7 +-
.../PolicyMgrUserGroupBuilderTest.java | 5 +
30 files changed, 594 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
index 65fc51c..51ba599 100644
--- a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
+++ b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
@@ -89,8 +89,10 @@ CREATE TABLE IF NOT EXISTS `x_ugsync_audit_info`(
`event_time` datetime NULL DEFAULT NULL,
`user_name` varchar(255) NOT NULL,
`sync_source` varchar(128) NOT NULL,
-`no_of_users` bigint(20) NOT NULL,
-`no_of_groups` bigint(20) NOT NULL,
+`no_of_new_users` bigint(20) NOT NULL,
+`no_of_new_groups` bigint(20) NOT NULL,
+`no_of_modified_users` bigint(20) NOT NULL,
+`no_of_modified_groups` bigint(20) NOT NULL,
`sync_source_info` varchar(4000) NOT NULL,
`session_id` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/mysql/patches/031-create-schema-for-usersync-audit-info.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/mysql/patches/031-create-schema-for-usersync-audit-info.sql b/security-admin/db/mysql/patches/031-create-schema-for-usersync-audit-info.sql
index 2405a3e..c17f19f 100644
--- a/security-admin/db/mysql/patches/031-create-schema-for-usersync-audit-info.sql
+++ b/security-admin/db/mysql/patches/031-create-schema-for-usersync-audit-info.sql
@@ -25,8 +25,10 @@ CREATE TABLE IF NOT EXISTS `x_ugsync_audit_info`(
`event_time` datetime NULL DEFAULT NULL,
`user_name` varchar(255) NOT NULL,
`sync_source` varchar(128) NOT NULL,
-`no_of_users` bigint(20) NOT NULL,
-`no_of_groups` bigint(20) NOT NULL,
+`no_of_new_users` bigint(20) NOT NULL,
+`no_of_new_groups` bigint(20) NOT NULL,
+`no_of_modified_users` bigint(20) NOT NULL,
+`no_of_modified_groups` bigint(20) NOT NULL,
`sync_source_info` varchar(4000) NOT NULL,
`session_id` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql b/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql
index 0634776..54228e9 100644
--- a/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql
+++ b/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql
@@ -1104,8 +1104,10 @@ upd_by_id NUMBER(20) DEFAULT NULL NULL,
event_time DATE DEFAULT NULL NULL,
user_name VARCHAR(255) NOT NULL,
sync_source VARCHAR(128) NOT NULL,
-no_of_users NUMBER(20) NOT NULL,
-no_of_groups NUMBER(20) NOT NULL,
+no_of_new_users NUMBER(20) NOT NULL,
+no_of_new_groups NUMBER(20) NOT NULL,
+no_of_modified_users NUMBER(20) NOT NULL,
+no_of_modified_groups NUMBER(20) NOT NULL,
sync_source_info VARCHAR(4000) NOT NULL,
session_id VARCHAR(255) DEFAULT NULL,
PRIMARY KEY (id)
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/oracle/patches/031-create-schema-for-usersync-audit-info.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/oracle/patches/031-create-schema-for-usersync-audit-info.sql b/security-admin/db/oracle/patches/031-create-schema-for-usersync-audit-info.sql
index e03e893..cb52065 100644
--- a/security-admin/db/oracle/patches/031-create-schema-for-usersync-audit-info.sql
+++ b/security-admin/db/oracle/patches/031-create-schema-for-usersync-audit-info.sql
@@ -24,8 +24,10 @@ upd_by_id NUMBER(20) DEFAULT NULL NULL,
event_time DATE DEFAULT NULL NULL,
user_name VARCHAR(255) NOT NULL,
sync_source VARCHAR(128) NOT NULL,
-no_of_users NUMBER(20) NOT NULL,
-no_of_groups NUMBER(20) NOT NULL,
+no_of_new_users NUMBER(20) NOT NULL,
+no_of_new_groups NUMBER(20) NOT NULL,
+no_of_modified_users NUMBER(20) NOT NULL,
+no_of_modified_groups NUMBER(20) NOT NULL,
sync_source_info VARCHAR(4000) NOT NULL,
session_id VARCHAR(255) DEFAULT NULL,
PRIMARY KEY (id)
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql b/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql
index 70f4bcc..8de469f 100644
--- a/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql
+++ b/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql
@@ -1221,8 +1221,10 @@ upd_by_id BIGINT DEFAULT NULL NULL,
event_time TIMESTAMP DEFAULT NULL NULL,
user_name varchar(255) NOT NULL,
sync_source varchar(128) NOT NULL,
-no_of_users BIGINT NOT NULL,
-no_of_groups BIGINT NOT NULL,
+no_of_new_users bigint NOT NULL,
+no_of_new_groups bigint NOT NULL,
+no_of_modified_users bigint NOT NULL,
+no_of_modified_groups bigint NOT NULL,
sync_source_info varchar(4000) NOT NULL,
session_id varchar(255) DEFAULT NULL,
primary key (id)
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/postgres/patches/031-create-schema-for-usersync-audit-info.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/postgres/patches/031-create-schema-for-usersync-audit-info.sql b/security-admin/db/postgres/patches/031-create-schema-for-usersync-audit-info.sql
index 2227f01..c2e126f 100644
--- a/security-admin/db/postgres/patches/031-create-schema-for-usersync-audit-info.sql
+++ b/security-admin/db/postgres/patches/031-create-schema-for-usersync-audit-info.sql
@@ -27,8 +27,10 @@ upd_by_id BIGINT DEFAULT NULL NULL,
event_time TIMESTAMP DEFAULT NULL NULL,
user_name varchar(255) NOT NULL,
sync_source varchar(128) NOT NULL,
-no_of_users bigint NOT NULL,
-no_of_groups bigint NOT NULL,
+no_of_new_users bigint NOT NULL,
+no_of_new_groups bigint NOT NULL,
+no_of_modified_users bigint NOT NULL,
+no_of_modified_groups bigint NOT NULL,
sync_source_info varchar(4000) NOT NULL,
session_id varchar(255) DEFAULT NULL,
primary key (id)
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql b/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql
index d59788c..594ee8a 100644
--- a/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql
+++ b/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql
@@ -876,8 +876,10 @@ CREATE TABLE dbo.x_ugsync_audit_info(
event_time datetime DEFAULT NULL NULL,
user_name varchar(255) NOT NULL,
sync_source varchar(128) NOT NULL,
- no_of_users bigint NOT NULL,
- no_of_groups bigint NOT NULL,
+ no_of_new_users bigint NOT NULL,
+ no_of_new_groups bigint NOT NULL,
+ no_of_modified_users bigint NOT NULL,
+ no_of_modified_groups bigint NOT NULL,
sync_source_info varchar(4000) NOT NULL,
session_id varchar(255) DEFAULT NULL NULL,
CONSTRAINT x_ugsync_audit_info_PK_id PRIMARY KEY CLUSTERED(id)
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/sqlanywhere/patches/031-create-schema-for-usersync-audit-info.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/sqlanywhere/patches/031-create-schema-for-usersync-audit-info.sql b/security-admin/db/sqlanywhere/patches/031-create-schema-for-usersync-audit-info.sql
index 26d857b..8ed84e3 100644
--- a/security-admin/db/sqlanywhere/patches/031-create-schema-for-usersync-audit-info.sql
+++ b/security-admin/db/sqlanywhere/patches/031-create-schema-for-usersync-audit-info.sql
@@ -22,8 +22,10 @@ CREATE TABLE dbo.x_ugsync_audit_info(
event_time datetime DEFAULT NULL NULL,
user_name varchar(255) NOT NULL,
sync_source varchar(128) NOT NULL,
- no_of_users bigint NOT NULL,
- no_of_groups bigint NOT NULL,
+ no_of_new_users bigint NOT NULL,
+ no_of_new_groups bigint NOT NULL,
+ no_of_modified_users bigint NOT NULL,
+ no_of_modified_groups bigint NOT NULL,
sync_source_info varchar(4000) NOT NULL,
session_id varchar(255) DEFAULT NULL NULL,
CONSTRAINT x_ugsync_audit_info_PK_id PRIMARY KEY CLUSTERED(id)
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql
index f67f109..f8591a3 100644
--- a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql
+++ b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql
@@ -1882,8 +1882,10 @@ CREATE TABLE [dbo].[x_ugsync_audit_info](
[event_time] [datetime2] DEFAULT NULL NULL,
[user_name] [varchar](255) NOT NULL,
[sync_source] [varchar](128) NOT NULL,
- [no_of_users] [bigint] NOT NULL,
- [no_of_groups] [bigint] NOT NULL,
+ [no_of_new_users] [bigint] NOT NULL,
+ [no_of_new_groups] [bigint] NOT NULL,
+ [no_of_modified_users] [bigint] NOT NULL,
+ [no_of_modified_groups] [bigint] NOT NULL,
[sync_source_info] [varchar](4000) NOT NULL,
[session_id] [varchar](255) DEFAULT NULL NULL,
PRIMARY KEY CLUSTERED
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/db/sqlserver/patches/031-create-schema-for-usersync-audit-info.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/sqlserver/patches/031-create-schema-for-usersync-audit-info.sql b/security-admin/db/sqlserver/patches/031-create-schema-for-usersync-audit-info.sql
index 362f415..e1fe33f 100644
--- a/security-admin/db/sqlserver/patches/031-create-schema-for-usersync-audit-info.sql
+++ b/security-admin/db/sqlserver/patches/031-create-schema-for-usersync-audit-info.sql
@@ -34,8 +34,10 @@ CREATE TABLE [dbo].[x_ugsync_audit_info](
[event_time] [datetime2] DEFAULT NULL NULL,
[user_name] [varchar](255) NOT NULL,
[sync_source] [varchar](128) NOT NULL,
- [no_of_users] [bigint] NOT NULL,
- [no_of_groups] [bigint] NOT NULL,
+ [no_of_new_users] [bigint] NOT NULL,
+ [no_of_new_groups] [bigint] NOT NULL,
+ [no_of_modified_users] [bigint] NOT NULL,
+ [no_of_modified_groups] [bigint] NOT NULL,
[sync_source_info] [varchar](4000) NOT NULL,
[session_id] [varchar](255) DEFAULT NULL NULL,
PRIMARY KEY CLUSTERED
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/src/main/java/org/apache/ranger/entity/XXUgsyncAuditInfo.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXUgsyncAuditInfo.java b/security-admin/src/main/java/org/apache/ranger/entity/XXUgsyncAuditInfo.java
index 5609833..afacb5d 100644
--- a/security-admin/src/main/java/org/apache/ranger/entity/XXUgsyncAuditInfo.java
+++ b/security-admin/src/main/java/org/apache/ranger/entity/XXUgsyncAuditInfo.java
@@ -48,11 +48,17 @@ public class XXUgsyncAuditInfo extends XXDBBase implements java.io.Serializable
@Column(name = "sync_source")
protected String syncSource;
- @Column(name = "no_of_users")
- protected Long noOfUsers;
+ @Column(name = "no_of_new_users")
+ protected Long noOfNewUsers;
- @Column(name = "no_of_groups")
- protected Long noOfGroups;
+ @Column(name = "no_of_new_groups")
+ protected Long noOfNewGroups;
+
+ @Column(name = "no_of_modified_users")
+ protected Long noOfModifiedUsers;
+
+ @Column(name = "no_of_modified_groups")
+ protected Long noOfModifiedGroups;
@Column(name = "sync_source_info")
protected String syncSourceInfo;
@@ -106,20 +112,37 @@ public class XXUgsyncAuditInfo extends XXDBBase implements java.io.Serializable
this.syncSource = syncSource;
}
- public Long getNoOfUsers() {
- return noOfUsers;
+
+ public Long getNoOfNewUsers() {
+ return noOfNewUsers;
+ }
+
+ public void setNoOfNewUsers(Long noOfUsers) {
+ this.noOfNewUsers = noOfUsers;
+ }
+
+ public Long getNoOfModifiedUsers() {
+ return noOfModifiedUsers;
}
- public void setNoOfUsers(Long noOfUsers) {
- this.noOfUsers = noOfUsers;
+ public void setNoOfModifiedUsers(Long noOfModifiedUsers) {
+ this.noOfModifiedUsers = noOfModifiedUsers;
}
- public Long getNoOfGroups() {
- return noOfGroups;
+ public Long getNoOfNewGroups() {
+ return noOfNewGroups;
}
- public void setNoOfGroups(Long noOfGroups) {
- this.noOfGroups = noOfGroups;
+ public void setNoOfNewGroups(Long noOfNewGroups) {
+ this.noOfNewGroups = noOfNewGroups;
+ }
+
+ public Long getNoOfModifiedGroups() {
+ return noOfModifiedGroups;
+ }
+
+ public void setNoOfModifiedGroups(Long noOfModifiedGroups) {
+ this.noOfModifiedGroups = noOfModifiedGroups;
}
public String getSyncSourceInfo() {
@@ -149,8 +172,10 @@ public class XXUgsyncAuditInfo extends XXDBBase implements java.io.Serializable
str += "eventTime={" + eventTime + "} ";
str += "userName={" + userName + "} ";
str += "syncSource={" + syncSource + "} ";
- str += "noOfUsers={" + noOfUsers + "} ";
- str += "noOfGroups={" + noOfGroups + "} ";
+ str += "noOfNewUsers={" + noOfNewUsers + "} ";
+ str += "noOfNewGroups={" + noOfNewGroups + "} ";
+ str += "noOfModifiedUsers={" + noOfModifiedUsers + "} ";
+ str += "noOfModifiedGroups={" + noOfModifiedGroups + "} ";
str += "syncSourceInfo={" + syncSourceInfo + "} ";
str += "sessionId={" + sessionId + "} ";
str += "}";
@@ -182,10 +207,16 @@ public class XXUgsyncAuditInfo extends XXDBBase implements java.io.Serializable
if ((this.syncSource == null && other.syncSource != null) || (this.syncSource != null && !this.syncSource.equals(other.syncSource))) {
return false;
}
- if ((this.noOfUsers == null && other.noOfUsers != null) || (this.noOfUsers != null && !this.noOfUsers.equals(other.noOfUsers))) {
+ if ((this.noOfNewUsers == null && other.noOfNewUsers != null) || (this.noOfNewUsers != null && !this.noOfNewUsers.equals(other.noOfNewUsers))) {
+ return false;
+ }
+ if ((this.noOfNewGroups == null && other.noOfNewGroups != null) || (this.noOfNewGroups != null && !this.noOfNewGroups.equals(other.noOfNewGroups))) {
+ return false;
+ }
+ if ((this.noOfModifiedUsers == null && other.noOfModifiedUsers != null) || (this.noOfModifiedUsers != null && !this.noOfModifiedUsers.equals(other.noOfModifiedUsers))) {
return false;
}
- if ((this.noOfGroups == null && other.noOfGroups != null) || (this.noOfGroups != null && !this.noOfGroups.equals(other.noOfGroups))) {
+ if ((this.noOfModifiedGroups == null && other.noOfModifiedGroups != null) || (this.noOfModifiedGroups != null && !this.noOfModifiedGroups.equals(other.noOfModifiedGroups))) {
return false;
}
if ((this.syncSourceInfo == null && other.syncSourceInfo != null) || (this.syncSourceInfo != null && !this.syncSourceInfo.equals(other.syncSourceInfo))) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoService.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoService.java b/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoService.java
index 85a9651..7f3d0c7 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoService.java
@@ -54,9 +54,13 @@ public class XUgsyncAuditInfoService extends XUgsyncAuditInfoServiceBase<XXUgsyn
DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL));
searchFields.add(new SearchField("syncSource", "obj.syncSource",
DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL));
- searchFields.add(new SearchField("noOfUsers", "obj.noOfUsers",
+ searchFields.add(new SearchField("noOfNewUsers", "obj.noOfNewUsers",
DATA_TYPE.INTEGER, SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("noOfGroups", "obj.noOfGroups",
+ searchFields.add(new SearchField("noOfNewGroups", "obj.noOfNewGroups",
+ DATA_TYPE.INTEGER, SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("noOfModifiedUsers", "obj.noOfModifiedUsers",
+ DATA_TYPE.INTEGER, SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("noOfModifiedGroups", "obj.noOfModifiedGroups",
DATA_TYPE.INTEGER, SEARCH_TYPE.FULL));
searchFields.add(new SearchField("syncSourceInfo", "obj.syncSourceInfo", DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL));
searchFields.add(new SearchField("startDate", "obj.eventTime",
@@ -70,8 +74,10 @@ public class XUgsyncAuditInfoService extends XUgsyncAuditInfoServiceBase<XXUgsyn
mObj.setEventTime(vObj.getEventTime());
mObj.setUserName(vObj.getUserName());
mObj.setSyncSource(vObj.getSyncSource());
- mObj.setNoOfUsers(vObj.getNoOfUsers());
- mObj.setNoOfGroups(vObj.getNoOfGroups());
+ mObj.setNoOfNewUsers(vObj.getNoOfNewUsers());
+ mObj.setNoOfNewGroups(vObj.getNoOfNewGroups());
+ mObj.setNoOfModifiedUsers(vObj.getNoOfModifiedUsers());
+ mObj.setNoOfModifiedGroups(vObj.getNoOfModifiedGroups());
mObj.setSyncSourceInfo(jsonUtil.readMapToString(vObj.getSyncSourceInfo()));
mObj.setSessionId(vObj.getSessionId());
return mObj;
@@ -82,8 +88,10 @@ public class XUgsyncAuditInfoService extends XUgsyncAuditInfoServiceBase<XXUgsyn
vObj.setEventTime(mObj.getEventTime());
vObj.setUserName(mObj.getUserName());
vObj.setSyncSource(mObj.getSyncSource());
- vObj.setNoOfUsers(mObj.getNoOfUsers());
- vObj.setNoOfGroups(mObj.getNoOfGroups());
+ vObj.setNoOfNewUsers(mObj.getNoOfNewUsers());
+ vObj.setNoOfNewGroups(mObj.getNoOfNewGroups());
+ vObj.setNoOfModifiedUsers(mObj.getNoOfModifiedUsers());
+ vObj.setNoOfModifiedGroups(mObj.getNoOfModifiedGroups());
String jsonString = mObj.getSyncSourceInfo();
vObj.setSyncSourceInfo(jsonUtil.jsonToMap(jsonString));
vObj.setSessionId( mObj.getSessionId());
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoServiceBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoServiceBase.java
index bf6f6e1..78e4c57 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoServiceBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/XUgsyncAuditInfoServiceBase.java
@@ -39,8 +39,10 @@ public abstract class XUgsyncAuditInfoServiceBase<T extends XXUgsyncAuditInfo, V
mObj.setEventTime(vObj.getEventTime());
mObj.setUserName(vObj.getUserName());
mObj.setSyncSource(vObj.getSyncSource());
- mObj.setNoOfGroups(vObj.getNoOfGroups());
- mObj.setNoOfUsers(vObj.getNoOfUsers());
+ mObj.setNoOfNewGroups(vObj.getNoOfNewGroups());
+ mObj.setNoOfNewUsers(vObj.getNoOfNewUsers());
+ mObj.setNoOfModifiedGroups(vObj.getNoOfModifiedGroups());
+ mObj.setNoOfModifiedUsers(vObj.getNoOfModifiedUsers());
return mObj;
}
@@ -49,8 +51,10 @@ public abstract class XUgsyncAuditInfoServiceBase<T extends XXUgsyncAuditInfo, V
vObj.setEventTime( mObj.getEventTime());
vObj.setUserName( mObj.getUserName());
vObj.setSyncSource( mObj.getSyncSource());
- vObj.setNoOfUsers( mObj.getNoOfUsers());
- vObj.setNoOfGroups( mObj.getNoOfGroups());
+ vObj.setNoOfNewGroups(mObj.getNoOfNewGroups());
+ vObj.setNoOfNewUsers(mObj.getNoOfNewUsers());
+ vObj.setNoOfModifiedGroups(mObj.getNoOfModifiedGroups());
+ vObj.setNoOfModifiedUsers(mObj.getNoOfModifiedUsers());
return vObj;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/src/main/java/org/apache/ranger/view/VXFileSyncSourceInfo.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXFileSyncSourceInfo.java b/security-admin/src/main/java/org/apache/ranger/view/VXFileSyncSourceInfo.java
index d8c66ca..b078a19 100644
--- a/security-admin/src/main/java/org/apache/ranger/view/VXFileSyncSourceInfo.java
+++ b/security-admin/src/main/java/org/apache/ranger/view/VXFileSyncSourceInfo.java
@@ -42,6 +42,8 @@ public class VXFileSyncSourceInfo implements java.io.Serializable {
private String fileName;
private String syncTime;
private String lastModified;
+ private long totalUsersSynced;
+ private long totalGroupsSynced;
public VXFileSyncSourceInfo() {
}
@@ -70,6 +72,22 @@ public class VXFileSyncSourceInfo implements java.io.Serializable {
this.lastModified = lastModified;
}
+ public long getTotalUsersSynced() {
+ return totalUsersSynced;
+ }
+
+ public void setTotalUsersSynced(long totalUsersSynced) {
+ this.totalUsersSynced = totalUsersSynced;
+ }
+
+ public long getTotalGroupsSynced() {
+ return totalGroupsSynced;
+ }
+
+ public void setTotalGroupsSynced(long totalGroupsSynced) {
+ this.totalGroupsSynced = totalGroupsSynced;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -81,6 +99,8 @@ public class VXFileSyncSourceInfo implements java.io.Serializable {
sb.append("{\"fileName\":\"").append(fileName);
sb.append("\", \"syncTime\":\"").append(syncTime);
sb.append("\", \"lastModified\":\"").append(lastModified);
+ sb.append("\", \"totalUsersSynced\":\"").append(totalUsersSynced);
+ sb.append("\", \"totalGroupsSynced\":\"").append(totalGroupsSynced);
sb.append("\"}");
return sb;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/src/main/java/org/apache/ranger/view/VXLdapSyncSourceInfo.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXLdapSyncSourceInfo.java b/security-admin/src/main/java/org/apache/ranger/view/VXLdapSyncSourceInfo.java
index fd2d8e1..be391de 100644
--- a/security-admin/src/main/java/org/apache/ranger/view/VXLdapSyncSourceInfo.java
+++ b/security-admin/src/main/java/org/apache/ranger/view/VXLdapSyncSourceInfo.java
@@ -41,9 +41,14 @@ public class VXLdapSyncSourceInfo implements java.io.Serializable {
private String ldapUrl;
private String incrementalSycn;
+ private String groupSearchFirstEnabled;
+ private String groupSearchEnabled;
+ private String userSearchEnabled;
private String userSearchFilter;
private String groupSearchFilter;
private String groupHierarchyLevel;
+ private long totalUsersSynced;
+ private long totalGroupsSynced;
public VXLdapSyncSourceInfo() {
}
@@ -88,6 +93,46 @@ public class VXLdapSyncSourceInfo implements java.io.Serializable {
this.groupHierarchyLevel = groupHierarchyLevel;
}
+ public long getTotalUsersSynced() {
+ return totalUsersSynced;
+ }
+
+ public void setTotalUsersSynced(long totalUsersSynced) {
+ this.totalUsersSynced = totalUsersSynced;
+ }
+
+ public long getTotalGroupsSynced() {
+ return totalGroupsSynced;
+ }
+
+ public void setTotalGroupsSynced(long totalGroupsSynced) {
+ this.totalGroupsSynced = totalGroupsSynced;
+ }
+
+ public String getGroupSearchFirstEnabled() {
+ return groupSearchFirstEnabled;
+ }
+
+ public void setGroupSearchFirstEnabled(String groupSearchFirstEnabled) {
+ this.groupSearchFirstEnabled = groupSearchFirstEnabled;
+ }
+
+ public String getGroupSearchEnabled() {
+ return groupSearchEnabled;
+ }
+
+ public void setGroupSearchEnabled(String groupSearchEnabled) {
+ this.groupSearchEnabled = groupSearchEnabled;
+ }
+
+ public String getUserSearchEnabled() {
+ return userSearchEnabled;
+ }
+
+ public void setUserSearchEnabled(String userSearchEnabled) {
+ this.userSearchEnabled = userSearchEnabled;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -98,9 +143,14 @@ public class VXLdapSyncSourceInfo implements java.io.Serializable {
public StringBuilder toString(StringBuilder sb) {
sb.append("{\"ldapUrl\":\"").append(ldapUrl);
sb.append("\", \"isIncrementalSync\":\"").append(incrementalSycn);
+ sb.append("\", \"userSearchEnabled\":\"").append(userSearchEnabled);
+ sb.append("\", \"groupSearchEnabled\":\"").append(groupSearchEnabled);
+ sb.append("\", \"groupSearchFirstEnabled\":\"").append(groupSearchFirstEnabled);
sb.append("\", \"userSearchFilter\":\"").append(userSearchFilter);
sb.append("\", \"groupSearchFilter\":\"").append(groupSearchFilter);
sb.append("\", \"groupHierarchyLevel\":\"").append(groupHierarchyLevel);
+ sb.append("\", \"totalUsersSynced\":\"").append(totalUsersSynced);
+ sb.append("\", \"totalGroupsSynced\":\"").append(totalGroupsSynced);
sb.append("\"}");
return sb;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/src/main/java/org/apache/ranger/view/VXUgsyncAuditInfo.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXUgsyncAuditInfo.java b/security-admin/src/main/java/org/apache/ranger/view/VXUgsyncAuditInfo.java
index 6fef928..ff569a1 100644
--- a/security-admin/src/main/java/org/apache/ranger/view/VXUgsyncAuditInfo.java
+++ b/security-admin/src/main/java/org/apache/ranger/view/VXUgsyncAuditInfo.java
@@ -47,8 +47,10 @@ public class VXUgsyncAuditInfo extends VXDataObject implements java.io.Serializa
private Date eventTime;
private String userName;
- private Long noOfUsers;
- private Long noOfGroups;
+ private Long noOfNewUsers;
+ private Long noOfNewGroups;
+ private Long noOfModifiedUsers;
+ private Long noOfModifiedGroups;
private String syncSource;
private String sessionId;
private Map<String, String> syncSourceInfo;
@@ -75,20 +77,36 @@ public class VXUgsyncAuditInfo extends VXDataObject implements java.io.Serializa
this.userName = userName;
}
- public Long getNoOfUsers() {
- return noOfUsers;
+ public Long getNoOfNewUsers() {
+ return noOfNewUsers;
}
- public void setNoOfUsers(Long noOfUsers) {
- this.noOfUsers = noOfUsers;
+ public void setNoOfNewUsers(Long noOfUsers) {
+ this.noOfNewUsers = noOfUsers;
}
- public Long getNoOfGroups() {
- return noOfGroups;
+ public Long getNoOfModifiedUsers() {
+ return noOfModifiedUsers;
}
- public void setNoOfGroups(Long noOfGroups) {
- this.noOfGroups = noOfGroups;
+ public void setNoOfModifiedUsers(Long noOfModifiedUsers) {
+ this.noOfModifiedUsers = noOfModifiedUsers;
+ }
+
+ public Long getNoOfNewGroups() {
+ return noOfNewGroups;
+ }
+
+ public void setNoOfNewGroups(Long noOfNewGroups) {
+ this.noOfNewGroups = noOfNewGroups;
+ }
+
+ public Long getNoOfModifiedGroups() {
+ return noOfModifiedGroups;
+ }
+
+ public void setNoOfModifiedGroups(Long noOfModifiedGroups) {
+ this.noOfModifiedGroups = noOfModifiedGroups;
}
public String getSyncSource() {
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/security-admin/src/main/java/org/apache/ranger/view/VXUnixSyncSourceInfo.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXUnixSyncSourceInfo.java b/security-admin/src/main/java/org/apache/ranger/view/VXUnixSyncSourceInfo.java
index 47bd839..55afec7 100644
--- a/security-admin/src/main/java/org/apache/ranger/view/VXUnixSyncSourceInfo.java
+++ b/security-admin/src/main/java/org/apache/ranger/view/VXUnixSyncSourceInfo.java
@@ -45,6 +45,8 @@ public class VXUnixSyncSourceInfo implements java.io.Serializable {
private String lastModified;
private String minUserId;
private String minGroupId;
+ private long totalUsersSynced;
+ private long totalGroupsSynced;
public VXUnixSyncSourceInfo() {
}
@@ -97,6 +99,22 @@ public class VXUnixSyncSourceInfo implements java.io.Serializable {
this.minGroupId = minGroupId;
}
+ public long getTotalUsersSynced() {
+ return totalUsersSynced;
+ }
+
+ public void setTotalUsersSynced(long totalUsersSynced) {
+ this.totalUsersSynced = totalUsersSynced;
+ }
+
+ public long getTotalGroupsSynced() {
+ return totalGroupsSynced;
+ }
+
+ public void setTotalGroupsSynced(long totalGroupsSynced) {
+ this.totalGroupsSynced = totalGroupsSynced;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -111,6 +129,8 @@ public class VXUnixSyncSourceInfo implements java.io.Serializable {
sb.append("\", \"lastModified\":\"").append(lastModified);
sb.append("\", \"minUserId\":\"").append(minUserId);
sb.append("\", \"minGroupId\":\"").append(minGroupId);
+ sb.append("\", \"totalUsersSynced\":\"").append(totalUsersSynced);
+ sb.append("\", \"totalGroupsSynced\":\"").append(totalGroupsSynced);
sb.append("\"}");
return sb;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java
index 7d85f33..2288ab8 100644
--- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapDeltaUserGroupBuilder.java
@@ -118,11 +118,14 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
private Table<String, String, String> groupUserTable;
private Map<String, String> userNameMap;
+ private HashSet<String> groupNames;
private BidiMap groupNameMap;
UgsyncAuditInfo ugsyncAuditInfo;
LdapSyncSourceInfo ldapSyncSourceInfo;
- int noOfUsers;
- int noOfGroups;
+ int noOfNewUsers;
+ int noOfNewGroups;
+ int noOfModifiedUsers;
+ int noOfModifiedGroups;
public static void main(String[] args) throws Throwable {
LdapDeltaUserGroupBuilder ugBuilder = new LdapDeltaUserGroupBuilder();
@@ -162,11 +165,15 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
deltaSyncUserTimeStamp = dateFormat.format(new Date(0));
deltaSyncGroupTimeStamp = dateFormat.format(new Date(0));
userNameMap = new HashMap<String, String>();
+ groupNames = new HashSet<String>();
setConfig();
ugsyncAuditInfo = new UgsyncAuditInfo();
ldapSyncSourceInfo = new LdapSyncSourceInfo();
ldapSyncSourceInfo.setLdapUrl(ldapUrl);
ldapSyncSourceInfo.setIncrementalSycn("True");
+ ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
+ ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
+ ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
ugsyncAuditInfo.setSyncSource("LDAP/AD");
ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
@@ -318,8 +325,10 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
LOG.info("LdapDeltaUserGroupBuilder updateSink started");
groupUserTable = HashBasedTable.create();
groupNameMap = new DualHashBidiMap();
- noOfUsers = 0;
- noOfGroups = 0;
+ noOfNewUsers = 0;
+ noOfNewGroups = 0;
+ noOfModifiedUsers = 0;
+ noOfModifiedGroups = 0;
if (!groupSearchFirstEnabled) {
LOG.info("Performing user search first");
@@ -339,8 +348,10 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
}
if (groupUserTable.isEmpty()) {
//System.out.println("groupUserTable is empty!!");
- ugsyncAuditInfo.setNoOfUsers(Integer.toUnsignedLong(noOfUsers));
- ugsyncAuditInfo.setNoOfGroups(Integer.toUnsignedLong(noOfGroups));
+ ugsyncAuditInfo.setNoOfNewUsers(Integer.toUnsignedLong(noOfNewUsers));
+ ugsyncAuditInfo.setNoOfNewGroups(Integer.toUnsignedLong(noOfNewGroups));
+ ugsyncAuditInfo.setNoOfModifiedUsers(Integer.toUnsignedLong(noOfModifiedUsers));
+ ugsyncAuditInfo.setNoOfModifiedGroups(Integer.toUnsignedLong(noOfModifiedGroups));
ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
try {
@@ -382,12 +393,16 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
+ ", users: " + userList);
}
}
- LOG.debug("postUserGroupAuditInfo(): noOfUsers = " + noOfUsers + " noOfGroups = " + noOfGroups);
+ LOG.debug("postUserGroupAuditInfo(): noOfUsers = " + noOfNewUsers + " noOfGroups = " + noOfNewGroups);
- ugsyncAuditInfo.setNoOfUsers(Integer.toUnsignedLong(noOfUsers));
- ugsyncAuditInfo.setNoOfGroups(Integer.toUnsignedLong(noOfGroups));
+ ugsyncAuditInfo.setNoOfNewUsers(Integer.toUnsignedLong(noOfNewUsers));
+ ugsyncAuditInfo.setNoOfNewGroups(Integer.toUnsignedLong(noOfNewGroups));
+ ugsyncAuditInfo.setNoOfModifiedUsers(Integer.toUnsignedLong(noOfModifiedUsers));
+ ugsyncAuditInfo.setNoOfModifiedGroups(Integer.toUnsignedLong(noOfModifiedGroups));
ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
+ ldapSyncSourceInfo.setTotalUsersSynced(userNameMap.size());
+ ldapSyncSourceInfo.setTotalGroupsSynced(groupNames.size());
try {
sink.postUserGroupAuditInfo(ugsyncAuditInfo);
@@ -513,6 +528,11 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
+ ", for user: " + transformUserName);
}
//System.out.println("Adding user fullname = " + userFullName + " username = " + transformUserName);
+ if (userNameMap.containsKey(userFullName)) {
+ noOfModifiedUsers++;
+ } else {
+ noOfNewUsers++;
+ }
userNameMap.put(userFullName, transformUserName);
Set<String> groups = new HashSet<String>();
@@ -541,30 +561,33 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
+ ", for user: " + transformUserName + " and groups: " + groupList);
}
counter++;
- noOfUsers++;
} else {
// If the user from the search result is present in the group user table,
// then addorupdate user to ranger admin.
LOG.debug("Chekcing if the user " + userFullName + " is part of the retrieved groups");
- if ((groupUserTable.containsColumn(userFullName) || groupUserTable.containsColumn(userName))
- && !userNameMap.containsKey(userFullName)) {
- String transformUserName = userNameTransform(userName);
- try {
- sink.addOrUpdateUser(transformUserName);
- } catch (Throwable t) {
- LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage()
- + ", for user: " + transformUserName);
- }
- userNameMap.put(userFullName, transformUserName);
- //Also update the username in the groupUserTable with the one from username attribute.
- Map<String,String> userMap = groupUserTable.column(userFullName);
- for(Map.Entry<String, String> entry : userMap.entrySet()){
- LOG.debug("Updating groupUserTable " + entry.getValue() + " with: " + transformUserName + " for " + entry.getKey());
- groupUserTable.put(entry.getKey(), userFullName, transformUserName);
+ if ((groupUserTable.containsColumn(userFullName) || groupUserTable.containsColumn(userName))) {
+ if (!userNameMap.containsKey(userFullName)) {
+ String transformUserName = userNameTransform(userName);
+ try {
+ sink.addOrUpdateUser(transformUserName);
+ } catch (Throwable t) {
+ LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage()
+ + ", for user: " + transformUserName);
+ }
+ userNameMap.put(userFullName, transformUserName);
+ //Also update the username in the groupUserTable with the one from username attribute.
+ Map<String, String> userMap = groupUserTable.column(userFullName);
+ for (Map.Entry<String, String> entry : userMap.entrySet()) {
+ LOG.debug("Updating groupUserTable " + entry.getValue() + " with: " + transformUserName + " for " + entry.getKey());
+ groupUserTable.put(entry.getKey(), userFullName, transformUserName);
+ }
+ counter++;
+ noOfNewUsers++;
+ } else {
+ noOfModifiedUsers++;
}
- counter++;
- noOfUsers++;
}
+
}
if (counter <= 2000) {
@@ -681,7 +704,6 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
continue;
}
counter++;
- noOfGroups++;
Attribute groupNameAttr = groupEntry.getAttributes().get(groupNameAttribute);
if (groupNameAttr == null) {
if (LOG.isInfoEnabled()) {
@@ -699,6 +721,7 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
LOG.debug("Update Ranger admin with " + transformGroupName);
sink.addOrUpdateGroup(transformGroupName);
}
+
Attribute timeStampAttr = groupEntry.getAttributes().get("uSNChanged");
if (timeStampAttr != null) {
String uSNChangedVal = (String) timeStampAttr.get();
@@ -723,6 +746,14 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
int userCount = 0;
if (groupMemberAttr == null || groupMemberAttr.size() <= 0) {
LOG.info("No members available for " + gName);
+ if (groupSearchFirstEnabled) {
+ if (groupNames.contains(gName)) {
+ noOfModifiedGroups++;
+ } else {
+ noOfNewGroups++;
+ }
+ groupNames.add(gName);
+ }
continue;
}
@@ -743,8 +774,12 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage()
+ ", for user: " + transformUserName);
}
+ if (userNameMap.containsKey(originalUserFullName)) {
+ noOfModifiedUsers++;
+ } else {
+ noOfNewUsers++;
+ }
userNameMap.put(originalUserFullName, transformUserName);
- noOfUsers++;
}
//System.out.println("Adding " + userNameMap.get(originalUserFullName) + " and fullname = " + originalUserFullName + " to " + gName);
if (userNameMap.get(originalUserFullName) != null) {
@@ -754,6 +789,12 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
}
groupNameMap.put(groupEntry.getNameInNamespace().toLowerCase(), gName);
}
+ if (groupNames.contains(gName)) {
+ noOfModifiedGroups++;
+ } else {
+ noOfNewGroups++;
+ }
+ groupNames.add(gName);
LOG.info("No. of members in the group " + gName + " = " + userCount);
}
// Examine the paged results control response
@@ -895,6 +936,7 @@ public class LdapDeltaUserGroupBuilder extends AbstractUserGroupSource {
// Add all members of sub group to the parent groups if the member is not a group in turn
Set<String> allMembers = groupUserTable.row(groupSName).keySet();
+ LOG.info("members of " + groupSName + " = " + allMembers);
for(String member : allMembers) {
String memberName = getShortGroupName(member);
if (!groupUserTable.containsRow(memberName)) { //Check if the member of a group is in turn a group
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapPolicyMgrUserGroupBuilder.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapPolicyMgrUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapPolicyMgrUserGroupBuilder.java
index 54a519a..d428e75 100644
--- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapPolicyMgrUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapPolicyMgrUserGroupBuilder.java
@@ -477,7 +477,8 @@ private static final Logger LOG = Logger.getLogger(LdapPolicyMgrUserGroupBuilder
}
private void addUserGroupAuditInfo(UgsyncAuditInfo auditInfo) {
- LOG.debug("INFO: addAuditInfo(" + auditInfo.getNoOfUsers() + ", " + auditInfo.getNoOfGroups() +
+ LOG.debug("INFO: addAuditInfo(" + auditInfo.getNoOfNewUsers() + ", " + auditInfo.getNoOfNewGroups() +
+ ", " + auditInfo.getNoOfModifiedUsers() + ", " + auditInfo.getNoOfModifiedGroups() +
", " + auditInfo.getSyncSource() + ")" );
if (authenticationType != null
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
index 85cba3c..8efa161 100644
--- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java
@@ -103,6 +103,7 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
private Map<String, UserInfo> userGroupMap;
//private Set<String> firstGroupDNs;
+ private Set<String> allUsers;
UgsyncAuditInfo ugsyncAuditInfo;
LdapSyncSourceInfo ldapSyncSourceInfo;
@@ -144,6 +145,9 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
ldapSyncSourceInfo = new LdapSyncSourceInfo();
ldapSyncSourceInfo.setLdapUrl(ldapUrl);
ldapSyncSourceInfo.setIncrementalSycn("False");
+ ldapSyncSourceInfo.setUserSearchEnabled(Boolean.toString(userSearchEnabled));
+ ldapSyncSourceInfo.setGroupSearchEnabled(Boolean.toString(groupSearchEnabled));
+ ldapSyncSourceInfo.setGroupSearchFirstEnabled(Boolean.toString(groupSearchFirstEnabled));
ldapSyncSourceInfo.setGroupHierarchyLevel(Integer.toString(groupHierarchyLevels));
ugsyncAuditInfo.setSyncSource("LDAP/AD");
ugsyncAuditInfo.setLdapSyncSourceInfo(ldapSyncSourceInfo);
@@ -312,6 +316,7 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
LOG.info("LDAPUserGroupBuilder updateSink started");
userGroupMap = new HashMap<String, UserInfo>();
Set<String> allGroups = new HashSet<String>();
+ allUsers = new HashSet<String>();
if (!groupSearchFirstEnabled) {
LOG.info("Performing user search first");
@@ -359,6 +364,8 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
}
ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
+ ldapSyncSourceInfo.setTotalUsersSynced(allUsers.size());
+ ldapSyncSourceInfo.setTotalGroupsSynced(allGroups.size());
try {
sink.postUserGroupAuditInfo(ugsyncAuditInfo);
} catch (Throwable t) {
@@ -370,6 +377,7 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
getGroups(sink, null);
// Go through the userInfo map and update ranger admin.
for (UserInfo userInfo : userGroupMap.values()) {
+ LOG.debug("userName from map = " + userInfo.getUserFullName());
String userName = getShortUserName(userInfo.getUserFullName());
if (groupHierarchyLevels > 0) {
//System.out.println("Going through group hierarchy for nested group evaluation");
@@ -384,6 +392,7 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
} else {
LOG.info("User search is disabled and hence using the group member attribute for username" + userName);
allGroups.addAll(groupList);
+ allUsers.add(userName); // Note:- in this case the usernames may contain groups as part of nested groups
if (userNameCaseConversionFlag) {
if (userNameLowerCaseFlag) {
userName = userName.toLowerCase();
@@ -407,6 +416,8 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
}
ldapSyncSourceInfo.setUserSearchFilter(extendedUserSearchFilter);
ldapSyncSourceInfo.setGroupSearchFilter(extendedAllGroupsSearchFilter);
+ ldapSyncSourceInfo.setTotalUsersSynced(allUsers.size());
+ ldapSyncSourceInfo.setTotalGroupsSynced(allGroups.size());
try {
sink.postUserGroupAuditInfo(ugsyncAuditInfo);
} catch (Throwable t) {
@@ -517,6 +528,7 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
LOG.warn("user object with username " + userName + " already exists and is replaced with the latest user object." );
}
userGroupMap.put(userName, userInfo);
+ allUsers.add(userName);
//List<String> groupList = new ArrayList<String>(groups);
List<String> groupList = userInfo.getGroups();
@@ -556,6 +568,7 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
counter++;
LOG.info("Updating username for " + userFullName + " with " + userName);
userInfo.updateUserName(userName);
+ allUsers.add(userName);
List<String> groupList = userInfo.getGroups();
if (userNameCaseConversionFlag) {
if (userNameLowerCaseFlag) {
@@ -705,12 +718,13 @@ public class LdapUserGroupBuilder extends AbstractUserGroupSource {
// check for group members and populate userInfo object with user's full name and group mapping
Attribute groupMemberAttr = groupEntry.getAttributes().get(groupMemberAttributeName);
LOG.debug("Update Ranger admin with " + gName);
- sink.addOrUpdateGroup(gName);
int userCount = 0;
if (groupMemberAttr == null || groupMemberAttr.size() <= 0) {
LOG.info("No members available for " + gName);
+ sink.addOrUpdateGroup(gName, null);
continue;
}
+ sink.addOrUpdateGroup(gName);
NamingEnumeration<?> userEnum = groupMemberAttr.getAll();
while (userEnum.hasMore()) {
String originalUserFullName = (String) userEnum.next();
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
index 45eeb1b..3efcb86 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
@@ -168,7 +168,7 @@ public class UserGroupSyncConfig {
* from the additional user search based on the user attribute configuration
*/
private static final String LGSYNC_USER_SEARCH_ENABLED = "ranger.usersync.user.searchenabled";
- private static final boolean DEFAULT_LGSYNC_USER_SEARCH_ENABLED = false;
+ private static final boolean DEFAULT_LGSYNC_USER_SEARCH_ENABLED = true;
private static final String LGSYNC_GROUP_USER_MAP_SYNC_ENABLED = "ranger.usersync.group.usermapsyncenabled";
private static final boolean DEFAULT_LGSYNC_GROUP_USER_MAP_SYNC_ENABLED = false;
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/unixusersync/model/FileSyncSourceInfo.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/FileSyncSourceInfo.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/FileSyncSourceInfo.java
index 02387e1..a6348d5 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/FileSyncSourceInfo.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/FileSyncSourceInfo.java
@@ -23,6 +23,8 @@ public class FileSyncSourceInfo {
private String fileName;
private String syncTime;
private String lastModified;
+ private long totalUsersSynced;
+ private long totalGroupsSynced;
public String getFileName() {
return fileName;
@@ -48,7 +50,23 @@ public class FileSyncSourceInfo {
this.lastModified = lastModified;
}
- @Override
+ public long getTotalUsersSynced() {
+ return totalUsersSynced;
+ }
+
+ public void setTotalUsersSynced(long totalUsersSynced) {
+ this.totalUsersSynced = totalUsersSynced;
+ }
+
+ public long getTotalGroupsSynced() {
+ return totalGroupsSynced;
+ }
+
+ public void setTotalGroupsSynced(long totalGroupsSynced) {
+ this.totalGroupsSynced = totalGroupsSynced;
+ }
+
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder();
toString(sb);
@@ -59,6 +77,8 @@ public class FileSyncSourceInfo {
sb.append("FileSycnSourceInfo [fileName= ").append(fileName);
sb.append(", syncTime= ").append(syncTime);
sb.append(", lastModified= ").append(lastModified);
+ sb.append(", totalUsersSynced= ").append(totalUsersSynced);
+ sb.append(", totalGroupsSynced= ").append(totalGroupsSynced);
sb.append("]");
return sb;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/unixusersync/model/LdapSyncSourceInfo.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/LdapSyncSourceInfo.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/LdapSyncSourceInfo.java
index 318138d..54802a0 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/LdapSyncSourceInfo.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/LdapSyncSourceInfo.java
@@ -22,10 +22,16 @@
public class LdapSyncSourceInfo {
private String ldapUrl;
private String incrementalSycn;
+ private String groupSearchFirstEnabled;
+ private String groupSearchEnabled;
+ private String userSearchEnabled;
private String userSearchFilter;
private String groupSearchFilter;
private String groupHierarchyLevel;
+ private long totalUsersSynced;
+ private long totalGroupsSynced;
+
public String getLdapUrl() {
return ldapUrl;
}
@@ -66,6 +72,46 @@ public class LdapSyncSourceInfo {
this.groupHierarchyLevel = groupHierarchyLevel;
}
+ public long getTotalUsersSynced() {
+ return totalUsersSynced;
+ }
+
+ public void setTotalUsersSynced(long totalUsersSynced) {
+ this.totalUsersSynced = totalUsersSynced;
+ }
+
+ public long getTotalGroupsSynced() {
+ return totalGroupsSynced;
+ }
+
+ public void setTotalGroupsSynced(long totalGroupsSynced) {
+ this.totalGroupsSynced = totalGroupsSynced;
+ }
+
+ public String getGroupSearchFirstEnabled() {
+ return groupSearchFirstEnabled;
+ }
+
+ public void setGroupSearchFirstEnabled(String groupSearchFirstEnabled) {
+ this.groupSearchFirstEnabled = groupSearchFirstEnabled;
+ }
+
+ public String getGroupSearchEnabled() {
+ return groupSearchEnabled;
+ }
+
+ public void setGroupSearchEnabled(String groupSearchEnabled) {
+ this.groupSearchEnabled = groupSearchEnabled;
+ }
+
+ public String getUserSearchEnabled() {
+ return userSearchEnabled;
+ }
+
+ public void setUserSearchEnabled(String userSearchEnabled) {
+ this.userSearchEnabled = userSearchEnabled;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -76,9 +122,14 @@ public class LdapSyncSourceInfo {
public StringBuilder toString(StringBuilder sb) {
sb.append("LdapSycnSourceInfo [ldapUrl= ").append(ldapUrl);
sb.append(", isIncrementalSync= ").append(incrementalSycn);
+ sb.append(", userSearchEnabled= ").append(userSearchEnabled);
+ sb.append(", groupSearchEnabled= ").append(groupSearchEnabled);
+ sb.append(", groupSearchFirstEnabled= ").append(groupSearchFirstEnabled);
sb.append(", userSearchFilter= ").append(userSearchFilter);
sb.append(", groupSearchFilter= ").append(groupSearchFilter);
sb.append(", groupHierarchyLevel= ").append(groupHierarchyLevel);
+ sb.append(", totalUsersSynced= ").append(totalUsersSynced);
+ sb.append(", totalGroupsSynced= ").append(totalGroupsSynced);
sb.append("]");
return sb;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UgsyncAuditInfo.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UgsyncAuditInfo.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UgsyncAuditInfo.java
index 9dfd3a5..65ac89b 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UgsyncAuditInfo.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UgsyncAuditInfo.java
@@ -22,28 +22,46 @@
public class UgsyncAuditInfo {
private String userName;
- private Long noOfUsers;
- private Long noOfGroups;
+ private Long noOfNewUsers;
+ private Long noOfNewGroups;
+ private Long noOfModifiedUsers;
+ private Long noOfModifiedGroups;
private String syncSource;
private String sessionId;
private LdapSyncSourceInfo ldapSyncSourceInfo;
private UnixSyncSourceInfo unixSyncSourceInfo;
private FileSyncSourceInfo fileSyncSourceInfo;
- public Long getNoOfUsers() {
- return noOfUsers;
+ public Long getNoOfNewUsers() {
+ return noOfNewUsers;
}
- public void setNoOfUsers(Long noOfUsers) {
- this.noOfUsers = noOfUsers;
+ public void setNoOfNewUsers(Long noOfUsers) {
+ this.noOfNewUsers = noOfUsers;
}
- public Long getNoOfGroups() {
- return noOfGroups;
+ public Long getNoOfModifiedUsers() {
+ return noOfModifiedUsers;
}
- public void setNoOfGroups(Long noOfGroups) {
- this.noOfGroups = noOfGroups;
+ public void setNoOfModifiedUsers(Long noOfModifiedUsers) {
+ this.noOfModifiedUsers = noOfModifiedUsers;
+ }
+
+ public Long getNoOfNewGroups() {
+ return noOfNewGroups;
+ }
+
+ public void setNoOfNewGroups(Long noOfNewGroups) {
+ this.noOfNewGroups = noOfNewGroups;
+ }
+
+ public Long getNoOfModifiedGroups() {
+ return noOfModifiedGroups;
+ }
+
+ public void setNoOfModifiedGroups(Long noOfModifiedGroups) {
+ this.noOfModifiedGroups = noOfModifiedGroups;
}
public String getSyncSource() {
@@ -102,8 +120,10 @@ public class UgsyncAuditInfo {
}
public StringBuilder toString(StringBuilder sb) {
- sb.append("UgsyncAuditInfo [No. of users= ").append(noOfUsers);
- sb.append(", No. of groups= ").append(noOfGroups);
+ sb.append("UgsyncAuditInfo [No. of New users= ").append(noOfNewUsers);
+ sb.append(", No. of New groups= ").append(noOfNewGroups);
+ sb.append(", No. of Modified users= ").append(noOfModifiedUsers);
+ sb.append(", No. of Modified groups= ").append(noOfModifiedGroups);
sb.append(", syncSource= ").append(syncSource);
sb.append(", ldapSyncSourceInfo= ").append(ldapSyncSourceInfo);
sb.append(", unixSyncSourceInfo= ").append(unixSyncSourceInfo);
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UnixSyncSourceInfo.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UnixSyncSourceInfo.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UnixSyncSourceInfo.java
index 97b95a1..6e5df9d 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UnixSyncSourceInfo.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/model/UnixSyncSourceInfo.java
@@ -26,6 +26,8 @@ public class UnixSyncSourceInfo {
private String lastModified;
private String minUserId;
private String minGroupId;
+ private long totalUsersSynced;
+ private long totalGroupsSynced;
public String getMinGroupId() {
return minGroupId;
@@ -75,7 +77,23 @@ public class UnixSyncSourceInfo {
this.lastModified = lastModified;
}
- @Override
+ public long getTotalUsersSynced() {
+ return totalUsersSynced;
+ }
+
+ public void setTotalUsersSynced(long totalUsersSynced) {
+ this.totalUsersSynced = totalUsersSynced;
+ }
+
+ public long getTotalGroupsSynced() {
+ return totalGroupsSynced;
+ }
+
+ public void setTotalGroupsSynced(long totalGroupsSynced) {
+ this.totalGroupsSynced = totalGroupsSynced;
+ }
+
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder();
toString(sb);
@@ -89,6 +107,8 @@ public class UnixSyncSourceInfo {
sb.append(", lastModified= ").append(lastModified);
sb.append(", minUserId= ").append(minUserId);
sb.append(", minGroupId= ").append(minGroupId);
+ sb.append(", totalUsersSynced= ").append(totalUsersSynced);
+ sb.append(", totalGroupsSynced= ").append(totalGroupsSynced);
sb.append("]");
return sb;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java
index a50b0c7..5850585 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/FileSourceUserGroupBuilder.java
@@ -22,10 +22,9 @@ package org.apache.ranger.unixusersync.process;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
@@ -49,6 +48,8 @@ public class FileSourceUserGroupBuilder extends AbstractUserGroupSource {
private long usergroupFileModified = 0;
private UgsyncAuditInfo ugsyncAuditInfo;
private FileSyncSourceInfo fileSyncSourceInfo;
+ private Set<String> groupNames;
+ private boolean isStartupFlag = false;
private boolean isUpdateSinkSucc = true;
@@ -78,6 +79,7 @@ public class FileSourceUserGroupBuilder extends AbstractUserGroupSource {
@Override
public void init() throws Throwable {
+ isStartupFlag = true;
if(userGroupFilename == null) {
userGroupFilename = config.getUserSyncFileSource();
}
@@ -108,39 +110,49 @@ public class FileSourceUserGroupBuilder extends AbstractUserGroupSource {
@Override
public void updateSink(UserGroupSink sink) throws Throwable {
isUpdateSinkSucc = true;
- buildUserGroupInfo();
String user=null;
List<String> groups=null;
- fileSyncSourceInfo.setLastModified(Long.toString(usergroupFileModified));
- fileSyncSourceInfo.setSyncTime(Long.toString(System.currentTimeMillis()));
+ DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ Date lastModifiedTime = new Date(usergroupFileModified);
+ Date syncTime = new Date(System.currentTimeMillis());
+ fileSyncSourceInfo.setLastModified(formatter.format(lastModifiedTime));
+ fileSyncSourceInfo.setSyncTime(formatter.format(syncTime));
- for (Map.Entry<String, List<String>> entry : user2GroupListMap.entrySet()) {
- user = entry.getKey();
- try{
- if (userNameRegExInst != null) {
- user = userNameRegExInst.transform(user);
- }
- groups = entry.getValue();
- if (groupNameRegExInst != null) {
- List<String> mappedGroups = new ArrayList<>();
- for (String group : groups) {
- mappedGroups.add(groupNameRegExInst.transform(group));
+ if (isChanged() || isStartupFlag) {
+ buildUserGroupInfo();
+
+ for (Map.Entry<String, List<String>> entry : user2GroupListMap.entrySet()) {
+ user = entry.getKey();
+ try {
+ if (userNameRegExInst != null) {
+ user = userNameRegExInst.transform(user);
+ }
+ groups = entry.getValue();
+ if (groupNameRegExInst != null) {
+ List<String> mappedGroups = new ArrayList<>();
+ for (String group : groups) {
+ mappedGroups.add(groupNameRegExInst.transform(group));
+ }
+ groups = mappedGroups;
}
- groups = mappedGroups;
+ groupNames.addAll(groups);
+ sink.addOrUpdateUser(user, groups);
+ } catch (Throwable t) {
+ LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage()
+ + ", for user: " + user
+ + ", groups: " + groups);
+ isUpdateSinkSucc = false;
}
- sink.addOrUpdateUser(user, groups);
- }catch (Throwable t) {
- LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage()
- + ", for user: " + user
- + ", groups: " + groups);
- isUpdateSinkSucc = false;
}
}
try {
+ fileSyncSourceInfo.setTotalUsersSynced(user2GroupListMap.size());
+ fileSyncSourceInfo.setTotalGroupsSynced(groupNames.size());
sink.postUserGroupAuditInfo(ugsyncAuditInfo);
} catch (Throwable t) {
LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage());
}
+ isStartupFlag = false;
}
private void setUserGroupFilename(String filename) {
@@ -160,6 +172,7 @@ public class FileSourceUserGroupBuilder extends AbstractUserGroupSource {
}
public void buildUserGroupInfo() throws Throwable {
+ groupNames = new HashSet<>();
buildUserGroupList();
if ( LOG.isDebugEnabled()) {
print();
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
index 9d3112d..b30b051 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/PolicyMgrUserGroupBuilder.java
@@ -116,8 +116,15 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
String nameRules;
Map<String, String> userMap = new LinkedHashMap<String, String>();
Map<String, String> groupMap = new LinkedHashMap<String, String>();
- private int noOfUsers;
- private int noOfGroups;
+ private int noOfNewUsers;
+ private int noOfNewGroups;
+ private int noOfModifiedUsers;
+ private int noOfModifiedGroups;
+ private HashSet<String> newUserList = new HashSet<String>();
+ private HashSet<String> modifiedUserList = new HashSet<String>();
+ private HashSet<String> newGroupList = new HashSet<String>();
+ private HashSet<String> modifiedGroupList = new HashSet<String>();
+ boolean isStartupFlag = false;
static {
try {
@@ -138,8 +145,11 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
recordsToPullPerCall = config.getMaxRecordsPerAPICall();
policyMgrBaseUrl = config.getPolicyManagerBaseURL();
isMockRun = config.isMockRunEnabled();
- noOfUsers = 0;
- noOfGroups = 0;
+ noOfNewUsers = 0;
+ noOfModifiedUsers = 0;
+ noOfNewGroups = 0;
+ noOfModifiedGroups = 0;
+ isStartupFlag = true;
if (isMockRun) {
LOG.setLevel(Level.DEBUG);
@@ -319,9 +329,16 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
}
if (user == null) { // Does not exists
- noOfUsers++;
- noOfGroups += groups.size();
-
+ //noOfNewUsers++;
+ newUserList.add(userName);
+ for (String group : groups) {
+ if (groupName2XGroupInfoMap.containsKey(group) && !newGroupList.contains(group)) {
+ modifiedGroupList.add(group);
+ } else {
+ //LOG.info("Adding new group " + group + " for user = " + userName);
+ newGroupList.add(group);
+ }
+ }
LOG.debug("INFO: addPMAccount(" + userName + ")" );
if (! isMockRun) {
if (addMUser(userName) == null) {
@@ -352,6 +369,11 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
for(String group : groups) {
if (! oldGroups.contains(group)) {
addGroups.add(group);
+ if (!groupName2XGroupInfoMap.containsKey(group)) {
+ newGroupList.add(group);
+ } else {
+ modifiedGroupList.add(group);
+ }
}else{
tempXGroupInfo=groupName2XGroupInfoMap.get(group);
if(tempXGroupInfo!=null && ! GROUP_SOURCE_EXTERNAL.equals(tempXGroupInfo.getGroupSource())){
@@ -403,10 +425,8 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
throw new Exception(msg);
}
} catch (Throwable t) {
- LOG.error("PolicyMgrUserGroupBuilder.addUserGroupInfo failed with exception: "
- + t.getMessage()
- + ", for user-group entry: "
- + ugInfo);
+ LOG.error("PolicyMgrUserGroupBuilder.addUserGroupInfo failed for user-group entry: "
+ + ugInfo.toString() + " with exception: ", t);
}
}
addXUserGroupInfo(user, addGroups);
@@ -507,7 +527,19 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
}
}
}
- noOfGroups += addGroups.size()+updateGroups.size();
+ //LOG.info("Adding new groups " + addGroups + " for user = " + userName);
+ if (isStartupFlag) {
+ modifiedGroupList.addAll(oldGroups);
+ LOG.debug("Adding user to modified user list: " + userName + ": " + oldGroups);
+ modifiedUserList.add(userName);
+
+ } else {
+ if (!addGroups.isEmpty() || !delGroups.isEmpty() || !updateGroups.isEmpty()) {
+ modifiedUserList.add(userName);
+ }
+ modifiedGroupList.addAll(updateGroups);
+ modifiedGroupList.addAll(delGroups);
+ }
}
}
@@ -558,8 +590,8 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
while (retrievedCount < totalCount) {
WebResource r = c.resource(getURL(PM_USER_LIST_URI))
- .queryParam("pageSize", recordsToPullPerCall)
- .queryParam("startIndex", String.valueOf(retrievedCount));
+ .queryParam("pageSize", recordsToPullPerCall)
+ .queryParam("startIndex", String.valueOf(retrievedCount));
String response = r.accept(MediaType.APPLICATION_JSON_TYPE).get(String.class);
@@ -594,8 +626,8 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
while (retrievedCount < totalCount) {
WebResource r = c.resource(getURL(PM_USER_GROUP_MAP_LIST_URI))
- .queryParam("pageSize", recordsToPullPerCall)
- .queryParam("startIndex", String.valueOf(retrievedCount));
+ .queryParam("pageSize", recordsToPullPerCall)
+ .queryParam("startIndex", String.valueOf(retrievedCount));
String response = r.accept(MediaType.APPLICATION_JSON_TYPE).get(String.class);
@@ -972,7 +1004,7 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
ret = gson.fromJson(response, MUserInfo.class);
- LOG.debug("MUser Creation successful " + ret);
+ LOG.debug("MUser Creation successful " + ret);
return ret;
}
@@ -1119,7 +1151,7 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
//* Build the group info object and do the rest call
if ( ! isMockRun ) {
- group = addGroupInfo(groupName);
+ group = addGroupInfo(groupName);
if ( group != null) {
addGroupToList(group);
} else {
@@ -1128,7 +1160,6 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
throw new Exception(msg);
}
}
- noOfGroups++;
}
}
@@ -1197,8 +1228,15 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
@Override
- public void addOrUpdateGroup(String group, List<String> users) throws Throwable {
- // TODO Auto-generated method stub
+ public void addOrUpdateGroup(String groupName, List<String> users) throws Throwable {
+ if (users == null || users.isEmpty()) {
+ if (groupName2XGroupInfoMap.containsKey(groupName)) {
+ modifiedGroupList.add(groupName);
+ } else {
+ newGroupList.add(groupName);
+ }
+ }
+ addOrUpdateGroup(groupName);
}
@@ -1208,8 +1246,15 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
if (! isMockRun) {
addUserGroupAuditInfo(ugsyncAuditInfo);
}
- noOfUsers = 0;
- noOfGroups = 0;
+ noOfNewUsers = 0;
+ noOfNewGroups = 0;
+ noOfModifiedUsers = 0;
+ noOfModifiedGroups = 0;
+ isStartupFlag = false;
+ newUserList.clear();
+ modifiedUserList.clear();
+ newGroupList.clear();
+ modifiedGroupList.clear();
}
private UgsyncAuditInfo addUserGroupAuditInfo(UgsyncAuditInfo auditInfo) {
@@ -1219,11 +1264,19 @@ public class PolicyMgrUserGroupBuilder implements UserGroupSink {
LOG.error("Failed to generate user group audit info");
return ret;
}
- auditInfo.setNoOfUsers(Integer.toUnsignedLong(noOfUsers));
- auditInfo.setNoOfGroups(Integer.toUnsignedLong(noOfGroups));
- //auditInfo.setUserName("rangerusersync");
+ noOfNewUsers = newUserList.size();
+ noOfModifiedUsers = modifiedUserList.size();
+ noOfNewGroups = newGroupList.size();
+ noOfModifiedGroups = modifiedGroupList.size();
+
+ auditInfo.setNoOfNewUsers(Integer.toUnsignedLong(noOfNewUsers));
+ auditInfo.setNoOfNewGroups(Integer.toUnsignedLong(noOfNewGroups));
+ auditInfo.setNoOfModifiedUsers(Integer.toUnsignedLong(noOfModifiedUsers));
+ auditInfo.setNoOfModifiedGroups(Integer.toUnsignedLong(noOfModifiedGroups));
auditInfo.setSessionId("");
- LOG.debug("INFO: addAuditInfo(" + auditInfo.getNoOfUsers() + ", " + auditInfo.getNoOfGroups() + ", " + auditInfo.getSyncSource() + ")");
+ LOG.debug("INFO: addAuditInfo(" + auditInfo.getNoOfNewUsers() + ", " + auditInfo.getNoOfNewGroups()
+ + ", " + auditInfo.getNoOfModifiedUsers() + ", " + auditInfo.getNoOfModifiedGroups()
+ + ", " + auditInfo.getSyncSource() + ")");
if (authenticationType != null
&& AUTH_KERBEROS.equalsIgnoreCase(authenticationType)
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
index c58589e..f314bd8 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
@@ -24,11 +24,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.ArrayList;
-import java.util.Arrays;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
import com.google.common.annotations.VisibleForTesting;
import org.apache.log4j.Logger;
@@ -87,6 +85,7 @@ public class UnixUserGroupBuilder implements UserGroupSource {
private long groupFileModifiedAt = 0;
private UgsyncAuditInfo ugsyncAuditInfo;
private UnixSyncSourceInfo unixSyncSourceInfo;
+ private boolean isStartupFlag = false;
public static void main(String[] args) throws Throwable {
UnixUserGroupBuilder ugbuilder = new UnixUserGroupBuilder();
@@ -95,6 +94,7 @@ public class UnixUserGroupBuilder implements UserGroupSource {
}
public UnixUserGroupBuilder() {
+ isStartupFlag = true;
minimumUserId = Integer.parseInt(config.getMinUserId());
minimumGroupId = Integer.parseInt(config.getMinGroupId());
unixPasswordFile = config.getUnixPasswordFile();
@@ -156,29 +156,37 @@ public class UnixUserGroupBuilder implements UserGroupSource {
@Override
public void updateSink(UserGroupSink sink) throws Throwable {
+ DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ Date lastModifiedTime = new Date(passwordFileModifiedAt);
+ Date syncTime = new Date(System.currentTimeMillis());
+ unixSyncSourceInfo.setLastModified(formatter.format(lastModifiedTime));
+ unixSyncSourceInfo.setSyncTime(formatter.format(syncTime));
isUpdateSinkSucc = true;
- buildUserGroupInfo();
- unixSyncSourceInfo.setLastModified(Long.toString(passwordFileModifiedAt));
- unixSyncSourceInfo.setSyncTime(Long.toString(System.currentTimeMillis()));
+ if (isChanged() || isStartupFlag) {
+ buildUserGroupInfo();
- for (Map.Entry<String, List<String>> entry : user2GroupListMap.entrySet()) {
- String user = entry.getKey();
- List<String> groups = entry.getValue();
-
- try{
- sink.addOrUpdateUser(user, groups);
- }catch (Throwable t) {
- LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage()
- + ", for user: " + user
- + ", groups: " + groups);
- isUpdateSinkSucc = false;
+ for (Map.Entry<String, List<String>> entry : user2GroupListMap.entrySet()) {
+ String user = entry.getKey();
+ List<String> groups = entry.getValue();
+
+ try {
+ sink.addOrUpdateUser(user, groups);
+ } catch (Throwable t) {
+ LOG.error("sink.addOrUpdateUser failed with exception: " + t.getMessage()
+ + ", for user: " + user
+ + ", groups: " + groups);
+ isUpdateSinkSucc = false;
+ }
}
}
try {
+ unixSyncSourceInfo.setTotalUsersSynced(user2GroupListMap.size());
+ unixSyncSourceInfo.setTotalGroupsSynced(groupId2groupNameMap.size());
sink.postUserGroupAuditInfo(ugsyncAuditInfo);
} catch (Throwable t) {
- LOG.error("sink.postUserGroupAuditInfo failed with exception: " + t.getMessage());
+ LOG.error("sink.postUserGroupAuditInfo failed with exception: ", t);
}
+ isStartupFlag = false;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java
index f078cb8..7d44971 100644
--- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java
+++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java
@@ -109,14 +109,11 @@ public class UserGroupSync implements Runnable {
try{
if (config.isUserSyncEnabled()) {
- if (forceSync || ugSource.isChanged()) {
+ //if (forceSync) {
LOG.info("Begin: update user/group from source==>sink");
ugSource.updateSink(ugSink);
LOG.info("End: update user/group from source==>sink");
- }
- else {
- LOG.debug("UserGroupSource: no change found for synchronization.");
- }
+ //}
}
}catch(Throwable t){
LOG.error("Failed to sync user/group : ", t);
http://git-wip-us.apache.org/repos/asf/ranger/blob/7f61b148/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java
----------------------------------------------------------------------
diff --git a/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java b/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java
index 312ea9b..2bc3951 100644
--- a/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java
+++ b/ugsync/src/test/java/org/apache/ranger/usergroupsync/PolicyMgrUserGroupBuilderTest.java
@@ -47,6 +47,11 @@ public class PolicyMgrUserGroupBuilderTest extends PolicyMgrUserGroupBuilder {
allGroups.add(group);
}
+ @Override
+ public void addOrUpdateGroup(String group, List<String> users) {
+ addOrUpdateGroup(group);
+ }
+
public int getTotalUsers() {
return allUsers.size();
}
[2/3] ranger git commit: RANGER-1985: Fixed PMD violations
Posted by sp...@apache.org.
RANGER-1985: Fixed PMD violations
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/73224464
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/73224464
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/73224464
Branch: refs/heads/master
Commit: 732244640a88b66bc62c0a57d840997d8a70768e
Parents: 7f61b14
Author: Sailaja Polavarapu <sp...@hortonworks.com>
Authored: Mon Apr 2 18:00:14 2018 -0700
Committer: Sailaja Polavarapu <sp...@hortonworks.com>
Committed: Mon Apr 2 18:00:14 2018 -0700
----------------------------------------------------------------------
.../ranger/usergroupsync/UserGroupSync.java | 18 +++++-------------
1 file changed, 5 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/73224464/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java
index 7d44971..2befe33 100644
--- a/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java
+++ b/ugsync/src/main/java/org/apache/ranger/usergroupsync/UserGroupSync.java
@@ -73,8 +73,6 @@ public class UserGroupSync implements Runnable {
}
}
- boolean forceSync = false;
-
while (! shutdownFlag ) {
try {
LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds");
@@ -84,14 +82,10 @@ public class UserGroupSync implements Runnable {
}
try {
- syncUserGroup(forceSync);
-
- forceSync = false;
+ syncUserGroup();
}
catch(Throwable t) {
LOG.error("Failed to synchronize UserGroup information. Error details: ", t);
-
- forceSync = true; // force sync to the destination in the next attempt
}
}
@@ -104,16 +98,14 @@ public class UserGroupSync implements Runnable {
}
}
- private void syncUserGroup(boolean forceSync) throws Throwable {
+ private void syncUserGroup() throws Throwable {
UserGroupSyncConfig config = UserGroupSyncConfig.getInstance();
try{
if (config.isUserSyncEnabled()) {
- //if (forceSync) {
- LOG.info("Begin: update user/group from source==>sink");
- ugSource.updateSink(ugSink);
- LOG.info("End: update user/group from source==>sink");
- //}
+ LOG.info("Begin: update user/group from source==>sink");
+ ugSource.updateSink(ugSink);
+ LOG.info("End: update user/group from source==>sink");
}
}catch(Throwable t){
LOG.error("Failed to sync user/group : ", t);
[3/3] ranger git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/ranger into BUG-98019
Posted by sp...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/ranger into BUG-98019
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/cafe7aee
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/cafe7aee
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/cafe7aee
Branch: refs/heads/master
Commit: cafe7aee09c1f4a51cdfd107d6701d8853928bae
Parents: 7322446 cf69754
Author: Sailaja Polavarapu <sp...@hortonworks.com>
Authored: Tue Apr 3 10:39:40 2018 -0700
Committer: Sailaja Polavarapu <sp...@hortonworks.com>
Committed: Tue Apr 3 10:39:40 2018 -0700
----------------------------------------------------------------------
.../authorization/kylin/authorizer/RangerAdminClientImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------