You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2022/10/05 22:19:49 UTC
[ranger] branch master updated: RANGER-3941: optimize cache refresh in RangerUserStoreCache
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 2e7573773 RANGER-3941: optimize cache refresh in RangerUserStoreCache
2e7573773 is described below
commit 2e757377352e223055086e42c65c4b79e875c3e5
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Tue Oct 4 10:37:43 2022 -0700
RANGER-3941: optimize cache refresh in RangerUserStoreCache
---
.../apache/ranger/plugin/util/RangerUserStore.java | 24 +++++++-
.../main/java/org/apache/ranger/biz/XUserMgr.java | 65 +++++++++++++---------
.../apache/ranger/common/RangerUserStoreCache.java | 56 +++++++++----------
.../java/org/apache/ranger/rest/XUserREST.java | 4 +-
.../java/org/apache/ranger/biz/TestXUserMgr.java | 2 +-
5 files changed, 93 insertions(+), 58 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerUserStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerUserStore.java
index 97e05d150..d0629152c 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerUserStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerUserStore.java
@@ -131,20 +131,40 @@ public class RangerUserStore implements Serializable {
sb.append("RangerUserStore={")
.append("userStoreVersion=").append(userStoreVersion).append(", ")
.append("userStoreUpdateTime=").append(userStoreUpdateTime).append(", ");
+
sb.append("users={");
if(MapUtils.isNotEmpty(userAttrMapping)) {
for(String user : userAttrMapping.keySet()) {
- sb.append(user);
+ sb.append(user).append(" ");
}
}
sb.append("}, ");
+
sb.append("groups={");
if(MapUtils.isNotEmpty(groupAttrMapping)) {
for(String group : groupAttrMapping.keySet()) {
- sb.append(group);
+ sb.append(group).append(" ");
+ }
+ }
+ sb.append("}");
+
+ sb.append(", userGroupMapping={");
+ if(MapUtils.isNotEmpty(userGroupMapping)) {
+ for(Map.Entry<String, Set<String>> entry : userGroupMapping.entrySet()) {
+ String user = entry.getKey();
+ Set<String> userGroups = entry.getValue();
+
+ sb.append(user).append("[");
+ if (CollectionUtils.isNotEmpty(userGroups)) {
+ for (String userGroup : userGroups) {
+ sb.append(userGroup).append(" ");
+ }
+ }
+ sb.append("] ");
}
}
sb.append("}");
+
sb.append("}");
return sb;
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java
index 04968ecc5..0a03da567 100755
--- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java
@@ -271,6 +271,8 @@ public class XUserMgr extends XUserMgrBase {
assignPermissionToUser(vXPortalUser, true);
}
+ updateUserStoreVersion("createXUser(" + vXUser.getName() + ")");
+
return createdXUser;
}
@@ -504,6 +506,8 @@ public class XUserMgr extends XUserMgrBase {
trxLogList.addAll(createOrDelGrpUserWithUpdatedGrpId(vXUser, groupIdList,userId, groupUsersToRemove));
xaBizUtil.createTrxLog(trxLogList);
+ updateUserStoreVersion("updateXUser(" + vXUser.getName() + ")");
+
return vXUser;
}
private List<XXTrxLog> createOrDelGrpUserWithUpdatedGrpId(VXUser vXUser, Collection<Long> groupIdList,Long userId, List<Long> groupUsersToRemove) {
@@ -605,11 +609,9 @@ public class XUserMgr extends XUserMgrBase {
assignPermissionToUser(vXPortalUser, true);
}
vxUGInfo.setXgroupInfo(vxg);
- try {
- daoManager.getXXGlobalState().onGlobalAppDataChange(RANGER_USER_GROUP_GLOBAL_STATE_NAME);
- } catch (Exception excp) {
- logger.error("createXUserGroupFromMap(" + vXUser.getName() + ") failed", excp);
- }
+
+ updateUserStoreVersion("createXUserGroupFromMap(" + vXUser.getName() + ")");
+
return vxUGInfo;
}
@@ -667,11 +669,7 @@ public class XUserMgr extends XUserMgrBase {
vxGUInfo.setXuserInfo(vxu);
- try {
- daoManager.getXXGlobalState().onGlobalAppDataChange(RANGER_USER_GROUP_GLOBAL_STATE_NAME);
- } catch (Exception excp) {
- logger.error("createXGroupUserFromMap(" + vXGroup.getName() + ") failed", excp);
- }
+ updateUserStoreVersion("createXGroupUserFromMap(" + vXGroup.getName() + ")");
return vxGUInfo;
}
@@ -725,7 +723,11 @@ public class XUserMgr extends XUserMgrBase {
checkAdminAccess();
xaBizUtil.blockAuditorRoleUser();
validatePassword(vXUser);
- return xUserService.createXUserWithOutLogin(vXUser);
+ VXUser ret = xUserService.createXUserWithOutLogin(vXUser);
+
+ updateUserStoreVersion("createXUserWithOutLogin(" + vXUser.getName() + ")");
+
+ return ret;
}
public VXUser createExternalUser(String userName) {
@@ -745,13 +747,21 @@ public class XUserMgr extends XUserMgrBase {
List<XXTrxLog> trxLogList = xGroupService.getTransactionLog(vXGroup,
"create");
xaBizUtil.createTrxLog(trxLogList);
+
+ updateUserStoreVersion("createXGroup(" + vXGroup.getName() + ")");
+
return vXGroup;
}
public VXGroup createXGroupWithoutLogin(VXGroup vXGroup) {
checkAdminAccess();
xaBizUtil.blockAuditorRoleUser();
- return xGroupService.createXGroupWithOutLogin(vXGroup);
+
+ VXGroup ret = xGroupService.createXGroupWithOutLogin(vXGroup);
+
+ updateUserStoreVersion("createXGroupWithoutLogin(" + vXGroup.getName() + ")");
+
+ return ret;
}
public VXGroupUser createXGroupUser(VXGroupUser vXGroupUser) {
@@ -2584,20 +2594,19 @@ public class XUserMgr extends XUserMgrBase {
return daoManager.getXXUser().findGroupsByUserIds();
}
- public RangerUserStore getRangerUserStore(Long lastKnownUserStoreVersion) throws Exception {
- RangerUserStore ret = null;
- Long rangerUserStoreVersionInDB = getUserStoreVersion();
-
+ public RangerUserStore getRangerUserStoreIfUpdated(Long lastKnownUserStoreVersion) throws Exception {
if (logger.isDebugEnabled()) {
- logger.debug("==> XUserMgr.getRangerUserStore() lastKnownUserStoreVersion= " + lastKnownUserStoreVersion + " rangerUserStoreVersionInDB= " + rangerUserStoreVersionInDB);
+ logger.debug("==> XUserMgr.getRangerUserStoreIfUpdated(lastKnownUserStoreVersion=" + lastKnownUserStoreVersion + ")");
}
- if (rangerUserStoreVersionInDB != null) {
- ret = RangerUserStoreCache.getInstance().getLatestRangerUserStoreOrCached(this, lastKnownUserStoreVersion, rangerUserStoreVersionInDB);
+ RangerUserStore ret = RangerUserStoreCache.getInstance().getLatestRangerUserStoreOrCached(this);
+
+ if (ret != null && Objects.equals(ret.getUserStoreVersion(), lastKnownUserStoreVersion)) {
+ ret = null;
}
if (logger.isDebugEnabled()) {
- logger.debug("<= XUserMgr.getRangerUserStore() lastKnownUserStoreVersion= " + lastKnownUserStoreVersion + " rangerUserStoreVersionInDB= " + rangerUserStoreVersionInDB + " RangerRoles= " + ret);
+ logger.debug("<== XUserMgr.getRangerUserStoreIfUpdated(lastKnownUserStoreVersion=" + lastKnownUserStoreVersion + "): ret=" + ret);
}
return ret;
@@ -2775,11 +2784,9 @@ public class XUserMgr extends XUserMgrBase {
}
createXGroupWithoutLogin(vXGroup);
}
- try {
- daoManager.getXXGlobalState().onGlobalAppDataChange(RANGER_USER_GROUP_GLOBAL_STATE_NAME);
- } catch (Exception excp) {
- logger.error("createOrUpdateXGroups() failed", excp);
- }
+
+ updateUserStoreVersion("createOrUpdateXGroups(groupsCount=" + groups.getListSize() + ")");
+
return groups.getListSize();
}
@@ -3266,4 +3273,12 @@ public class XUserMgr extends XUserMgrBase {
}
}
}
+
+ private void updateUserStoreVersion(String label) {
+ try {
+ daoManager.getXXGlobalState().onGlobalAppDataChange(RANGER_USER_GROUP_GLOBAL_STATE_NAME);
+ } catch (Exception excp) {
+ logger.error(label + ": userStore version update failed", excp);
+ }
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerUserStoreCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerUserStoreCache.java
index fd53bae45..0cb0ccca8 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/RangerUserStoreCache.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/RangerUserStoreCache.java
@@ -62,56 +62,56 @@ public class RangerUserStoreCache {
return this.rangerUserStore;
}
- public RangerUserStore getLatestRangerUserStoreOrCached(XUserMgr xUserMgr, Long lastKnownUserStoreVersion, Long rangerUserStoreVersionInDB) throws Exception {
- RangerUserStore ret = null;
-
- if (lastKnownUserStoreVersion == null || !lastKnownUserStoreVersion.equals(rangerUserStoreVersionInDB)) {
- ret = getLatestRangerUserStore(xUserMgr, lastKnownUserStoreVersion, rangerUserStoreVersionInDB);
- } else {
- ret = null;
- }
-
- return ret;
- }
-
- public RangerUserStore getLatestRangerUserStore(XUserMgr xUserMgr, Long lastKnownUserStoreVersion, Long rangerUserStoreVersionInDB) throws Exception {
- RangerUserStore ret = null;
- boolean lockResult = false;
+ public RangerUserStore getLatestRangerUserStoreOrCached(XUserMgr xUserMgr) throws Exception {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> RangerUserStoreCache.getLatestRangerUserStore(lastKnownUserStoreVersion= " + lastKnownUserStoreVersion + " rangerUserStoreVersionInDB= " + rangerUserStoreVersionInDB + ")");
+ LOG.debug("==> RangerUserStoreCache.getLatestRangerUserStoreOrCached()");
}
+ RangerUserStore ret = null;
+ boolean lockResult = false;
+
try {
lockResult = lock.tryLock(waitTimeInSeconds, TimeUnit.SECONDS);
if (lockResult) {
- final Set<UserInfo> rangerUsersInDB = xUserMgr.getUsers();
- final Set<GroupInfo> rangerGroupsInDB = xUserMgr.getGroups();
- final Map<String, Set<String>> userGroups = xUserMgr.getUserGroups();
- if (LOG.isDebugEnabled()) {
- LOG.debug("No. of users from DB = " + rangerUsersInDB.size() + " and no. of groups from DB = " + rangerGroupsInDB.size());
- LOG.debug("No. of userGroupMappings = " + userGroups.size());
- }
+ Long cachedUserStoreVersion = rangerUserStore.getUserStoreVersion();
+ Long dbUserStoreVersion = xUserMgr.getUserStoreVersion();
+
+ if (!Objects.equals(cachedUserStoreVersion, dbUserStoreVersion)) {
+ LOG.info("RangerUserStoreCache refreshing from version " + cachedUserStoreVersion + " to " + dbUserStoreVersion);
+
+ final Set<UserInfo> rangerUsersInDB = xUserMgr.getUsers();
+ final Set<GroupInfo> rangerGroupsInDB = xUserMgr.getGroups();
+ final Map<String, Set<String>> userGroups = xUserMgr.getUserGroups();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No. of users from DB = " + rangerUsersInDB.size() + " and no. of groups from DB = " + rangerGroupsInDB.size());
+ LOG.debug("No. of userGroupMappings = " + userGroups.size());
+ }
- ret = new RangerUserStore(rangerUserStoreVersionInDB, rangerUsersInDB, rangerGroupsInDB, userGroups);
- rangerUserStore = ret;
+ rangerUserStore = new RangerUserStore(dbUserStoreVersion, rangerUsersInDB, rangerGroupsInDB, userGroups);
+
+ LOG.info("RangerUserStoreCache refreshed from version " + cachedUserStoreVersion + " to " + dbUserStoreVersion + ": users=" + rangerUsersInDB.size() + ", groups=" + rangerGroupsInDB.size() + ", userGroupMappings=" + userGroups.size());
+ }
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Could not get lock in [" + waitTimeInSeconds + "] seconds, returning cached RangerUserStore");
}
- ret = getRangerUserStore();
}
} catch (InterruptedException exception) {
- LOG.error("RangerUserStoreCache.getLatestRangerUserStore:lock got interrupted..", exception);
+ LOG.error("RangerUserStoreCache.getLatestRangerUserStoreOrCached:lock got interrupted..", exception);
} finally {
+ ret = rangerUserStore;
+
if (lockResult) {
lock.unlock();
}
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== RangerUserStoreCache.getLatestRangerUserStore(lastKnownUserStoreVersion= " + lastKnownUserStoreVersion + " rangerUserStoreVersionInDB= " + rangerUserStoreVersionInDB + " RangerUserStore= " + ret + ")");
+ LOG.debug("<== RangerUserStoreCache.getLatestRangerUserStoreOrCached(): ret=" + ret);
}
+
return ret;
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java
index 2b9df4946..02ab8a1cf 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/XUserREST.java
@@ -1333,7 +1333,7 @@ public class XUserREST {
if (xService != null) {
- RangerUserStore rangerUserStore = xUserMgr.getRangerUserStore(lastKnownUserStoreVersion);
+ RangerUserStore rangerUserStore = xUserMgr.getRangerUserStoreIfUpdated(lastKnownUserStoreVersion);
if (rangerUserStore == null) {
downloadedVersion = lastKnownUserStoreVersion;
@@ -1416,7 +1416,7 @@ public class XUserREST {
}
if (isAllowed) {
- RangerUserStore rangerUserStore = xUserMgr.getRangerUserStore(lastKnownUserStoreVersion);
+ RangerUserStore rangerUserStore = xUserMgr.getRangerUserStoreIfUpdated(lastKnownUserStoreVersion);
if (rangerUserStore == null) {
downloadedVersion = lastKnownUserStoreVersion;
httpCode = HttpServletResponse.SC_NOT_MODIFIED;
diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java
index 5ebd42eef..629cfc771 100644
--- a/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java
+++ b/security-admin/src/test/java/org/apache/ranger/biz/TestXUserMgr.java
@@ -4138,7 +4138,7 @@ public class TestXUserMgr {
XXGlobalStateDao xxGlobalStateDao = Mockito.mock(XXGlobalStateDao.class);
Mockito.when(daoManager.getXXGlobalState()).thenReturn(xxGlobalStateDao);
Mockito.when(xxGlobalStateDao.getAppDataVersion(RANGER_USER_GROUP_GLOBAL_STATE_NAME)).thenReturn(lastKnownUserStoreVersion);
- xUserMgr.getRangerUserStore(lastKnownUserStoreVersion);
+ xUserMgr.getRangerUserStoreIfUpdated(lastKnownUserStoreVersion);
}
@Test