You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by pr...@apache.org on 2020/06/02 08:58:22 UTC
[ranger] 01/02: RANGER-2789: GET API service/xusers/users turns
very slow when there are more than 1000 users
This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit f45054d1b933b5e65cb19ace014d3b2cb0d78132
Author: RickyMa <mh...@gmail.com>
AuthorDate: Tue Jun 2 13:50:48 2020 +0530
RANGER-2789: GET API service/xusers/users turns very slow when there are more than 1000 users
Signed-off-by: pradeep <pr...@apache.org>
---
.../main/java/org/apache/ranger/entity/XXUser.java | 6 +
.../org/apache/ranger/service/XUserService.java | 141 ++++++++++++++++++++-
.../apache/ranger/service/XUserServiceBase.java | 130 +++++++++++++++++++
.../main/java/org/apache/ranger/view/VXUser.java | 19 +++
4 files changed, 295 insertions(+), 1 deletion(-)
diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXUser.java b/security-admin/src/main/java/org/apache/ranger/entity/XXUser.java
index 0464e7b..14f0b64 100644
--- a/security-admin/src/main/java/org/apache/ranger/entity/XXUser.java
+++ b/security-admin/src/main/java/org/apache/ranger/entity/XXUser.java
@@ -37,6 +37,8 @@ import org.apache.ranger.common.AppConstants;
import org.apache.ranger.common.RangerCommonEnums;
import org.apache.ranger.common.RangerConstants;
+import java.util.Objects;
+
@Entity
@Table(name="x_user")
@@ -285,4 +287,8 @@ public class XXUser extends XXDBBase implements java.io.Serializable {
return null;
}
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), name, description, status, credStoreId);
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java
index 6ff8823..75aa006 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/XUserService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/XUserService.java
@@ -25,14 +25,20 @@ import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
import org.apache.ranger.biz.RangerBizUtil;
import org.apache.ranger.common.AppConstants;
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.PropertiesUtil;
import org.apache.ranger.common.RangerCommonEnums;
import org.apache.ranger.common.RangerConstants;
+import org.apache.ranger.common.SearchCriteria;
import org.apache.ranger.common.SearchField;
import org.apache.ranger.common.SortField;
import org.apache.ranger.common.StringUtil;
@@ -45,10 +51,13 @@ import org.apache.ranger.entity.XXUser;
import org.apache.ranger.util.RangerEnumUtil;
import org.apache.ranger.view.VXPortalUser;
import org.apache.ranger.view.VXUser;
+import org.apache.ranger.view.VXUserList;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
+
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.toMap;
@Service
@Scope("singleton")
@@ -207,6 +216,19 @@ public class XUserService extends XUserServiceBase<XXUser, VXUser> {
}
@Override
+ protected List<VXUser> mapEntityToViewBeans(Map<VXUser, XXUser> vxUserXXUserMap) {
+ if (MapUtils.isNotEmpty(vxUserXXUserMap)) {
+ Map<String, VXUser> userNameVXUserMap = vxUserXXUserMap.keySet()
+ .stream()
+ .collect(toMap(VXUser::getName, identity()));
+ super.mapEntityToViewBeans(vxUserXXUserMap);
+ populateUserListAttributes(userNameVXUserMap);
+ return new ArrayList<>(vxUserXXUserMap.keySet());
+ }
+ return new ArrayList<>();
+ }
+
+ @Override
public VXUser populateViewBean(XXUser xUser) {
VXUser vObj = super.populateViewBean(xUser);
vObj.setIsVisible(xUser.getIsVisible());
@@ -214,6 +236,38 @@ public class XUserService extends XUserServiceBase<XXUser, VXUser> {
return vObj;
}
+ @Override
+ public List<VXUser> populateViewBeans(List<XXUser> xUsers) {
+ List<VXUser> vObjList = super.populateViewBeans(xUsers);
+ if (CollectionUtils.isNotEmpty(vObjList) && CollectionUtils.isNotEmpty(xUsers) && xUsers.size() == vObjList.size()) {
+ Map<Long, VXUser> xUserIdVObjMap = new HashMap<>(xUsers.size());
+ for (int i = 0; i < xUsers.size(); ++i) {
+ VXUser vObj = vObjList.get(i);
+ XXUser xUser = xUsers.get(i);
+ vObj.setIsVisible(xUser.getIsVisible());
+ xUserIdVObjMap.put(xUser.getId(), vObj);
+ }
+ populateGroupList(xUserIdVObjMap);
+ }
+ return vObjList;
+ }
+
+ /**
+ * @param searchCriteria
+ * @return
+ */
+ @Override
+ public VXUserList searchXUsers(SearchCriteria searchCriteria) {
+ VXUserList returnList = new VXUserList();
+
+ @SuppressWarnings("unchecked")
+ List<XXUser> resultList = searchResources(searchCriteria,
+ searchFields, sortFields, returnList);
+
+ returnList.setVXUsers(populateViewBeans(resultList));
+ return returnList;
+ }
+
private void populateGroupList(Long xUserId, VXUser vObj) {
List<XXGroupUser> xGroupUserList = daoManager.getXXGroupUser()
.findByUserId(xUserId);
@@ -231,6 +285,40 @@ public class XUserService extends XUserServiceBase<XXUser, VXUser> {
vObj.setGroupNameList(groupNames);
}
+ private void populateGroupList(Map<Long, VXUser> xUserIdVObjMap) {
+ List<XXGroupUser> allXXGroupUsers = daoManager.getXXGroupUser().getAll();
+ if (MapUtils.isNotEmpty(xUserIdVObjMap) && CollectionUtils.isNotEmpty(allXXGroupUsers)) {
+ Map<Long, List<XXGroupUser>> userIdXXGroupUserMap = new HashMap<>(xUserIdVObjMap.size());
+ for (Map.Entry<Long, VXUser> xUserIdVXUserEntry : xUserIdVObjMap.entrySet()) {
+ Long xUserId = xUserIdVXUserEntry.getKey();
+ List<XXGroupUser> xxGroupUsers = allXXGroupUsers
+ .stream()
+ .filter(xXGroupUser -> Objects.equals(xXGroupUser.getUserId(), xUserId))
+ .collect(Collectors.toList());
+ userIdXXGroupUserMap.put(xUserId, xxGroupUsers);
+ }
+ for (Map.Entry<Long, List<XXGroupUser>> xUserIdXXGroupUserListEntry : userIdXXGroupUserMap.entrySet()) {
+ Long xUserId = xUserIdXXGroupUserListEntry.getKey();
+ List<XXGroupUser> xGroupUserList = xUserIdXXGroupUserListEntry.getValue();
+ Set<Long> groupIdList = new LinkedHashSet<>();
+ Set<String> groupNameList = new LinkedHashSet<>();
+ if (xGroupUserList != null) {
+ for (XXGroupUser xGroupUser : xGroupUserList) {
+ groupIdList.add(xGroupUser.getParentGroupId());
+ groupNameList.add(xGroupUser.getName());
+ }
+ }
+ List<Long> groups = new ArrayList<>(groupIdList);
+ List<String> groupNames = new ArrayList<>(groupNameList);
+ VXUser vObj = xUserIdVObjMap.get(xUserId);
+ if (vObj != null) {
+ vObj.setGroupIdList(groups);
+ vObj.setGroupNameList(groupNames);
+ }
+ }
+ }
+ }
+
private void populateUserAttributes(String userName, VXUser vObj) {
if (userName != null && !userName.isEmpty()) {
List<String> userRoleList =new ArrayList<String>();
@@ -260,6 +348,57 @@ public class XUserService extends XUserServiceBase<XXUser, VXUser> {
}
}
+ private void populateUserListAttributes(Map<String, VXUser> userNameVObjMap) {
+ List<XXPortalUser> allXPortalUsers = daoManager.getXXPortalUser().findAllXPortalUser();
+ List<XXPortalUserRole> allXPortalUserRoles = daoManager.getXXPortalUserRole().getAll();
+ if (MapUtils.isNotEmpty(userNameVObjMap) && CollectionUtils.isNotEmpty(allXPortalUsers)) {
+ Map<String, XXPortalUser> loginIdXXPortalUserMap = new HashMap<>(allXPortalUsers.size());
+ Map<Long, List<XXPortalUserRole>> userIdRoleMap = new HashMap<>();
+ for (XXPortalUser xPortalUser : allXPortalUsers) {
+ loginIdXXPortalUserMap.put(xPortalUser.getLoginId(), xPortalUser);
+ List<XXPortalUserRole> xxPortalUserRoles = new ArrayList<>();
+ if (allXPortalUserRoles != null) {
+ for (XXPortalUserRole xPortalUserRole : allXPortalUserRoles) {
+ if (Objects.equals(xPortalUserRole.getUserId(), xPortalUser.getId())) {
+ xxPortalUserRoles.add(xPortalUserRole);
+ }
+ }
+ }
+ userIdRoleMap.put(xPortalUser.getId(), xxPortalUserRoles);
+ }
+
+ for (Map.Entry<String, VXUser> userNameVObjEntry : userNameVObjMap.entrySet()) {
+ String userName = userNameVObjEntry.getKey();
+ VXUser vObj = userNameVObjEntry.getValue();
+ if (userName != null && !userName.isEmpty()) {
+ List<String> userRoleList = new ArrayList<>();
+ XXPortalUser xXPortalUser = loginIdXXPortalUserMap.get(userName);
+ if (xXPortalUser != null) {
+ vObj.setFirstName(xXPortalUser.getFirstName());
+ vObj.setLastName(xXPortalUser.getLastName());
+ vObj.setPassword(PropertiesUtil.getProperty("ranger.password.hidden"));
+ String emailAddress = xXPortalUser.getEmailAddress();
+ if (emailAddress != null
+ && stringUtil.validateEmail(emailAddress)) {
+ vObj.setEmailAddress(xXPortalUser.getEmailAddress());
+ }
+ vObj.setStatus(xXPortalUser.getStatus());
+ vObj.setUserSource(xXPortalUser.getUserSource());
+ List<XXPortalUserRole> gjUserRoleList = userIdRoleMap.get(
+ xXPortalUser.getId());
+ for (XXPortalUserRole gjUserRole : gjUserRoleList) {
+ userRoleList.add(gjUserRole.getUserRole());
+ }
+ }
+ if(userRoleList==null || userRoleList.isEmpty()){
+ userRoleList.add(RangerConstants.ROLE_USER);
+ }
+ vObj.setUserRoleList(userRoleList);
+ }
+ }
+ }
+ }
+
public List<XXTrxLog> getTransactionLog(VXUser vResource, String action) {
return getTransactionLog(vResource, null, action);
}
diff --git a/security-admin/src/main/java/org/apache/ranger/service/XUserServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/XUserServiceBase.java
index 9cdc14e..4fcdda2 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/XUserServiceBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/XUserServiceBase.java
@@ -24,17 +24,27 @@
*/
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
+
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.ranger.common.ContextUtil;
import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.common.UserSessionBase;
+import org.apache.ranger.entity.XXPortalUser;
import org.apache.ranger.entity.XXUser;
import org.apache.ranger.plugin.model.UserInfo;
import org.apache.ranger.view.VXUser;
import org.apache.ranger.view.VXUserList;
+import static java.util.stream.Collectors.toMap;
+
public abstract class XUserServiceBase<T extends XXUser, V extends VXUser>
extends AbstractBaseResourceService<T, V> {
public static final String NAME = "XUser";
@@ -66,6 +76,126 @@ public abstract class XUserServiceBase<T extends XXUser, V extends VXUser>
return vObj;
}
+ protected List<VXUser> mapEntityToViewBeans(Map<VXUser, XXUser> vxUserXXUserMap) {
+ List<VXUser> vxUsers = new ArrayList<>();
+ if (MapUtils.isNotEmpty(vxUserXXUserMap)) {
+ for (Map.Entry<VXUser, XXUser> vxUserXXUserEntry : vxUserXXUserMap.entrySet()) {
+ VXUser vObj = vxUserXXUserEntry.getKey();
+ XXUser mObj = vxUserXXUserEntry.getValue();
+ vObj.setName(mObj.getName());
+ vObj.setIsVisible(mObj.getIsVisible());
+ vObj.setDescription(mObj.getDescription());
+ vObj.setCredStoreId(mObj.getCredStoreId());
+ vObj.setOtherAttributes(mObj.getOtherAttributes());
+ vxUsers.add(vObj);
+ }
+ }
+ return vxUsers;
+ }
+
+ public List<VXUser> populateViewBeans(List<XXUser> resources) {
+ List<VXUser> viewBeans = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(resources)) {
+ Map<XXUser, VXUser> resourceViewBeanMap = new HashMap<>(resources.size());
+ Map<VXUser, XXUser> viewBeanResourceMap = new HashMap<>(resources.size());
+ for (XXUser resource : resources) {
+ VXUser viewBean = createViewObject();
+ viewBean.setCredStoreId(resource.getCredStoreId());
+ viewBean.setDescription(resource.getDescription());
+ viewBean.setName(resource.getName());
+ viewBean.setStatus(resource.getStatus());
+ resourceViewBeanMap.put(resource, viewBean);
+ viewBeanResourceMap.put(viewBean, resource);
+ viewBeans.add(viewBean);
+ }
+ populateViewBeans(resourceViewBeanMap);
+ mapEntityToViewBeans(viewBeanResourceMap);
+ }
+ return viewBeans;
+ }
+
+ protected void populateViewBeans(Map<XXUser, VXUser> resourceViewBeanMap) {
+ mapBaseAttributesToViewBeans(resourceViewBeanMap);
+ }
+
+ private void mapBaseAttributesToViewBeans(Map<XXUser, VXUser> resourceViewBeanMap) {
+ List<XXPortalUser> allXPortalUsers = daoManager.getXXPortalUser().findAllXPortalUser();
+ if (MapUtils.isNotEmpty(resourceViewBeanMap) && CollectionUtils.isNotEmpty(allXPortalUsers)) {
+ Map<Long, XXPortalUser> idXXPortalUserMap = allXPortalUsers
+ .stream()
+ .collect(toMap(XXPortalUser::getId, Function.identity()));
+ resourceViewBeanMap.forEach((resource, viewBean) -> {
+ viewBean.setId(resource.getId());
+
+ // TBD: Need to review this change later
+ viewBean.setMObj(resource);
+ viewBean.setCreateDate(resource.getCreateTime());
+ viewBean.setUpdateDate(resource.getUpdateTime());
+
+ Long ownerId = resource.getAddedByUserId();
+ UserSessionBase currentUserSession = ContextUtil
+ .getCurrentUserSession();
+
+ if (currentUserSession == null) {
+ return;
+ }
+
+ if (ownerId != null) {
+ XXPortalUser tUser = idXXPortalUserMap.get(
+ resource.getAddedByUserId());
+ if (tUser != null) {
+ if (tUser.getPublicScreenName() != null
+ && !tUser.getPublicScreenName().trim().isEmpty()
+ && !"null".equalsIgnoreCase(tUser.getPublicScreenName().trim())) {
+ viewBean.setOwner(tUser.getPublicScreenName());
+ } else {
+ if (tUser.getFirstName() != null
+ && !tUser.getFirstName().trim().isEmpty()
+ && !"null".equalsIgnoreCase(tUser.getFirstName().trim())) {
+ if (tUser.getLastName() != null
+ && !tUser.getLastName().trim().isEmpty()
+ && !"null".equalsIgnoreCase(tUser.getLastName().trim())) {
+ viewBean.setOwner(tUser.getFirstName() + " "
+ + tUser.getLastName());
+ } else {
+ viewBean.setOwner(tUser.getFirstName());
+ }
+ } else {
+ viewBean.setOwner(tUser.getLoginId());
+ }
+ }
+ }
+ }
+ if (resource.getUpdatedByUserId() != null) {
+ XXPortalUser tUser = idXXPortalUserMap.get(
+ resource.getUpdatedByUserId());
+ if (tUser != null) {
+ if (tUser.getPublicScreenName() != null
+ && !tUser.getPublicScreenName().trim().isEmpty()
+ && !"null".equalsIgnoreCase(tUser.getPublicScreenName().trim())) {
+ viewBean.setUpdatedBy(tUser.getPublicScreenName());
+ } else {
+ if (tUser.getFirstName() != null
+ && !tUser.getFirstName().trim().isEmpty()
+ && !"null".equalsIgnoreCase(tUser.getFirstName().trim())) {
+ if (tUser.getLastName() != null
+ && !tUser.getLastName().trim().isEmpty()
+ && !"null".equalsIgnoreCase(tUser.getLastName().trim())) {
+ viewBean.setUpdatedBy(tUser.getFirstName() + " "
+ + tUser.getLastName());
+ } else {
+ viewBean.setUpdatedBy(tUser.getFirstName());
+ }
+ } else {
+ viewBean.setUpdatedBy(tUser.getLoginId());
+ }
+ }
+ }
+ }
+ });
+ }
+ }
+
/**
* @param searchCriteria
* @return
diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXUser.java b/security-admin/src/main/java/org/apache/ranger/view/VXUser.java
index 96f6468..d6f53fd 100644
--- a/security-admin/src/main/java/org/apache/ranger/view/VXUser.java
+++ b/security-admin/src/main/java/org/apache/ranger/view/VXUser.java
@@ -25,6 +25,7 @@
*/
import java.util.Collection;
+import java.util.Objects;
import javax.xml.bind.annotation.XmlRootElement;
@@ -305,6 +306,24 @@ public class VXUser extends VXDataObject implements java.io.Serializable {
this.otherAttributes = otherAttributes;
}
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ VXUser vxUser = (VXUser) o;
+
+ return Objects.equals(status, vxUser.status) &&
+ Objects.equals(name, vxUser.name) &&
+ Objects.equals(description, vxUser.description) &&
+ Objects.equals(credStoreId, vxUser.credStoreId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), name, description, credStoreId, status);
+ }
+
/**
* This return the bean content in string format
* @return formatedStr