You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ve...@apache.org on 2016/06/09 13:02:49 UTC
incubator-ranger git commit: RANGER-997 : Improve Policy Listing
performance on Reports page having more than 1000 policies
Repository: incubator-ranger
Updated Branches:
refs/heads/master 23f540e56 -> c38a718ed
RANGER-997 : Improve Policy Listing performance on Reports page having more than 1000 policies
Signed-off-by: Velmurugan Periasamy <ve...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/c38a718e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/c38a718e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/c38a718e
Branch: refs/heads/master
Commit: c38a718ede6d9888e6c1a413a66f26df0b45a04e
Parents: 23f540e
Author: pradeep agrawal <pr...@freestoneinfotech.com>
Authored: Tue Jun 7 20:00:55 2016 +0530
Committer: Velmurugan Periasamy <ve...@apache.org>
Committed: Thu Jun 9 09:02:28 2016 -0400
----------------------------------------------------------------------
.../org/apache/ranger/biz/ServiceDBStore.java | 96 +++++++++++++++++++-
.../ranger/service/RangerBaseModelService.java | 2 +-
.../ranger/service/RangerPolicyServiceBase.java | 52 -----------
.../apache/ranger/biz/TestServiceDBStore.java | 13 ---
.../service/TestRangerPolicyServiceBase.java | 36 --------
5 files changed, 93 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index a659d45..bf03e30 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -22,10 +22,14 @@ package org.apache.ranger.biz;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.Map.Entry;
import java.util.StringTokenizer;
import java.io.ByteArrayInputStream;
@@ -1905,7 +1909,7 @@ public class ServiceDBStore extends AbstractServiceStore {
if(LOG.isDebugEnabled()) {
LOG.debug("==> ServiceDBStore.getPolicies()");
}
- RangerPolicyList policyList = policyService.searchRangerPolicies(filter);
+ RangerPolicyList policyList = searchRangerPolicies(filter);
List<RangerPolicy> ret = policyList.getPolicies();
if(LOG.isDebugEnabled()) {
LOG.debug("<== ServiceDBStore.getPolicies()");
@@ -1926,7 +1930,7 @@ public class ServiceDBStore extends AbstractServiceStore {
if (!CollectionUtils.isEmpty(serviceTypeList)) {
for (String serviceType : serviceTypeList) {
filter.setParam("serviceType", serviceType);
- RangerPolicyList policyList = policyService.searchRangerPolicies(filter);
+ RangerPolicyList policyList = searchRangerPolicies(filter);
if (policyList!=null){
retTemp = policyList.getPolicies();
if(!CollectionUtils.isEmpty(retTemp)) {
@@ -1947,7 +1951,7 @@ public class ServiceDBStore extends AbstractServiceStore {
}
}
} else {
- RangerPolicyList policyList = policyService.searchRangerPolicies(filter);
+ RangerPolicyList policyList = searchRangerPolicies(filter);
ret = policyList.getPolicies();
if (!CollectionUtils.isEmpty(ret)) {
for (RangerPolicy policy : ret) {
@@ -2015,7 +2019,7 @@ public class ServiceDBStore extends AbstractServiceStore {
LOG.debug("==> ServiceDBStore.getPaginatedPolicies(+ " + filter + ")");
}
- RangerPolicyList policyList = policyService.searchRangerPolicies(filter);
+ RangerPolicyList policyList = searchRangerPolicies(filter);
if (LOG.isDebugEnabled()) {
LOG.debug("before filter: count=" + policyList.getListSize());
@@ -3627,4 +3631,88 @@ public class ServiceDBStore extends AbstractServiceStore {
cellStatus.setCellStyle(cellStyle);
cellStatus.setCellValue("Status");
}
+
+
+ private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) {
+ List<RangerPolicy> policyList = new ArrayList<RangerPolicy>();
+ RangerPolicyList retList = new RangerPolicyList();
+ Map<Long,RangerPolicy> policyMap=new HashMap<Long,RangerPolicy>();
+ Set<Long> processedServices=new HashSet<Long>();
+ Set<Long> processedPolicies=new HashSet<Long>();
+ Comparator<RangerPolicy> comparator = new Comparator<RangerPolicy>() {
+ public int compare(RangerPolicy c1, RangerPolicy c2) {
+ return (int) ((c1.getId()).compareTo(c2.getId()));
+ }
+ };
+
+ List<XXPolicy> xPolList = (List<XXPolicy>) policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList);
+ if (!CollectionUtils.isEmpty(xPolList)) {
+ for (XXPolicy xXPolicy : xPolList) {
+ if(!processedServices.contains(xXPolicy.getService())){
+ loadRangerPolicies(xXPolicy.getService(),processedServices,policyMap);
+ }
+ }
+ }
+ String userName = searchFilter.getParam("user");
+ if (!StringUtils.isEmpty(userName)) {
+ searchFilter.removeParam("user");
+ Set<String> groupNames = daoMgr.getXXGroupUser().findGroupNamesByUserName(userName);
+ if (!CollectionUtils.isEmpty(groupNames)) {
+ List<XXPolicy> xPolList2 = null;
+ for (String groupName : groupNames) {
+ xPolList2 = new ArrayList<XXPolicy>();
+ searchFilter.setParam("group", groupName);
+ xPolList2 = (List<XXPolicy>) policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList);
+ if (!CollectionUtils.isEmpty(xPolList2)) {
+ for (XXPolicy xPol2 : xPolList2) {
+ if(xPol2!=null){
+ if(!processedPolicies.contains(xPol2.getId())){
+ if(!processedServices.contains(xPol2.getService())){
+ loadRangerPolicies(xPol2.getService(),processedServices,policyMap);
+ }
+ if(policyMap.containsKey(xPol2.getId())){
+ policyList.add(policyMap.get(xPol2.getId()));
+ processedPolicies.add(xPol2.getId());
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ if (!CollectionUtils.isEmpty(xPolList)) {
+ for (XXPolicy xPol : xPolList) {
+ if(xPol!=null){
+ if(!processedPolicies.contains(xPol.getId())){
+ if(!processedServices.contains(xPol.getService())){
+ loadRangerPolicies(xPol.getService(),processedServices,policyMap);
+ }
+ if(policyMap.containsKey(xPol.getId())){
+ policyList.add(policyMap.get(xPol.getId()));
+ processedPolicies.add(xPol.getId());
+ }
+ }
+ }
+ }
+ Collections.sort(policyList, comparator);
+ }
+ retList.setPolicies(policyList);
+ return retList;
+ }
+
+ private void loadRangerPolicies(Long serviceId,Set<Long> processedServices,Map<Long,RangerPolicy> policyMap){
+ try {
+ List<RangerPolicy> tempPolicyList = getServicePolicies(serviceId,null);
+ if(!CollectionUtils.isEmpty(tempPolicyList)){
+ for (RangerPolicy rangerPolicy : tempPolicyList) {
+ if(!policyMap.containsKey(rangerPolicy.getId())){
+ policyMap.put(rangerPolicy.getId(), rangerPolicy);
+ }
+ }
+ processedServices.add(serviceId);
+ }
+ } catch (Exception e) {
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
index 2d5440d..07765e0 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
@@ -307,7 +307,7 @@ public abstract class RangerBaseModelService<T extends XXDBBase, V extends Range
*
*/
- protected List<T> searchResources(SearchFilter searchCriteria,
+ public List<T> searchResources(SearchFilter searchCriteria,
List<SearchField> searchFieldList, List<SortField> sortFieldList,
VList vList) {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
index bde18bd..b7598ea 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
@@ -17,14 +17,7 @@
package org.apache.ranger.service;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-
import org.apache.commons.lang.StringUtils;
-import org.apache.ranger.authorization.utils.StringUtil;
import org.apache.ranger.common.GUIDUtil;
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.SearchField;
@@ -32,14 +25,11 @@ import org.apache.ranger.common.SortField;
import org.apache.ranger.common.SearchField.DATA_TYPE;
import org.apache.ranger.common.SearchField.SEARCH_TYPE;
import org.apache.ranger.common.SortField.SORT_ORDER;
-import org.apache.ranger.entity.XXPolicy;
import org.apache.ranger.entity.XXPolicyBase;
import org.apache.ranger.entity.XXService;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.util.SearchFilter;
-import org.apache.ranger.view.RangerPolicyList;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.util.CollectionUtils;
public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends RangerPolicy> extends
RangerBaseModelService<T, V> {
@@ -120,46 +110,4 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends
vObj.setIsAuditEnabled(xObj.getIsAuditEnabled());
return vObj;
}
-
- @SuppressWarnings("unchecked")
- public RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) {
- List<RangerPolicy> policyList = new ArrayList<RangerPolicy>();
- RangerPolicyList retList = new RangerPolicyList();
-
- Comparator<RangerPolicy> comparator = new Comparator<RangerPolicy>() {
- public int compare(RangerPolicy c1, RangerPolicy c2) {
- return (int) ((c1.getId()).compareTo(c2.getId()));
- }
- };
-
- List<XXPolicy> xPolList = (List<XXPolicy>) searchResources(searchFilter, searchFields, sortFields, retList);
- String userName = searchFilter.getParam("user");
- if (!StringUtil.isEmpty(userName)) {
- searchFilter.removeParam("user");
- Set<String> groupNames = daoMgr.getXXGroupUser().findGroupNamesByUserName(userName);
- if (!CollectionUtils.isEmpty(groupNames)) {
- List<XXPolicy> xPolList2 = null;
- for (String groupName : groupNames) {
- xPolList2 = new ArrayList<XXPolicy>();
- searchFilter.setParam("group", groupName);
- xPolList2 = (List<XXPolicy>) searchResources(searchFilter, searchFields, sortFields, retList);
- if (!CollectionUtils.isEmpty(xPolList2)) {
- for (XXPolicy xPol2 : xPolList2) {
- if(xPol2!=null){
- policyList.add(populateViewBean((T) xPol2));
- }
- }
- }
- }
- }
- }
- if (!CollectionUtils.isEmpty(xPolList)) {
- for (XXPolicy xPol : xPolList) {
- policyList.add(populateViewBean((T) xPol));
- }
- Collections.sort(policyList, comparator);
- }
- retList.setPolicies(policyList);
- return retList;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
index a3cd613..ea1563e 100644
--- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
+++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
@@ -2462,11 +2462,8 @@ public class TestServiceDBStore {
policyListObj.setStartIndex(0);
policyListObj.setTotalCount(10);
- Mockito.when(policyService.searchRangerPolicies(filter)).thenReturn(
- policyListObj);
List<RangerPolicy> dbRangerPolicy = serviceDBStore.getPolicies(filter);
Assert.assertNotNull(dbRangerPolicy);
- Mockito.verify(policyService).searchRangerPolicies(filter);
}
@Test
@@ -2483,13 +2480,9 @@ public class TestServiceDBStore {
policyListObj.setStartIndex(0);
policyListObj.setTotalCount(10);
- Mockito.when(policyService.searchRangerPolicies(filter)).thenReturn(
- policyListObj);
-
PList<RangerPolicy> dbRangerPolicyList = serviceDBStore
.getPaginatedPolicies(filter);
Assert.assertNotNull(dbRangerPolicyList);
- Mockito.verify(policyService).searchRangerPolicies(filter);
}
@Test
@@ -2596,14 +2589,9 @@ public class TestServiceDBStore {
filter.setParam(SearchFilter.POLICY_NAME, "policyName");
filter.setParam(SearchFilter.SERVICE_NAME, "serviceName");
- Mockito.when(policyService.searchRangerPolicies(filter)).thenReturn(
- policyList);
-
-
PList<RangerPolicy> dbRangerPolicyList = serviceDBStore
.getPaginatedServicePolicies(serviceName, filter);
Assert.assertNotNull(dbRangerPolicyList);
- Mockito.verify(policyService).searchRangerPolicies(filter);
}
@Test
@@ -2619,7 +2607,6 @@ public class TestServiceDBStore {
Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao);
Mockito.when(xServiceDao.getById(Id)).thenReturn(xService);
- thrown.expect(Exception.class);
PList<RangerPolicy> dbRangerPolicyList = serviceDBStore
.getPaginatedServicePolicies(rangerService.getId(), filter);
Mockito.verify(daoManager).getXXService();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java b/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java
index 75e2ebd..7910cbd 100644
--- a/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java
+++ b/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java
@@ -21,18 +21,13 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
-import javax.persistence.EntityManager;
import javax.ws.rs.WebApplicationException;
-
import org.apache.ranger.biz.RangerBizUtil;
import org.apache.ranger.common.ContextUtil;
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.RESTErrorUtil;
import org.apache.ranger.common.RangerSearchUtil;
-import org.apache.ranger.common.SearchField;
import org.apache.ranger.common.UserSessionBase;
-import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.db.XXServiceDao;
import org.apache.ranger.entity.XXDBBase;
@@ -43,10 +38,8 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
-import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.security.context.RangerContextHolder;
import org.apache.ranger.security.context.RangerSecurityContext;
-import org.apache.ranger.view.RangerPolicyList;
import org.junit.Assert;
import org.junit.FixMethodOrder;
import org.junit.Rule;
@@ -243,33 +236,4 @@ public class TestRangerPolicyServiceBase {
Mockito.verify(daoManager).getXXService();
}
- @Test
- public void test4searchRangerPolicies() {
- EntityManager entityManager = Mockito.mock(EntityManager.class);
- SearchFilter searchFilter = new SearchFilter();
- searchFilter.setParam(SearchFilter.POLICY_NAME, "policyName");
- searchFilter.setParam(SearchFilter.SERVICE_NAME, "serviceName");
-
- String searchString = "policyName";
- String sortString = "asc";
- List<SearchField> searchFieldList = new ArrayList<SearchField>();
- boolean isCountQuery = false;
-
- BaseDao baseDao = Mockito.mock(BaseDao.class);
-
- Mockito.when(daoManager.getDaoForClassName(Mockito.anyString()))
- .thenReturn(baseDao);
- Mockito.when(daoManager.getEntityManager()).thenReturn(entityManager);
- Mockito.when(
- searchUtil
- .createSearchQuery(entityManager, searchString,
- sortString, searchFilter, searchFieldList,
- isCountQuery)).thenReturn(null);
-
- RangerPolicyList dbRangerPolicyList = policyService
- .searchRangerPolicies(searchFilter);
- Assert.assertNotNull(dbRangerPolicyList);
- Mockito.verify(daoManager).getDaoForClassName(Mockito.anyString());
- }
-
}