You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ve...@apache.org on 2016/06/09 13:02:49 UTC

incubator-ranger git commit: RANGER-997 : Improve Policy Listing performance on Reports page having more than 1000 policies

Repository: incubator-ranger
Updated Branches:
  refs/heads/master 23f540e56 -> c38a718ed


RANGER-997 : Improve Policy Listing performance on Reports page having more than 1000 policies

Signed-off-by: Velmurugan Periasamy <ve...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/c38a718e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/c38a718e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/c38a718e

Branch: refs/heads/master
Commit: c38a718ede6d9888e6c1a413a66f26df0b45a04e
Parents: 23f540e
Author: pradeep agrawal <pr...@freestoneinfotech.com>
Authored: Tue Jun 7 20:00:55 2016 +0530
Committer: Velmurugan Periasamy <ve...@apache.org>
Committed: Thu Jun 9 09:02:28 2016 -0400

----------------------------------------------------------------------
 .../org/apache/ranger/biz/ServiceDBStore.java   | 96 +++++++++++++++++++-
 .../ranger/service/RangerBaseModelService.java  |  2 +-
 .../ranger/service/RangerPolicyServiceBase.java | 52 -----------
 .../apache/ranger/biz/TestServiceDBStore.java   | 13 ---
 .../service/TestRangerPolicyServiceBase.java    | 36 --------
 5 files changed, 93 insertions(+), 106 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index a659d45..bf03e30 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -22,10 +22,14 @@ package org.apache.ranger.biz;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.Map.Entry;
 import java.util.StringTokenizer;
 import java.io.ByteArrayInputStream;
@@ -1905,7 +1909,7 @@ public class ServiceDBStore extends AbstractServiceStore {
 		if(LOG.isDebugEnabled()) {
 			LOG.debug("==> ServiceDBStore.getPolicies()");
 		}
-		RangerPolicyList policyList = policyService.searchRangerPolicies(filter);
+		RangerPolicyList policyList = searchRangerPolicies(filter);
 		List<RangerPolicy> ret = policyList.getPolicies();
 		if(LOG.isDebugEnabled()) {
 			LOG.debug("<== ServiceDBStore.getPolicies()");
@@ -1926,7 +1930,7 @@ public class ServiceDBStore extends AbstractServiceStore {
 			if (!CollectionUtils.isEmpty(serviceTypeList)) {
 				for (String serviceType : serviceTypeList) {
 					filter.setParam("serviceType", serviceType);
-					RangerPolicyList policyList = policyService.searchRangerPolicies(filter);
+					RangerPolicyList policyList = searchRangerPolicies(filter);
 					if (policyList!=null){
 						retTemp = policyList.getPolicies();
 						if(!CollectionUtils.isEmpty(retTemp)) {
@@ -1947,7 +1951,7 @@ public class ServiceDBStore extends AbstractServiceStore {
 				}
 			}
 		} else {
-			RangerPolicyList policyList = policyService.searchRangerPolicies(filter);
+			RangerPolicyList policyList = searchRangerPolicies(filter);
 			ret = policyList.getPolicies();
 			if (!CollectionUtils.isEmpty(ret)) {
 				for (RangerPolicy policy : ret) {
@@ -2015,7 +2019,7 @@ public class ServiceDBStore extends AbstractServiceStore {
 			LOG.debug("==> ServiceDBStore.getPaginatedPolicies(+ " + filter + ")");
 		}
 
-		RangerPolicyList policyList = policyService.searchRangerPolicies(filter);
+		RangerPolicyList policyList = searchRangerPolicies(filter);
 
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("before filter: count=" + policyList.getListSize());
@@ -3627,4 +3631,88 @@ public class ServiceDBStore extends AbstractServiceStore {
 		cellStatus.setCellStyle(cellStyle);
 		cellStatus.setCellValue("Status");
 	}
+
+
+	private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) {
+		List<RangerPolicy> policyList = new ArrayList<RangerPolicy>();
+		RangerPolicyList retList = new RangerPolicyList();
+		Map<Long,RangerPolicy> policyMap=new HashMap<Long,RangerPolicy>();
+		Set<Long> processedServices=new HashSet<Long>();
+		Set<Long> processedPolicies=new HashSet<Long>();
+		Comparator<RangerPolicy> comparator = new Comparator<RangerPolicy>() {
+			public int compare(RangerPolicy c1, RangerPolicy c2) {
+				return (int) ((c1.getId()).compareTo(c2.getId()));
+			}
+		};
+
+		List<XXPolicy> xPolList = (List<XXPolicy>) policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList);
+		if (!CollectionUtils.isEmpty(xPolList)) {
+			for (XXPolicy xXPolicy : xPolList) {
+				if(!processedServices.contains(xXPolicy.getService())){
+					loadRangerPolicies(xXPolicy.getService(),processedServices,policyMap);
+				}
+			}
+		}
+		String userName = searchFilter.getParam("user");
+		if (!StringUtils.isEmpty(userName)) {
+			searchFilter.removeParam("user");
+			Set<String> groupNames = daoMgr.getXXGroupUser().findGroupNamesByUserName(userName);
+			if (!CollectionUtils.isEmpty(groupNames)) {
+				List<XXPolicy> xPolList2 = null;
+				for (String groupName : groupNames) {
+					xPolList2 = new ArrayList<XXPolicy>();
+					searchFilter.setParam("group", groupName);
+					xPolList2 = (List<XXPolicy>) policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList);
+					if (!CollectionUtils.isEmpty(xPolList2)) {
+						for (XXPolicy xPol2 : xPolList2) {
+							if(xPol2!=null){
+								if(!processedPolicies.contains(xPol2.getId())){
+									if(!processedServices.contains(xPol2.getService())){
+										loadRangerPolicies(xPol2.getService(),processedServices,policyMap);
+									}
+									if(policyMap.containsKey(xPol2.getId())){
+										policyList.add(policyMap.get(xPol2.getId()));
+										processedPolicies.add(xPol2.getId());
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+		if (!CollectionUtils.isEmpty(xPolList)) {
+			for (XXPolicy xPol : xPolList) {
+				if(xPol!=null){
+					if(!processedPolicies.contains(xPol.getId())){
+						if(!processedServices.contains(xPol.getService())){
+							loadRangerPolicies(xPol.getService(),processedServices,policyMap);
+						}
+						if(policyMap.containsKey(xPol.getId())){
+							policyList.add(policyMap.get(xPol.getId()));
+							processedPolicies.add(xPol.getId());
+						}
+					}
+				}
+			}
+			Collections.sort(policyList, comparator);
+		}
+		retList.setPolicies(policyList);
+		return retList;
+	}
+
+	private void loadRangerPolicies(Long serviceId,Set<Long> processedServices,Map<Long,RangerPolicy> policyMap){
+		try {
+			List<RangerPolicy> tempPolicyList = getServicePolicies(serviceId,null);
+			if(!CollectionUtils.isEmpty(tempPolicyList)){
+				for (RangerPolicy rangerPolicy : tempPolicyList) {
+					if(!policyMap.containsKey(rangerPolicy.getId())){
+						policyMap.put(rangerPolicy.getId(), rangerPolicy);
+					}
+				}
+				processedServices.add(serviceId);
+			}
+		} catch (Exception e) {
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
index 2d5440d..07765e0 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
@@ -307,7 +307,7 @@ public abstract class RangerBaseModelService<T extends XXDBBase, V extends Range
 	 * 
 	 */
 	
-	protected List<T> searchResources(SearchFilter searchCriteria,
+	public List<T> searchResources(SearchFilter searchCriteria,
 			List<SearchField> searchFieldList, List<SortField> sortFieldList,
 			VList vList) {
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
index bde18bd..b7598ea 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java
@@ -17,14 +17,7 @@
 
 package org.apache.ranger.service;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-
 import org.apache.commons.lang.StringUtils;
-import org.apache.ranger.authorization.utils.StringUtil;
 import org.apache.ranger.common.GUIDUtil;
 import org.apache.ranger.common.MessageEnums;
 import org.apache.ranger.common.SearchField;
@@ -32,14 +25,11 @@ import org.apache.ranger.common.SortField;
 import org.apache.ranger.common.SearchField.DATA_TYPE;
 import org.apache.ranger.common.SearchField.SEARCH_TYPE;
 import org.apache.ranger.common.SortField.SORT_ORDER;
-import org.apache.ranger.entity.XXPolicy;
 import org.apache.ranger.entity.XXPolicyBase;
 import org.apache.ranger.entity.XXService;
 import org.apache.ranger.plugin.model.RangerPolicy;
 import org.apache.ranger.plugin.util.SearchFilter;
-import org.apache.ranger.view.RangerPolicyList;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.util.CollectionUtils;
 
 public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends RangerPolicy> extends
 		RangerBaseModelService<T, V> {
@@ -120,46 +110,4 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends
 		vObj.setIsAuditEnabled(xObj.getIsAuditEnabled());
 		return vObj;
 	}
-
-	@SuppressWarnings("unchecked")
-	public RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) {
-		List<RangerPolicy> policyList = new ArrayList<RangerPolicy>();
-		RangerPolicyList retList = new RangerPolicyList();
-
-		Comparator<RangerPolicy> comparator = new Comparator<RangerPolicy>() {
-			public int compare(RangerPolicy c1, RangerPolicy c2) {
-				return (int) ((c1.getId()).compareTo(c2.getId()));
-			}
-		};
-
-		List<XXPolicy> xPolList = (List<XXPolicy>) searchResources(searchFilter, searchFields, sortFields, retList);
-		String userName = searchFilter.getParam("user");
-		if (!StringUtil.isEmpty(userName)) {
-			searchFilter.removeParam("user");
-			Set<String> groupNames = daoMgr.getXXGroupUser().findGroupNamesByUserName(userName);
-			if (!CollectionUtils.isEmpty(groupNames)) {
-				List<XXPolicy> xPolList2 = null;
-				for (String groupName : groupNames) {
-					xPolList2 = new ArrayList<XXPolicy>();
-					searchFilter.setParam("group", groupName);
-					xPolList2 = (List<XXPolicy>) searchResources(searchFilter, searchFields, sortFields, retList);
-					if (!CollectionUtils.isEmpty(xPolList2)) {
-						for (XXPolicy xPol2 : xPolList2) {
-							if(xPol2!=null){
-								policyList.add(populateViewBean((T) xPol2));
-							}
-						}
-					}
-				}
-			}
-		}
-		if (!CollectionUtils.isEmpty(xPolList)) {
-			for (XXPolicy xPol : xPolList) {
-				policyList.add(populateViewBean((T) xPol));
-			}
-			Collections.sort(policyList, comparator);
-		}
-		retList.setPolicies(policyList);
-		return retList;
-	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
index a3cd613..ea1563e 100644
--- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
+++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
@@ -2462,11 +2462,8 @@ public class TestServiceDBStore {
 		policyListObj.setStartIndex(0);
 		policyListObj.setTotalCount(10);
 
-		Mockito.when(policyService.searchRangerPolicies(filter)).thenReturn(
-				policyListObj);
 		List<RangerPolicy> dbRangerPolicy = serviceDBStore.getPolicies(filter);
 		Assert.assertNotNull(dbRangerPolicy);
-		Mockito.verify(policyService).searchRangerPolicies(filter);
 	}
 
 	@Test
@@ -2483,13 +2480,9 @@ public class TestServiceDBStore {
 		policyListObj.setStartIndex(0);
 		policyListObj.setTotalCount(10);
 
-		Mockito.when(policyService.searchRangerPolicies(filter)).thenReturn(
-				policyListObj);
-
 		PList<RangerPolicy> dbRangerPolicyList = serviceDBStore
 				.getPaginatedPolicies(filter);
 		Assert.assertNotNull(dbRangerPolicyList);
-		Mockito.verify(policyService).searchRangerPolicies(filter);
 	}
 
 	@Test
@@ -2596,14 +2589,9 @@ public class TestServiceDBStore {
 		filter.setParam(SearchFilter.POLICY_NAME, "policyName");
 		filter.setParam(SearchFilter.SERVICE_NAME, "serviceName");
 
-		Mockito.when(policyService.searchRangerPolicies(filter)).thenReturn(
-				policyList);
-
-
 		PList<RangerPolicy> dbRangerPolicyList = serviceDBStore
 				.getPaginatedServicePolicies(serviceName, filter);
 		Assert.assertNotNull(dbRangerPolicyList);
-		Mockito.verify(policyService).searchRangerPolicies(filter);
 	}
 
 	@Test
@@ -2619,7 +2607,6 @@ public class TestServiceDBStore {
 		Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao);
 		Mockito.when(xServiceDao.getById(Id)).thenReturn(xService);
 
-		thrown.expect(Exception.class);
 		PList<RangerPolicy> dbRangerPolicyList = serviceDBStore
 				.getPaginatedServicePolicies(rangerService.getId(), filter);
 		Mockito.verify(daoManager).getXXService();

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java b/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java
index 75e2ebd..7910cbd 100644
--- a/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java
+++ b/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java
@@ -21,18 +21,13 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
-import javax.persistence.EntityManager;
 import javax.ws.rs.WebApplicationException;
-
 import org.apache.ranger.biz.RangerBizUtil;
 import org.apache.ranger.common.ContextUtil;
 import org.apache.ranger.common.MessageEnums;
 import org.apache.ranger.common.RESTErrorUtil;
 import org.apache.ranger.common.RangerSearchUtil;
-import org.apache.ranger.common.SearchField;
 import org.apache.ranger.common.UserSessionBase;
-import org.apache.ranger.common.db.BaseDao;
 import org.apache.ranger.db.RangerDaoManager;
 import org.apache.ranger.db.XXServiceDao;
 import org.apache.ranger.entity.XXDBBase;
@@ -43,10 +38,8 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
-import org.apache.ranger.plugin.util.SearchFilter;
 import org.apache.ranger.security.context.RangerContextHolder;
 import org.apache.ranger.security.context.RangerSecurityContext;
-import org.apache.ranger.view.RangerPolicyList;
 import org.junit.Assert;
 import org.junit.FixMethodOrder;
 import org.junit.Rule;
@@ -243,33 +236,4 @@ public class TestRangerPolicyServiceBase {
 		Mockito.verify(daoManager).getXXService();
 	}
 
-	@Test
-	public void test4searchRangerPolicies() {
-		EntityManager entityManager = Mockito.mock(EntityManager.class);
-		SearchFilter searchFilter = new SearchFilter();
-		searchFilter.setParam(SearchFilter.POLICY_NAME, "policyName");
-		searchFilter.setParam(SearchFilter.SERVICE_NAME, "serviceName");
-
-		String searchString = "policyName";
-		String sortString = "asc";
-		List<SearchField> searchFieldList = new ArrayList<SearchField>();
-		boolean isCountQuery = false;
-
-		BaseDao baseDao = Mockito.mock(BaseDao.class);
-
-		Mockito.when(daoManager.getDaoForClassName(Mockito.anyString()))
-				.thenReturn(baseDao);
-		Mockito.when(daoManager.getEntityManager()).thenReturn(entityManager);
-		Mockito.when(
-				searchUtil
-						.createSearchQuery(entityManager, searchString,
-								sortString, searchFilter, searchFieldList,
-								isCountQuery)).thenReturn(null);
-
-		RangerPolicyList dbRangerPolicyList = policyService
-				.searchRangerPolicies(searchFilter);
-		Assert.assertNotNull(dbRangerPolicyList);
-		Mockito.verify(daoManager).getDaoForClassName(Mockito.anyString());
-	}
-
 }