You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by me...@apache.org on 2019/10/30 07:32:15 UTC
[ranger] branch master updated (79b0c12 -> b348291)
This is an automated email from the ASF dual-hosted git repository.
mehul pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git.
from 79b0c12 RANGER-2630: Ensure that entity deletes are handled even when Atlas sets deleted entity's state as not ACTIVE
new c76f8ee RANGER-2594: Improve policy validation performance during delete
new b348291 RANGER-2595: Improve policy delete performance using batch delete
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../model/validation/RangerPolicyValidator.java | 2 +-
.../plugin/model/validation/RangerValidator.java | 9 ++++
.../apache/ranger/plugin/store/ServiceStore.java | 2 +
.../org/apache/ranger/biz/PolicyRefUpdater.java | 49 ++++------------------
.../java/org/apache/ranger/biz/ServiceDBStore.java | 5 +++
.../java/org/apache/ranger/db/XXPolicyDao.java | 7 ++++
.../apache/ranger/db/XXPolicyRefAccessTypeDao.java | 8 ++++
.../apache/ranger/db/XXPolicyRefConditionDao.java | 8 ++++
.../ranger/db/XXPolicyRefDataMaskTypeDao.java | 9 ++++
.../org/apache/ranger/db/XXPolicyRefGroupDao.java | 8 ++++
.../apache/ranger/db/XXPolicyRefResourceDao.java | 9 ++++
.../org/apache/ranger/db/XXPolicyRefRoleDao.java | 8 ++++
.../org/apache/ranger/db/XXPolicyRefUserDao.java | 8 ++++
.../main/resources/META-INF/jpa_named_queries.xml | 32 ++++++++++++++
14 files changed, 121 insertions(+), 43 deletions(-)
[ranger] 02/02: RANGER-2595: Improve policy delete performance
using batch delete
Posted by me...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mehul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit b348291eaeeefc3723bb5507d3770b463544c24a
Author: Andrew <an...@outlook.com>
AuthorDate: Sat Sep 28 23:10:46 2019 -0700
RANGER-2595: Improve policy delete performance using batch delete
Signed-off-by: Mehul Parikh <me...@apache.org>
---
.../org/apache/ranger/biz/PolicyRefUpdater.java | 49 ++++------------------
.../apache/ranger/db/XXPolicyRefAccessTypeDao.java | 8 ++++
.../apache/ranger/db/XXPolicyRefConditionDao.java | 8 ++++
.../ranger/db/XXPolicyRefDataMaskTypeDao.java | 9 ++++
.../org/apache/ranger/db/XXPolicyRefGroupDao.java | 8 ++++
.../apache/ranger/db/XXPolicyRefResourceDao.java | 9 ++++
.../org/apache/ranger/db/XXPolicyRefRoleDao.java | 8 ++++
.../org/apache/ranger/db/XXPolicyRefUserDao.java | 8 ++++
.../main/resources/META-INF/jpa_named_queries.xml | 28 +++++++++++++
9 files changed, 93 insertions(+), 42 deletions(-)
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
index 055cd38..7b2356b 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
@@ -26,13 +26,6 @@ import java.util.Set;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.ranger.db.RangerDaoManager;
-import org.apache.ranger.db.XXPolicyRefAccessTypeDao;
-import org.apache.ranger.db.XXPolicyRefConditionDao;
-import org.apache.ranger.db.XXPolicyRefDataMaskTypeDao;
-import org.apache.ranger.db.XXPolicyRefGroupDao;
-import org.apache.ranger.db.XXPolicyRefResourceDao;
-import org.apache.ranger.db.XXPolicyRefRoleDao;
-import org.apache.ranger.db.XXPolicyRefUserDao;
import org.apache.ranger.entity.XXAccessTypeDef;
import org.apache.ranger.entity.XXDataMaskTypeDef;
import org.apache.ranger.entity.XXGroup;
@@ -252,41 +245,13 @@ public class PolicyRefUpdater {
return false;
}
- XXPolicyRefResourceDao xPolResDao = daoMgr.getXXPolicyRefResource();
- XXPolicyRefRoleDao xPolRoleDao = daoMgr.getXXPolicyRefRole();
- XXPolicyRefGroupDao xPolGroupDao = daoMgr.getXXPolicyRefGroup();
- XXPolicyRefUserDao xPolUserDao = daoMgr.getXXPolicyRefUser();
- XXPolicyRefAccessTypeDao xPolAccessDao = daoMgr.getXXPolicyRefAccessType();
- XXPolicyRefConditionDao xPolCondDao = daoMgr.getXXPolicyRefCondition();
- XXPolicyRefDataMaskTypeDao xPolDataMaskDao = daoMgr.getXXPolicyRefDataMaskType();
-
- for (XXPolicyRefResource resource : xPolResDao.findByPolicyId(policyId)) {
- xPolResDao.remove(resource);
- }
-
- for(XXPolicyRefRole role : xPolRoleDao.findByPolicyId(policyId)) {
- xPolRoleDao.remove(role);
- }
-
- for(XXPolicyRefGroup group : xPolGroupDao.findByPolicyId(policyId)) {
- xPolGroupDao.remove(group);
- }
-
- for(XXPolicyRefUser user : xPolUserDao.findByPolicyId(policyId)) {
- xPolUserDao.remove(user);
- }
-
- for(XXPolicyRefAccessType access : xPolAccessDao.findByPolicyId(policyId)) {
- xPolAccessDao.remove(access);
- }
-
- for(XXPolicyRefCondition condVal : xPolCondDao.findByPolicyId(policyId)) {
- xPolCondDao.remove(condVal);
- }
-
- for(XXPolicyRefDataMaskType dataMask : xPolDataMaskDao.findByPolicyId(policyId)) {
- xPolDataMaskDao.remove(dataMask);
- }
+ daoMgr.getXXPolicyRefResource().deleteByPolicyId(policyId);
+ daoMgr.getXXPolicyRefRole().deleteByPolicyId(policyId);
+ daoMgr.getXXPolicyRefGroup().deleteByPolicyId(policyId);
+ daoMgr.getXXPolicyRefUser().deleteByPolicyId(policyId);
+ daoMgr.getXXPolicyRefAccessType().deleteByPolicyId(policyId);
+ daoMgr.getXXPolicyRefCondition().deleteByPolicyId(policyId);
+ daoMgr.getXXPolicyRefDataMaskType().deleteByPolicyId(policyId);
return true;
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java
index 1ef01bb..b9a60cb 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java
@@ -97,4 +97,12 @@ public class XXPolicyRefAccessTypeDao extends BaseDao<XXPolicyRefAccessType> {
return ret;
}
+ public void deleteByPolicyId(Long policyId) {
+ if(policyId == null) {
+ return;
+ }
+ getEntityManager()
+ .createNamedQuery("XXPolicyRefAccessType.deleteByPolicyId", tClass)
+ .setParameter("policyId", policyId).executeUpdate();
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java
index 2c04ab4..e14bc14 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java
@@ -108,4 +108,12 @@ public class XXPolicyRefConditionDao extends BaseDao<XXPolicyRefCondition> {
return ret;
}
+ public void deleteByPolicyId(Long policyId) {
+ if(policyId == null) {
+ return;
+ }
+ getEntityManager()
+ .createNamedQuery("XXPolicyRefCondition.deleteByPolicyId", tClass)
+ .setParameter("policyId", policyId).executeUpdate();
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java
index 258e3b0..7e7b8d4 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java
@@ -83,4 +83,13 @@ public class XXPolicyRefDataMaskTypeDao extends BaseDao<XXPolicyRefDataMaskType>
}
return ret;
}
+
+ public void deleteByPolicyId(Long policyId) {
+ if(policyId == null) {
+ return;
+ }
+ getEntityManager()
+ .createNamedQuery("XXPolicyRefDataMaskType.deleteByPolicyId", tClass)
+ .setParameter("policyId", policyId).executeUpdate();
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java
index 08829d4..5f9d9ed 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java
@@ -96,4 +96,12 @@ public class XXPolicyRefGroupDao extends BaseDao<XXPolicyRefGroup>{
return ret;
}
+ public void deleteByPolicyId(Long policyId) {
+ if(policyId == null) {
+ return;
+ }
+ getEntityManager()
+ .createNamedQuery("XXPolicyRefGroup.deleteByPolicyId", tClass)
+ .setParameter("policyId", policyId).executeUpdate();
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java
index e259ee8..0ea7de9 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java
@@ -62,6 +62,15 @@ public class XXPolicyRefResourceDao extends BaseDao<XXPolicyRefResource>{
}
}
+ public void deleteByPolicyId(Long policyId) {
+ if(policyId == null) {
+ return;
+ }
+ getEntityManager()
+ .createNamedQuery("XXPolicyRefResource.deleteByPolicyId", tClass)
+ .setParameter("policyId", policyId).executeUpdate();
+ }
+
@SuppressWarnings("unchecked")
public List<RangerPolicyRetriever.PolicyTextNameMap> findUpdatedResourceNamesByPolicy(Long policyId) {
List<RangerPolicyRetriever.PolicyTextNameMap> ret = new ArrayList<>();
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefRoleDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefRoleDao.java
index dbcacb7..3ae7e7a 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefRoleDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefRoleDao.java
@@ -108,5 +108,13 @@ public class XXPolicyRefRoleDao extends BaseDao<XXPolicyRefRole>{
return ret;
}
+ public void deleteByPolicyId(Long policyId) {
+ if(policyId == null) {
+ return;
+ }
+ getEntityManager()
+ .createNamedQuery("XXPolicyRefRole.deleteByPolicyId", tClass)
+ .setParameter("policyId", policyId).executeUpdate();
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java
index f7b6131..518139a 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java
@@ -108,4 +108,12 @@ public class XXPolicyRefUserDao extends BaseDao<XXPolicyRefUser>{
return ret;
}
+ public void deleteByPolicyId(Long policyId) {
+ if(policyId == null) {
+ return;
+ }
+ getEntityManager()
+ .createNamedQuery("XXPolicyRefUser.deleteByPolicyId", tClass)
+ .setParameter("policyId", policyId).executeUpdate();
+ }
}
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index bc8062c..b619b7a 100755
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -592,6 +592,10 @@
<query>select obj from XXPolicyRefAccessType obj where obj.accessDefId = :accessDefId</query>
</named-query>
+ <named-query name="XXPolicyRefAccessType.deleteByPolicyId">
+ <query>DELETE FROM XXPolicyRefAccessType obj WHERE obj.policyId = :policyId</query>
+ </named-query>
+
<!-- XXPolicyRefCondition -->
<named-query name="XXPolicyRefCondition.findByPolicyId">
<query>select obj from XXPolicyRefCondition obj where obj.policyId = :policyId </query>
@@ -601,6 +605,10 @@
<query>select obj from XXPolicyRefCondition obj where obj.conditionName = :conditionName</query>
</named-query>
+ <named-query name="XXPolicyRefCondition.deleteByPolicyId">
+ <query>DELETE FROM XXPolicyRefCondition obj WHERE obj.policyId = :policyId </query>
+ </named-query>
+
<!-- XXPolicyRefGroup -->
<named-query name="XXPolicyRefGroup.findByPolicyId">
<query>select obj from XXPolicyRefGroup obj where obj.policyId = :policyId </query>
@@ -615,6 +623,10 @@
<query>select obj from XXPolicyRefGroup obj where obj.groupId = :groupId and obj.policyId = :policyId </query>
</named-query>
+ <named-query name="XXPolicyRefGroup.deleteByPolicyId">
+ <query>DELETE FROM XXPolicyRefGroup obj WHERE obj.policyId = :policyId</query>
+ </named-query>
+
<named-query name="XXPolicyRefCondition.findByConditionDefIdAndPolicyId">
<query>select obj from XXPolicyRefCondition obj where obj.conditionDefId = :conditionDefId and obj.policyId = :policyId </query>
</named-query>
@@ -766,6 +778,10 @@
<query>select obj from XXPolicyRefDataMaskType obj where obj.dataMaskTypeName = :dataMaskTypeName</query>
</named-query>
+ <named-query name="XXPolicyRefDataMaskType.deleteByPolicyId">
+ <query>DELETE FROM XXPolicyRefDataMaskType obj WHERE obj.policyId = :policyId </query>
+ </named-query>
+
<!-- XXPolicyRefResource -->
<named-query name="XXPolicyRefResource.findByPolicyId">
<query>select obj from XXPolicyRefResource obj where obj.policyId = :policyId </query>
@@ -775,6 +791,10 @@
<query>select obj from XXPolicyRefResource obj where obj.resourceDefId = :resourceDefId</query>
</named-query>
+ <named-query name="XXPolicyRefResource.deleteByPolicyId">
+ <query>DELETE FROM XXPolicyRefResource obj WHERE obj.policyId = :policyId </query>
+ </named-query>
+
<!-- XXPolicyRefUser -->
<named-query name="XXPolicyRefUser.findByPolicyId">
<query>select obj from XXPolicyRefUser obj where obj.policyId = :policyId </query>
@@ -788,6 +808,10 @@
<query>select obj from XXPolicyRefUser obj where obj.userId = :userId</query>
</named-query>
+ <named-query name="XXPolicyRefUser.deleteByPolicyId">
+ <query>DELETE FROM XXPolicyRefUser obj WHERE obj.policyId = :policyId</query>
+ </named-query>
+
<!-- XXPolicyItemCondition -->
<named-query name="XXPolicyItemCondition.findByPolicyId">
@@ -1587,6 +1611,10 @@
<query>select count(obj.policyId) from XXPolicyRefRole obj where obj.roleName = :roleName </query>
</named-query>
+ <named-query name="XXPolicyRefRole.deleteByPolicyId">
+ <query>DELETE FROM XXPolicyRefRole obj WHERE obj.policyId = :policyId </query>
+ </named-query>
+
<!-- XXTagChangeLog -->
<named-query name="XXTagChangeLog.findSinceVersion">
<query>
[ranger] 01/02: RANGER-2594: Improve policy validation performance
during delete
Posted by me...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mehul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit c76f8eecf181f813774d8e7a6546f1666287eefe
Author: Andrew <an...@outlook.com>
AuthorDate: Thu Oct 17 00:24:28 2019 -0700
RANGER-2594: Improve policy validation performance during delete
Signed-off-by: Mehul Parikh <me...@apache.org>
---
.../ranger/plugin/model/validation/RangerPolicyValidator.java | 2 +-
.../apache/ranger/plugin/model/validation/RangerValidator.java | 9 +++++++++
.../main/java/org/apache/ranger/plugin/store/ServiceStore.java | 2 ++
.../src/main/java/org/apache/ranger/biz/ServiceDBStore.java | 5 +++++
.../src/main/java/org/apache/ranger/db/XXPolicyDao.java | 7 +++++++
security-admin/src/main/resources/META-INF/jpa_named_queries.xml | 4 ++++
6 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java
index a854107..fb0afba 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerPolicyValidator.java
@@ -94,7 +94,7 @@ public class RangerPolicyValidator extends RangerValidator {
.becauseOf(error.getMessage("id"))
.build());
valid = false;
- } else if (getPolicy(id) == null) {
+ } else if (policyExists(id)) {
if (LOG.isDebugEnabled()) {
LOG.debug("No policy found for id[" + id + "]! ok!");
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java
index 74653b2..c4ec63b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java
@@ -263,6 +263,15 @@ public abstract class RangerValidator {
return result;
}
+ boolean policyExists(Long id) {
+ try {
+ return _store.policyExists(id);
+ } catch (Exception e) {
+ LOG.debug("Encountred exception while retrieving policy from service store!", e);
+ return false;
+ }
+ }
+
RangerPolicy getPolicy(Long id) {
if(LOG.isDebugEnabled()) {
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
index ba7407f..4af457e 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
@@ -73,6 +73,8 @@ public interface ServiceStore {
void deletePolicy(RangerPolicy policy) throws Exception;
+ boolean policyExists(Long id) throws Exception;
+
RangerPolicy getPolicy(Long id) throws Exception;
List<RangerPolicy> getPolicies(SearchFilter filter) throws Exception;
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index ec44aa1..866eed9 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -2192,6 +2192,11 @@ public class ServiceDBStore extends AbstractServiceStore {
}
@Override
+ public boolean policyExists(Long id) throws Exception {
+ return daoMgr.getXXPolicy().getCountById(id) > 0;
+ }
+
+ @Override
public RangerPolicy getPolicy(Long id) throws Exception {
return policyService.read(id);
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
index b242171..4c501e4 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
@@ -40,6 +40,13 @@ public class XXPolicyDao extends BaseDao<XXPolicy> {
super(daoManager);
}
+ public long getCountById(Long policyId) {
+ return getEntityManager()
+ .createNamedQuery("XXPolicy.countById", Long.class)
+ .setParameter("policyId", policyId)
+ .getSingleResult();
+ }
+
public XXPolicy findByNameAndServiceId(String polName, Long serviceId) {
return findByNameAndServiceIdAndZoneId(polName, serviceId, RangerSecurityZone.RANGER_UNZONED_SECURITY_ZONE_ID);
}
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index 6cc4799..bc8062c 100755
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -266,6 +266,10 @@
<!-- XXPolicy -->
+ <named-query name="XXPolicy.countById">
+ <query>select count(obj.id) from XXPolicy obj where obj.id = :policyId</query>
+ </named-query>
+
<named-query name="XXPolicy.findByPolicyName">
<query>select obj from XXPolicy obj where obj.name = :polName order by obj.id</query>
</named-query>