You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2022/03/29 06:59:20 UTC
[ranger] branch master updated: RANGER-3681: Ranger Database deadlock when createPolicy is running parallel
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new d74f3fc RANGER-3681: Ranger Database deadlock when createPolicy is running parallel
d74f3fc is described below
commit d74f3fced96459ec8bc8f487b2138e4d680d3b49
Author: Xuze Yang <xu...@163.com>
AuthorDate: Sat Mar 26 09:47:26 2022 +0800
RANGER-3681: Ranger Database deadlock when createPolicy is running parallel
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
---
.../apache/ranger/db/XXPolicyRefAccessTypeDao.java | 17 ++++++-
.../apache/ranger/db/XXPolicyRefConditionDao.java | 17 ++++++-
.../ranger/db/XXPolicyRefDataMaskTypeDao.java | 17 ++++++-
.../org/apache/ranger/db/XXPolicyRefGroupDao.java | 17 ++++++-
.../apache/ranger/db/XXPolicyRefResourceDao.java | 18 ++++++-
.../org/apache/ranger/db/XXPolicyRefRoleDao.java | 19 ++++++--
.../org/apache/ranger/db/XXPolicyRefUserDao.java | 17 ++++++-
.../main/resources/META-INF/jpa_named_queries.xml | 56 ++++++++++++++++------
8 files changed, 149 insertions(+), 29 deletions(-)
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java
index b9a60cb..a8233e3 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefAccessTypeDao.java
@@ -25,6 +25,7 @@ import java.util.List;
import javax.persistence.NoResultException;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.ranger.biz.RangerPolicyRetriever;
import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.entity.XXPolicyRefAccessType;
@@ -101,8 +102,20 @@ public class XXPolicyRefAccessTypeDao extends BaseDao<XXPolicyRefAccessType> {
if(policyId == null) {
return;
}
+
+ // First select ids according to policyId, then delete records according to ids
+ // The purpose of dividing the delete sql into these two steps is to avoid deadlocks at rr isolation level
+ List<Long> ids = getEntityManager()
+ .createNamedQuery("XXPolicyRefAccessType.findIdsByPolicyId", Long.class)
+ .setParameter("policyId", policyId)
+ .getResultList();
+
+ if (CollectionUtils.isEmpty(ids)) {
+ return;
+ }
+
getEntityManager()
- .createNamedQuery("XXPolicyRefAccessType.deleteByPolicyId", tClass)
- .setParameter("policyId", policyId).executeUpdate();
+ .createNamedQuery("XXPolicyRefAccessType.deleteByIds", tClass)
+ .setParameter("ids", ids).executeUpdate();
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java
index e14bc14..bc17fcd 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefConditionDao.java
@@ -25,6 +25,7 @@ import java.util.List;
import javax.persistence.NoResultException;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.ranger.biz.RangerPolicyRetriever;
import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.entity.XXPolicyRefCondition;
@@ -112,8 +113,20 @@ public class XXPolicyRefConditionDao extends BaseDao<XXPolicyRefCondition> {
if(policyId == null) {
return;
}
+
+ // First select ids according to policyId, then delete records according to ids
+ // The purpose of dividing the delete sql into these two steps is to avoid deadlocks at rr isolation level
+ List<Long> ids = getEntityManager()
+ .createNamedQuery("XXPolicyRefCondition.findIdsByPolicyId", Long.class)
+ .setParameter("policyId", policyId)
+ .getResultList();
+
+ if (CollectionUtils.isEmpty(ids)) {
+ return;
+ }
+
getEntityManager()
- .createNamedQuery("XXPolicyRefCondition.deleteByPolicyId", tClass)
- .setParameter("policyId", policyId).executeUpdate();
+ .createNamedQuery("XXPolicyRefCondition.deleteByIds", tClass)
+ .setParameter("ids", ids).executeUpdate();
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java
index 7e7b8d4..df5f7cd 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefDataMaskTypeDao.java
@@ -25,6 +25,7 @@ import java.util.List;
import javax.persistence.NoResultException;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.ranger.biz.RangerPolicyRetriever;
import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.entity.XXPolicyRefDataMaskType;
@@ -88,8 +89,20 @@ public class XXPolicyRefDataMaskTypeDao extends BaseDao<XXPolicyRefDataMaskType>
if(policyId == null) {
return;
}
+
+ // First select ids according to policyId, then delete records according to ids
+ // The purpose of dividing the delete sql into these two steps is to avoid deadlocks at rr isolation level
+ List<Long> ids = getEntityManager()
+ .createNamedQuery("XXPolicyRefDataMaskType.findIdsByPolicyId", Long.class)
+ .setParameter("policyId", policyId)
+ .getResultList();
+
+ if (CollectionUtils.isEmpty(ids)) {
+ return;
+ }
+
getEntityManager()
- .createNamedQuery("XXPolicyRefDataMaskType.deleteByPolicyId", tClass)
- .setParameter("policyId", policyId).executeUpdate();
+ .createNamedQuery("XXPolicyRefDataMaskType.deleteByIds", tClass)
+ .setParameter("ids", ids).executeUpdate();
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java
index 5f9d9ed..dc41aea 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefGroupDao.java
@@ -25,6 +25,7 @@ import java.util.List;
import javax.persistence.NoResultException;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.ranger.biz.RangerPolicyRetriever;
import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.entity.XXPolicyRefGroup;
@@ -100,8 +101,20 @@ public class XXPolicyRefGroupDao extends BaseDao<XXPolicyRefGroup>{
if(policyId == null) {
return;
}
+
+ // First select ids according to policyId, then delete records according to ids
+ // The purpose of dividing the delete sql into these two steps is to avoid deadlocks at rr isolation level
+ List<Long> ids = getEntityManager()
+ .createNamedQuery("XXPolicyRefGroup.findIdsByPolicyId", Long.class)
+ .setParameter("policyId", policyId)
+ .getResultList();
+
+ if (CollectionUtils.isEmpty(ids)) {
+ return;
+ }
+
getEntityManager()
- .createNamedQuery("XXPolicyRefGroup.deleteByPolicyId", tClass)
- .setParameter("policyId", policyId).executeUpdate();
+ .createNamedQuery("XXPolicyRefGroup.deleteByIds", tClass)
+ .setParameter("ids", ids).executeUpdate();
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java
index 0ea7de9..738c6ff 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefResourceDao.java
@@ -25,6 +25,7 @@ import java.util.List;
import javax.persistence.NoResultException;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.ranger.biz.RangerPolicyRetriever;
import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.entity.XXPolicyRefResource;
@@ -66,9 +67,22 @@ public class XXPolicyRefResourceDao extends BaseDao<XXPolicyRefResource>{
if(policyId == null) {
return;
}
+
+ // First select ids according to policyId, then delete records according to ids
+ // The purpose of dividing the delete sql into these two steps is to avoid deadlocks at rr isolation level
+ List<Long> ids = getEntityManager()
+ .createNamedQuery("XXPolicyRefResource.findIdsByPolicyId", Long.class)
+ .setParameter("policyId", policyId)
+ .getResultList();
+
+ if (CollectionUtils.isEmpty(ids)) {
+ return;
+ }
+
getEntityManager()
- .createNamedQuery("XXPolicyRefResource.deleteByPolicyId", tClass)
- .setParameter("policyId", policyId).executeUpdate();
+ .createNamedQuery("XXPolicyRefResource.deleteByIds", tClass)
+ .setParameter("ids", ids).executeUpdate();
+
}
@SuppressWarnings("unchecked")
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefRoleDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefRoleDao.java
index 3ae7e7a..35433c7 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefRoleDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefRoleDao.java
@@ -25,6 +25,7 @@ import java.util.List;
import javax.persistence.NoResultException;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.ranger.biz.RangerPolicyRetriever;
import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.entity.XXPolicyRefRole;
@@ -112,9 +113,21 @@ public class XXPolicyRefRoleDao extends BaseDao<XXPolicyRefRole>{
if(policyId == null) {
return;
}
- getEntityManager()
- .createNamedQuery("XXPolicyRefRole.deleteByPolicyId", tClass)
- .setParameter("policyId", policyId).executeUpdate();
+
+ // First select ids according to policyId, then delete records according to ids
+ // The purpose of dividing the delete sql into these two steps is to avoid deadlocks at rr isolation level
+ List<Long> ids = getEntityManager()
+ .createNamedQuery("XXPolicyRefRole.findIdsByPolicyId", Long.class)
+ .setParameter("policyId", policyId)
+ .getResultList();
+
+ if (CollectionUtils.isEmpty(ids)) {
+ return;
+ }
+
+ getEntityManager()
+ .createNamedQuery("XXPolicyRefRole.deleteByIds", tClass)
+ .setParameter("ids", ids).executeUpdate();
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java
index 518139a..eced7b2 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyRefUserDao.java
@@ -25,6 +25,7 @@ import java.util.List;
import javax.persistence.NoResultException;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.ranger.biz.RangerPolicyRetriever;
import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.entity.XXPolicyRefUser;
@@ -112,8 +113,20 @@ public class XXPolicyRefUserDao extends BaseDao<XXPolicyRefUser>{
if(policyId == null) {
return;
}
+
+ // First select ids according to policyId, then delete records according to ids
+ // The purpose of dividing the delete sql into these two steps is to avoid deadlocks at rr isolation level
+ List<Long> ids = getEntityManager()
+ .createNamedQuery("XXPolicyRefUser.findIdsByPolicyId", Long.class)
+ .setParameter("policyId", policyId)
+ .getResultList();
+
+ if (CollectionUtils.isEmpty(ids)) {
+ return;
+ }
+
getEntityManager()
- .createNamedQuery("XXPolicyRefUser.deleteByPolicyId", tClass)
- .setParameter("policyId", policyId).executeUpdate();
+ .createNamedQuery("XXPolicyRefUser.deleteByIds", tClass)
+ .setParameter("ids", ids).executeUpdate();
}
}
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index 2f50d71..b2a748f 100755
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -722,12 +722,16 @@
<query>select obj from XXPolicyRefAccessType obj where obj.policyId = :policyId </query>
</named-query>
+ <named-query name="XXPolicyRefAccessType.findIdsByPolicyId">
+ <query>select obj.id from XXPolicyRefAccessType obj where obj.policyId = :policyId </query>
+ </named-query>
+
<named-query name="XXPolicyRefAccessType.findByAccessTypeDefId">
<query>select obj from XXPolicyRefAccessType obj where obj.accessDefId = :accessDefId</query>
</named-query>
- <named-query name="XXPolicyRefAccessType.deleteByPolicyId">
- <query>DELETE FROM XXPolicyRefAccessType obj WHERE obj.policyId = :policyId</query>
+ <named-query name="XXPolicyRefAccessType.deleteByIds">
+ <query>DELETE FROM XXPolicyRefAccessType obj WHERE obj.id IN :ids</query>
</named-query>
<!-- XXPolicyRefCondition -->
@@ -735,12 +739,16 @@
<query>select obj from XXPolicyRefCondition obj where obj.policyId = :policyId </query>
</named-query>
+ <named-query name="XXPolicyRefCondition.findIdsByPolicyId">
+ <query>select obj.id from XXPolicyRefCondition obj where obj.policyId = :policyId </query>
+ </named-query>
+
<named-query name="XXPolicyRefCondition.findByConditionName">
<query>select obj from XXPolicyRefCondition obj where obj.conditionName = :conditionName</query>
</named-query>
- <named-query name="XXPolicyRefCondition.deleteByPolicyId">
- <query>DELETE FROM XXPolicyRefCondition obj WHERE obj.policyId = :policyId </query>
+ <named-query name="XXPolicyRefCondition.deleteByIds">
+ <query>DELETE FROM XXPolicyRefCondition obj WHERE obj.id IN :ids </query>
</named-query>
<!-- XXPolicyRefGroup -->
@@ -748,6 +756,10 @@
<query>select obj from XXPolicyRefGroup obj where obj.policyId = :policyId </query>
</named-query>
+ <named-query name="XXPolicyRefGroup.findIdsByPolicyId">
+ <query>select obj.id from XXPolicyRefGroup obj where obj.policyId = :policyId </query>
+ </named-query>
+
<named-query name="XXPolicyRefGroup.findByGroupName">
<query>select obj from XXPolicyRefGroup obj where obj.groupName = :groupName</query>
</named-query>
@@ -757,8 +769,8 @@
<query>select obj from XXPolicyRefGroup obj where obj.groupId = :groupId and obj.policyId = :policyId </query>
</named-query>
- <named-query name="XXPolicyRefGroup.deleteByPolicyId">
- <query>DELETE FROM XXPolicyRefGroup obj WHERE obj.policyId = :policyId</query>
+ <named-query name="XXPolicyRefGroup.deleteByIds">
+ <query>DELETE FROM XXPolicyRefGroup obj WHERE obj.id IN :ids</query>
</named-query>
<named-query name="XXPolicyRefCondition.findByConditionDefIdAndPolicyId">
@@ -908,12 +920,16 @@
<query>select obj from XXPolicyRefDataMaskType obj where obj.policyId = :policyId </query>
</named-query>
+ <named-query name="XXPolicyRefDataMaskType.findIdsByPolicyId">
+ <query>select obj.id from XXPolicyRefDataMaskType obj where obj.policyId = :policyId </query>
+ </named-query>
+
<named-query name="XXPolicyRefDataMaskType.findByDataMaskTypeName">
<query>select obj from XXPolicyRefDataMaskType obj where obj.dataMaskTypeName = :dataMaskTypeName</query>
</named-query>
- <named-query name="XXPolicyRefDataMaskType.deleteByPolicyId">
- <query>DELETE FROM XXPolicyRefDataMaskType obj WHERE obj.policyId = :policyId </query>
+ <named-query name="XXPolicyRefDataMaskType.deleteByIds">
+ <query>DELETE FROM XXPolicyRefDataMaskType obj WHERE obj.id IN :ids </query>
</named-query>
<!-- XXPolicyRefResource -->
@@ -921,12 +937,16 @@
<query>select obj from XXPolicyRefResource obj where obj.policyId = :policyId </query>
</named-query>
+ <named-query name="XXPolicyRefResource.findIdsByPolicyId">
+ <query>select obj.id from XXPolicyRefResource obj where obj.policyId = :policyId </query>
+ </named-query>
+
<named-query name="XXPolicyRefResource.findByResourceDefId">
<query>select obj from XXPolicyRefResource obj where obj.resourceDefId = :resourceDefId</query>
</named-query>
- <named-query name="XXPolicyRefResource.deleteByPolicyId">
- <query>DELETE FROM XXPolicyRefResource obj WHERE obj.policyId = :policyId </query>
+ <named-query name="XXPolicyRefResource.deleteByIds">
+ <query>DELETE FROM XXPolicyRefResource obj WHERE obj.id IN :ids </query>
</named-query>
<!-- XXPolicyRefUser -->
@@ -934,6 +954,10 @@
<query>select obj from XXPolicyRefUser obj where obj.policyId = :policyId </query>
</named-query>
+ <named-query name="XXPolicyRefUser.findIdsByPolicyId">
+ <query>select obj.id from XXPolicyRefUser obj where obj.policyId = :policyId </query>
+ </named-query>
+
<named-query name="XXPolicyRefUser.findByUserName">
<query>select obj from XXPolicyRefUser obj where obj.userName = :userName</query>
</named-query>
@@ -942,8 +966,8 @@
<query>select obj from XXPolicyRefUser obj where obj.userId = :userId</query>
</named-query>
- <named-query name="XXPolicyRefUser.deleteByPolicyId">
- <query>DELETE FROM XXPolicyRefUser obj WHERE obj.policyId = :policyId</query>
+ <named-query name="XXPolicyRefUser.deleteByIds">
+ <query>DELETE FROM XXPolicyRefUser obj WHERE obj.id IN :ids</query>
</named-query>
<!-- XXPolicyItemCondition -->
@@ -1758,6 +1782,10 @@
<query>select obj from XXPolicyRefRole obj where obj.policyId = :policyId </query>
</named-query>
+ <named-query name="XXPolicyRefRole.findIdsByPolicyId">
+ <query>select obj.id from XXPolicyRefRole obj where obj.policyId = :policyId </query>
+ </named-query>
+
<named-query name="XXPolicyRefRole.findByRoleName">
<query>select obj from XXPolicyRefRole obj where obj.roleName = :roleName</query>
</named-query>
@@ -1785,8 +1813,8 @@
<query>select count(obj.policyId) from XXPolicyRefRole obj where obj.roleName = :roleName </query>
</named-query>
- <named-query name="XXPolicyRefRole.deleteByPolicyId">
- <query>DELETE FROM XXPolicyRefRole obj WHERE obj.policyId = :policyId </query>
+ <named-query name="XXPolicyRefRole.deleteByIds">
+ <query>DELETE FROM XXPolicyRefRole obj WHERE obj.id IN :ids </query>
</named-query>
<!-- XXTagChangeLog -->