You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by pr...@apache.org on 2019/08/09 10:44:34 UTC
[ranger] branch master updated: RANGER-2499: Improve bulk
create/update policy performance
This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 1893a36 RANGER-2499: Improve bulk create/update policy performance
1893a36 is described below
commit 1893a36d64a78e3d18cf74e74bf6a88649d15515
Author: Pradeep <pr...@apache.org>
AuthorDate: Fri Aug 9 13:07:46 2019 +0530
RANGER-2499: Improve bulk create/update policy performance
---
.../apache/ranger/plugin/store/ServiceStore.java | 2 +
.../org/apache/ranger/biz/PolicyRefUpdater.java | 18 +--
.../java/org/apache/ranger/biz/RangerBizUtil.java | 58 +++------
.../apache/ranger/biz/RangerPolicyRetriever.java | 29 +++--
.../java/org/apache/ranger/biz/ServiceDBStore.java | 137 ++++++++++-----------
.../java/org/apache/ranger/common/ContextUtil.java | 10 ++
.../java/org/apache/ranger/common/db/BaseDao.java | 21 +++-
.../patch/PatchForUpdatingPolicyJson_J10019.java | 2 +-
.../java/org/apache/ranger/rest/ServiceREST.java | 65 ++++++----
...ontextHolder.java => RangerAdminOpContext.java} | 43 +++----
.../security/context/RangerContextHolder.java | 15 ++-
.../RangerSecurityContextFormationFilter.java | 1 +
.../ranger/service/RangerBaseModelService.java | 9 +-
.../ranger/service/RangerDataHistService.java | 6 +-
.../main/resources/META-INF/jpa_named_queries.xml | 6 +-
.../conf.dist/ranger-admin-default-site.xml | 12 ++
.../org/apache/ranger/biz/TestServiceDBStore.java | 6 +-
.../ranger/service/TestRangerDataHistService.java | 2 +-
18 files changed, 237 insertions(+), 205 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
index d487976..2af5845 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
@@ -69,6 +69,8 @@ public interface ServiceStore {
RangerPolicy updatePolicy(RangerPolicy policy) throws Exception;
+ void deletePolicy(RangerPolicy policy, RangerService service) throws Exception;
+
void deletePolicy(Long id) throws Exception;
RangerPolicy getPolicy(Long id) throws Exception;
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
index e242d90..055cd38 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
@@ -73,7 +73,7 @@ public class PolicyRefUpdater {
return;
}
- cleanupRefTables(policy, true);
+ cleanupRefTables(policy);
final Set<String> resourceNames = policy.getResources().keySet();
final Set<String> roleNames = new HashSet<>();
@@ -245,7 +245,7 @@ public class PolicyRefUpdater {
}
}
- public Boolean cleanupRefTables(RangerPolicy policy, boolean flush) {
+ public Boolean cleanupRefTables(RangerPolicy policy) {
final Long policyId = policy == null ? null : policy.getId();
if (policyId == null) {
@@ -261,31 +261,31 @@ public class PolicyRefUpdater {
XXPolicyRefDataMaskTypeDao xPolDataMaskDao = daoMgr.getXXPolicyRefDataMaskType();
for (XXPolicyRefResource resource : xPolResDao.findByPolicyId(policyId)) {
- xPolResDao.remove(resource, flush);
+ xPolResDao.remove(resource);
}
for(XXPolicyRefRole role : xPolRoleDao.findByPolicyId(policyId)) {
- xPolRoleDao.remove(role, flush);
+ xPolRoleDao.remove(role);
}
for(XXPolicyRefGroup group : xPolGroupDao.findByPolicyId(policyId)) {
- xPolGroupDao.remove(group, flush);
+ xPolGroupDao.remove(group);
}
for(XXPolicyRefUser user : xPolUserDao.findByPolicyId(policyId)) {
- xPolUserDao.remove(user, flush);
+ xPolUserDao.remove(user);
}
for(XXPolicyRefAccessType access : xPolAccessDao.findByPolicyId(policyId)) {
- xPolAccessDao.remove(access, flush);
+ xPolAccessDao.remove(access);
}
for(XXPolicyRefCondition condVal : xPolCondDao.findByPolicyId(policyId)) {
- xPolCondDao.remove(condVal, flush);
+ xPolCondDao.remove(condVal);
}
for(XXPolicyRefDataMaskType dataMask : xPolDataMaskDao.findByPolicyId(policyId)) {
- xPolDataMaskDao.remove(dataMask, flush);
+ xPolDataMaskDao.remove(dataMask);
}
return true;
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
index d49ea98..58cf790 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
@@ -46,6 +46,7 @@ import org.apache.ranger.common.RangerConstants;
import org.apache.ranger.common.StringUtil;
import org.apache.ranger.common.UserSessionBase;
import org.apache.ranger.db.RangerDaoManager;
+import org.apache.ranger.db.XXDBBaseDao;
import org.apache.ranger.entity.XXAsset;
import org.apache.ranger.entity.XXDBBase;
import org.apache.ranger.entity.XXGroup;
@@ -100,6 +101,8 @@ public class RangerBizUtil {
private static int PATH_CHAR_SET_LEN = PATH_CHAR_SET.length;
public static final String AUDIT_STORE_RDBMS = "DB";
public static final String AUDIT_STORE_SOLR = "solr";
+ public static final boolean batchClearEnabled = PropertiesUtil.getBooleanProperty("ranger.jpa.jdbc.batch-clear.enable", true);
+ public static final int batchSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-clear.size", 10);
String auditDBType = AUDIT_STORE_RDBMS;
@@ -1148,46 +1151,6 @@ public class RangerBizUtil {
}
}
- public void createTrxLog(List<XXTrxLog> trxLogList, boolean flush) {
- if (trxLogList == null) {
- return;
- }
-
- UserSessionBase usb = ContextUtil.getCurrentUserSession();
- Long authSessionId = null;
- if (usb != null) {
- authSessionId = ContextUtil.getCurrentUserSession().getSessionId();
- }
- if(guidUtil != null){
- Long trxId = guidUtil.genLong();
- for (XXTrxLog xTrxLog : trxLogList) {
- if (xTrxLog != null) {
- if ("Password".equalsIgnoreCase(StringUtil.trim(xTrxLog.getAttributeName()))) {
- if (xTrxLog.getPreviousValue() != null
- && !xTrxLog.getPreviousValue().trim().isEmpty()
- && !"null".equalsIgnoreCase(xTrxLog
- .getPreviousValue().trim())) {
- xTrxLog.setPreviousValue(AppConstants.Masked_String);
- }
- if (xTrxLog.getNewValue() != null
- && !xTrxLog.getNewValue().trim().isEmpty()
- && !"null".equalsIgnoreCase(xTrxLog.getNewValue()
- .trim())) {
- xTrxLog.setNewValue(AppConstants.Masked_String);
- }
- }
- xTrxLog.setTransactionId(trxId.toString());
- if (authSessionId != null) {
- xTrxLog.setSessionId("" + authSessionId);
- }
- xTrxLog.setSessionType("Spring Authenticated Session");
- xTrxLog.setRequestId(trxId.toString());
- daoManager.getXXTrxLog().create(xTrxLog, flush);
- }
- }
- }
- }
-
public static int getDBFlavor() {
String[] propertyNames = { "xa.db.flavor",
"ranger.jpa.jdbc.dialect",
@@ -1499,4 +1462,19 @@ public class RangerBizUtil {
}
}
}
+
+ public static boolean isBulkMode() {
+ return ContextUtil.isBulkModeContext();
+ }
+
+ //should be used only in bulk operation like importPolicies, policies delete.
+ public void bulkModeOnlyFlushAndClear() {
+ if (batchClearEnabled) {
+ XXDBBaseDao xXDBBaseDao = daoManager.getXXDBBase();
+ if (xXDBBaseDao != null) {
+ xXDBBaseDao.flush();
+ xXDBBaseDao.clear();
+ }
+ }
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index 4815f5b..c3c25ce 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -508,7 +508,8 @@ public class RangerPolicyRetriever {
this.service = xService;
this.iterPolicy = asList(xPolicy).listIterator();
- this.iterPolicyLabels = daoMgr.getXXPolicyLabelMap().findByPolicyId(policyId).listIterator();
+ List<XXPolicyLabelMap> policyLabels = daoMgr.getXXPolicyLabelMap().findByPolicyId(policyId);
+ this.iterPolicyLabels = policyLabels != null ? policyLabels.listIterator() : null;
}
RangerPolicy getNextPolicy() {
@@ -545,19 +546,21 @@ public class RangerPolicyRetriever {
private void getPolicyLabels(RangerPolicy ret) {
List<String> xPolicyLabels = new ArrayList<String>();
- while (iterPolicyLabels.hasNext()) {
- XXPolicyLabelMap xPolicyLabel = iterPolicyLabels.next();
- if (xPolicyLabel.getPolicyId().equals(ret.getId())) {
- String policyLabel = lookupCache.getPolicyLabelName(xPolicyLabel.getPolicyLabelId());
- if (policyLabel != null) {
- xPolicyLabels.add(policyLabel);
- }
- ret.setPolicyLabels(xPolicyLabels);
- } else {
- if (iterPolicyLabels.hasPrevious()) {
- iterPolicyLabels.previous();
+ if (iterPolicyLabels != null) {
+ while (iterPolicyLabels.hasNext()) {
+ XXPolicyLabelMap xPolicyLabel = iterPolicyLabels.next();
+ if (xPolicyLabel.getPolicyId().equals(ret.getId())) {
+ String policyLabel = lookupCache.getPolicyLabelName(xPolicyLabel.getPolicyLabelId());
+ if (policyLabel != null) {
+ xPolicyLabels.add(policyLabel);
+ }
+ ret.setPolicyLabels(xPolicyLabels);
+ } else {
+ if (iterPolicyLabels.hasPrevious()) {
+ iterPolicyLabels.previous();
+ }
+ break;
}
- break;
}
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index ef22354..113e727 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -94,7 +94,6 @@ import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.db.XXAccessTypeDefDao;
import org.apache.ranger.db.XXAccessTypeDefGrantsDao;
import org.apache.ranger.db.XXContextEnricherDefDao;
-import org.apache.ranger.db.XXDBBaseDao;
import org.apache.ranger.db.XXDataMaskTypeDefDao;
import org.apache.ranger.db.XXEnumDefDao;
import org.apache.ranger.db.XXEnumElementDefDao;
@@ -617,7 +616,7 @@ public class ServiceDBStore extends AbstractServiceStore {
}
RangerServiceDef createdServiceDef = serviceDefService.getPopulatedViewObject(createdSvcDef);
- dataHistService.createObjectDataHistory(createdServiceDef, RangerDataHistService.ACTION_CREATE, true);
+ dataHistService.createObjectDataHistory(createdServiceDef, RangerDataHistService.ACTION_CREATE);
postCreate(createdServiceDef);
@@ -678,7 +677,7 @@ public class ServiceDBStore extends AbstractServiceStore {
updateChildObjectsOfServiceDef(createdSvcDef, configs, resources, accessTypes, policyConditions, contextEnrichers, enums, dataMaskDef, rowFilterDef);
RangerServiceDef updatedSvcDef = getServiceDef(serviceDefId);
- dataHistService.createObjectDataHistory(updatedSvcDef, RangerDataHistService.ACTION_UPDATE, true);
+ dataHistService.createObjectDataHistory(updatedSvcDef, RangerDataHistService.ACTION_UPDATE);
postUpdate(updatedSvcDef);
@@ -1186,7 +1185,7 @@ public class ServiceDBStore extends AbstractServiceStore {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ServiceDBStore.deleteServiceDef(" + serviceDefId + ", " + forceDelete + ")");
}
- bizUtil.blockAuditorRoleUser();
+ bizUtil.blockAuditorRoleUser();
UserSessionBase session = ContextUtil.getCurrentUserSession();
if (session == null) {
throw restErrorUtil.createRESTException(
@@ -1205,7 +1204,7 @@ public class ServiceDBStore extends AbstractServiceStore {
throw restErrorUtil.createRESTException("No Service Definiton found for Id: " + serviceDefId,
MessageEnums.DATA_NOT_FOUND);
}
-
+
List<XXService> serviceList = daoMgr.getXXService().findByServiceDefId(serviceDefId);
if (!forceDelete) {
if(CollectionUtils.isNotEmpty(serviceList)) {
@@ -1225,13 +1224,13 @@ public class ServiceDBStore extends AbstractServiceStore {
for(XXAccessTypeDef accessType : accTypeDefs) {
deleteXXAccessTypeDef(accessType);
}
-
+
XXContextEnricherDefDao xContextEnricherDao = daoMgr.getXXContextEnricherDef();
List<XXContextEnricherDef> contextEnrichers = xContextEnricherDao.findByServiceDefId(serviceDefId);
for(XXContextEnricherDef context : contextEnrichers) {
xContextEnricherDao.remove(context);
}
-
+
XXEnumDefDao enumDefDao = daoMgr.getXXEnumDef();
List<XXEnumDef> enumDefList = enumDefDao.findByServiceDefId(serviceDefId);
for (XXEnumDef enumDef : enumDefList) {
@@ -1241,10 +1240,10 @@ public class ServiceDBStore extends AbstractServiceStore {
}
enumDefDao.remove(enumDef);
}
-
+
XXPolicyConditionDefDao policyCondDao = daoMgr.getXXPolicyConditionDef();
List<XXPolicyConditionDef> policyCondList = policyCondDao.findByServiceDefId(serviceDefId);
-
+
for (XXPolicyConditionDef policyCond : policyCondList) {
List<XXPolicyRefCondition> xxPolicyRefConditions = daoMgr.getXXPolicyRefCondition().findByConditionDefId(policyCond.getId());
for (XXPolicyRefCondition XXPolicyRefCondition : xxPolicyRefConditions) {
@@ -1252,24 +1251,24 @@ public class ServiceDBStore extends AbstractServiceStore {
}
policyCondDao.remove(policyCond);
}
-
+
List<XXResourceDef> resDefList = daoMgr.getXXResourceDef().findByServiceDefId(serviceDefId);
for(XXResourceDef resDef : resDefList) {
deleteXXResourceDef(resDef);
}
-
+
XXServiceConfigDefDao configDefDao = daoMgr.getXXServiceConfigDef();
List<XXServiceConfigDef> configDefList = configDefDao.findByServiceDefId(serviceDefId);
for(XXServiceConfigDef configDef : configDefList) {
configDefDao.remove(configDef);
}
-
+
if(CollectionUtils.isNotEmpty(serviceList)) {
for(XXService service : serviceList) {
deleteService(service.getId());
}
}
-
+
Long version = serviceDef.getVersion();
if(version == null) {
version = Long.valueOf(1);
@@ -1278,11 +1277,11 @@ public class ServiceDBStore extends AbstractServiceStore {
version = Long.valueOf(version.longValue() + 1);
}
serviceDef.setVersion(version);
-
+
serviceDefService.delete(serviceDef);
LOG.info("ServiceDefinition has been deleted successfully. Service-Def Name: " + serviceDef.getName());
-
- dataHistService.createObjectDataHistory(serviceDef, RangerDataHistService.ACTION_DELETE, true);
+
+ dataHistService.createObjectDataHistory(serviceDef, RangerDataHistService.ACTION_DELETE);
postDelete(serviceDef);
@@ -1486,7 +1485,7 @@ public class ServiceDBStore extends AbstractServiceStore {
throw restErrorUtil.createRESTException("Could not create service - Internal error ", MessageEnums.ERROR_CREATING_OBJECT);
}
- dataHistService.createObjectDataHistory(createdService, RangerDataHistService.ACTION_CREATE, true);
+ dataHistService.createObjectDataHistory(createdService, RangerDataHistService.ACTION_CREATE);
List<XXTrxLog> trxLogList = svcService.getTransactionLog(createdService,
RangerServiceService.OPERATION_CREATE_CONTEXT);
@@ -1679,7 +1678,7 @@ public class ServiceDBStore extends AbstractServiceStore {
LOG.debug("vXUser:[" + vXUser + "]");
}
RangerService updService = svcService.getPopulatedViewObject(xUpdService);
- dataHistService.createObjectDataHistory(updService, RangerDataHistService.ACTION_UPDATE, true);
+ dataHistService.createObjectDataHistory(updService, RangerDataHistService.ACTION_UPDATE);
bizUtil.createTrxLog(trxLogList);
return updService;
@@ -1699,13 +1698,17 @@ public class ServiceDBStore extends AbstractServiceStore {
restrictIfZoneService(service);
List<Long> policyIds = daoMgr.getXXPolicy().findPolicyIdsByServiceId(service.getId());
if (CollectionUtils.isNotEmpty(policyIds)) {
- List<RangerPolicy> rangerPolicies = new ArrayList<RangerPolicy>();
- for(Long policyID : policyIds) {
- RangerPolicy rangerPolicy=getPolicy(policyID);
+ long totalDeletedPolicies = 0;
+ for (Long policyID : policyIds) {
+ RangerPolicy rangerPolicy = getPolicy(policyID);
deletePolicy(rangerPolicy, service);
- rangerPolicies.add(rangerPolicy);
+ totalDeletedPolicies = totalDeletedPolicies + 1;
+ // its a bulk policy delete call flush and clear
+ if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) {
+ bizUtil.bulkModeOnlyFlushAndClear();
+ }
}
- createTrxLogsAndHistoryAfterDelete(rangerPolicies, service);
+ bizUtil.bulkModeOnlyFlushAndClear();
}
XXServiceConfigMapDao configDao = daoMgr.getXXServiceConfigMap();
@@ -1725,7 +1728,7 @@ public class ServiceDBStore extends AbstractServiceStore {
svcService.delete(service);
- dataHistService.createObjectDataHistory(service, RangerDataHistService.ACTION_DELETE, true);
+ dataHistService.createObjectDataHistory(service, RangerDataHistService.ACTION_DELETE);
List<XXTrxLog> trxLogList = svcService.getTransactionLog(service, RangerServiceService.OPERATION_DELETE_CONTEXT);
bizUtil.createTrxLog(trxLogList);
@@ -1897,13 +1900,13 @@ public class ServiceDBStore extends AbstractServiceStore {
assignedIdPolicyService.setPopulateExistingBaseFields(true);
daoMgr.getXXPolicy().setIdentityInsert(true);
- policy = assignedIdPolicyService.create(policy);
+ policy = assignedIdPolicyService.create(policy, true);
daoMgr.getXXPolicy().setIdentityInsert(false);
daoMgr.getXXPolicy().updateSequence();
assignedIdPolicyService.setPopulateExistingBaseFields(false);
} else {
- policy = policyService.create(policy);
+ policy = policyService.create(policy, true);
}
XXPolicy xCreatedPolicy = daoMgr.getXXPolicy().getById(policy.getId());
@@ -1912,7 +1915,7 @@ public class ServiceDBStore extends AbstractServiceStore {
RangerPolicy createdPolicy = policyService.getPopulatedViewObject(xCreatedPolicy);
handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE, createdPolicy);
- dataHistService.createObjectDataHistory(createdPolicy, RangerDataHistService.ACTION_CREATE, true);
+ dataHistService.createObjectDataHistory(createdPolicy, RangerDataHistService.ACTION_CREATE);
List<XXTrxLog> trxLogList = getTransactionLogList(createdPolicy,
RangerPolicyService.OPERATION_IMPORT_CREATE_CONTEXT, RangerPolicyService.OPERATION_CREATE_CONTEXT);
@@ -2033,14 +2036,14 @@ public class ServiceDBStore extends AbstractServiceStore {
policy = policyService.update(policy);
XXPolicy newUpdPolicy = daoMgr.getXXPolicy().getById(policy.getId());
- policyRefUpdater.cleanupRefTables(policy, true);
- deleteExistingPolicyLabel(policy, true);
+ policyRefUpdater.cleanupRefTables(policy);
+ deleteExistingPolicyLabel(policy);
policyRefUpdater.createNewPolMappingForRefTable(policy, newUpdPolicy, xServiceDef);
createOrMapLabels(newUpdPolicy, uniquePolicyLabels);
RangerPolicy updPolicy = policyService.getPopulatedViewObject(newUpdPolicy);
handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE, updPolicy);
- dataHistService.createObjectDataHistory(updPolicy, RangerDataHistService.ACTION_UPDATE, true);
+ dataHistService.createObjectDataHistory(updPolicy, RangerDataHistService.ACTION_UPDATE);
bizUtil.createTrxLog(trxLogList);
@@ -2061,11 +2064,11 @@ public class ServiceDBStore extends AbstractServiceStore {
String policyName = policy.getName();
RangerService service = getServiceByName(policy.getService());
-
+
if(service == null) {
throw new Exception("service does not exist - name='" + policy.getService());
}
-
+
Long version = policy.getVersion();
if(version == null) {
version = Long.valueOf(1);
@@ -2073,25 +2076,26 @@ public class ServiceDBStore extends AbstractServiceStore {
} else {
version = Long.valueOf(version.longValue() + 1);
}
-
+
policy.setVersion(version);
-
+
List<XXTrxLog> trxLogList = getTransactionLogList(policy, RangerPolicyService.OPERATION_IMPORT_DELETE_CONTEXT,
RangerPolicyService.OPERATION_DELETE_CONTEXT);
- policyRefUpdater.cleanupRefTables(policy, true);
- deleteExistingPolicyLabel(policy, true);
+ policyRefUpdater.cleanupRefTables(policy);
+ deleteExistingPolicyLabel(policy);
policyService.delete(policy);
handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE, policy);
- dataHistService.createObjectDataHistory(policy, RangerDataHistService.ACTION_DELETE, true);
-
+ dataHistService.createObjectDataHistory(policy, RangerDataHistService.ACTION_DELETE);
+
bizUtil.createTrxLog(trxLogList);
-
+
LOG.info("Policy Deleted Successfully. PolicyName : " + policyName);
}
+ @Override
public void deletePolicy(RangerPolicy policy, RangerService service) throws Exception {
if(LOG.isDebugEnabled()) {
LOG.debug("==> ServiceDBStore.deletePolicy()");
@@ -2113,9 +2117,13 @@ public class ServiceDBStore extends AbstractServiceStore {
version = Long.valueOf(version.longValue() + 1);
}
policy.setVersion(version);
- policyRefUpdater.cleanupRefTables(policy, false);
- deleteExistingPolicyLabel(policy, false);
- policyService.delete(policy, false);
+ policyRefUpdater.cleanupRefTables(policy);
+ deleteExistingPolicyLabel(policy);
+ policyService.delete(policy);
+ List<XXTrxLog> trxLogList = getTransactionLogList(policy, RangerPolicyService.OPERATION_IMPORT_DELETE_CONTEXT, RangerPolicyService.OPERATION_DELETE_CONTEXT);
+ handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE, policy);
+ dataHistService.createObjectDataHistory(policy, RangerDataHistService.ACTION_DELETE);
+ bizUtil.createTrxLog(trxLogList);
}
}
if(LOG.isDebugEnabled()) {
@@ -2123,30 +2131,6 @@ public class ServiceDBStore extends AbstractServiceStore {
}
}
- public void createTrxLogsAndHistoryAfterDelete(List<RangerPolicy> policies, RangerService service) throws Exception {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> ServiceDBStore.createTrxLogsAndHistoryAfterDelete");
- }
- if(CollectionUtils.isNotEmpty(policies)) {
- for (RangerPolicy policy : policies) {
- List<XXTrxLog> trxLogList = getTransactionLogList(policy, RangerPolicyService.OPERATION_IMPORT_DELETE_CONTEXT, RangerPolicyService.OPERATION_DELETE_CONTEXT);
-
- if(service==null) {
- service = getServiceByName(policy.getService());
- }
- handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE, policy);
-
- dataHistService.createObjectDataHistory(policy, RangerDataHistService.ACTION_DELETE, false);
-
- bizUtil.createTrxLog(trxLogList, false);
- }
- XXDBBaseDao xXDBBaseDao = daoMgr.getXXDBBase();
- if(xXDBBaseDao!=null) {
- xXDBBaseDao.flush();
- }
- }
- }
-
List<XXTrxLog> getTransactionLogList(RangerPolicy policy, int operationImportContext, int operationContext) {
List<XXTrxLog> trxLogList;
StackTraceElement[] trace = Thread.currentThread().getStackTrace();
@@ -2290,9 +2274,11 @@ public class ServiceDBStore extends AbstractServiceStore {
public List<RangerPolicy> noZoneFilter(List<RangerPolicy> servicePolicies) {
List<RangerPolicy> noZonePolicies = new ArrayList<RangerPolicy>();
- for (RangerPolicy policy : servicePolicies) {
- if (StringUtils.isBlank(policy.getZoneName())) {
- noZonePolicies.add(policy);
+ if (CollectionUtils.isNotEmpty(servicePolicies)) {
+ for (RangerPolicy policy : servicePolicies) {
+ if (StringUtils.isBlank(policy.getZoneName())) {
+ noZonePolicies.add(policy);
+ }
}
}
return noZonePolicies;
@@ -3064,14 +3050,19 @@ public class ServiceDBStore extends AbstractServiceStore {
RangerService service = getServiceByName(serviceName);
List<Long> policyIds = policyDao.findPolicyIdsByServiceNameAndZoneId(serviceName, zoneId);
if (CollectionUtils.isNotEmpty(policyIds)) {
- List<RangerPolicy> rangerPolicyList=new ArrayList<RangerPolicy>();
+ List<RangerPolicy> rangerPolicyList = new ArrayList<RangerPolicy>();
for (Long id : policyIds) {
rangerPolicyList.add(getPolicy(id));
}
+ long totalDeletedPolicies = 0;
for (RangerPolicy rangerPolicy : rangerPolicyList) {
deletePolicy(rangerPolicy, service);
+ totalDeletedPolicies = totalDeletedPolicies + 1;
+ if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) {
+ bizUtil.bulkModeOnlyFlushAndClear();
+ }
}
- createTrxLogsAndHistoryAfterDelete(rangerPolicyList, service);
+ bizUtil.bulkModeOnlyFlushAndClear();
}
}
}
@@ -3376,7 +3367,7 @@ public class ServiceDBStore extends AbstractServiceStore {
}
- private Boolean deleteExistingPolicyLabel(RangerPolicy policy, boolean flush) {
+ private Boolean deleteExistingPolicyLabel(RangerPolicy policy) {
if (policy == null) {
return false;
}
@@ -3384,7 +3375,7 @@ public class ServiceDBStore extends AbstractServiceStore {
List<XXPolicyLabelMap> xxPolicyLabelMaps = daoMgr.getXXPolicyLabelMap().findByPolicyId(policy.getId());
XXPolicyLabelMapDao policyLabelMapDao = daoMgr.getXXPolicyLabelMap();
for (XXPolicyLabelMap xxPolicyLabelMap : xxPolicyLabelMaps) {
- policyLabelMapDao.remove(xxPolicyLabelMap, flush);
+ policyLabelMapDao.remove(xxPolicyLabelMap);
}
return true;
}
diff --git a/security-admin/src/main/java/org/apache/ranger/common/ContextUtil.java b/security-admin/src/main/java/org/apache/ranger/common/ContextUtil.java
index 9554b4f..dd7b73e 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/ContextUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/ContextUtil.java
@@ -19,6 +19,7 @@
package org.apache.ranger.common;
+import org.apache.ranger.security.context.RangerAdminOpContext;
import org.apache.ranger.security.context.RangerContextHolder;
import org.apache.ranger.security.context.RangerSecurityContext;
@@ -81,4 +82,13 @@ public class ContextUtil {
return null;
}
+ public static boolean isBulkModeContext() {
+ RangerAdminOpContext context = RangerContextHolder.getOpContext();
+ boolean bulkMode = false;
+ if (context != null) {
+ bulkMode = context.isBulkModeContext();
+ }
+ return bulkMode;
+ }
+
}
diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java
index 4f33908..bdd8fbb 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java
@@ -86,15 +86,18 @@ public abstract class BaseDao<T> {
T ret = null;
em.persist(obj);
- em.flush();
-
+ if (!RangerBizUtil.isBulkMode()) {
+ em.flush();
+ }
ret = obj;
return ret;
}
public T update(T obj) {
em.merge(obj);
- em.flush();
+ if (!RangerBizUtil.isBulkMode()) {
+ em.flush();
+ }
return obj;
}
@@ -106,10 +109,13 @@ public abstract class BaseDao<T> {
if (obj == null) {
return true;
}
-
+ if (!em.contains(obj)) {
+ obj = em.merge(obj);
+ }
em.remove(obj);
- em.flush();
-
+ if (!RangerBizUtil.isBulkMode()) {
+ em.flush();
+ }
return true;
}
@@ -117,6 +123,9 @@ public abstract class BaseDao<T> {
em.flush();
}
+ public void clear() {
+ em.clear();
+ }
public T create(T obj, boolean flush) {
T ret = null;
em.persist(obj);
diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java
index 0788fb5..1a1cc23 100644
--- a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java
+++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingPolicyJson_J10019.java
@@ -235,7 +235,7 @@ public class PatchForUpdatingPolicyJson_J10019 extends BaseLoader {
public String doInTransaction(TransactionStatus status) {
String ret = null;
try {
- policyRefUpdater.cleanupRefTables(policy, true);
+ policyRefUpdater.cleanupRefTables(policy);
portPolicy(service.getType(), policy);
} catch (Throwable e) {
logger.error("PortPolicy failed for policy:[" + policy + "]", e);
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
index 348d072..af74daf 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
@@ -118,6 +118,8 @@ import org.apache.ranger.plugin.util.RangerPerfTracer;
import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.plugin.util.ServicePolicies;
import org.apache.ranger.security.context.RangerAPIList;
+import org.apache.ranger.security.context.RangerAdminOpContext;
+import org.apache.ranger.security.context.RangerContextHolder;
import org.apache.ranger.security.web.filter.RangerCSRFPreventionFilter;
import org.apache.ranger.service.RangerPluginInfoService;
import org.apache.ranger.service.RangerPolicyLabelsService;
@@ -142,7 +144,6 @@ import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
-import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.sun.jersey.core.header.FormDataContentDisposition;
import com.sun.jersey.multipart.FormDataParam;
@@ -336,7 +337,9 @@ public class ServiceREST {
if(LOG.isDebugEnabled()) {
LOG.debug("==> ServiceREST.deleteServiceDef(" + id + ")");
}
-
+ RangerAdminOpContext opContext = new RangerAdminOpContext();
+ opContext.setBulkModeContext(true);
+ RangerContextHolder.setOpContext(opContext);
RangerPerfTracer perf = null;
try {
@@ -789,7 +792,9 @@ public class ServiceREST {
if(LOG.isDebugEnabled()) {
LOG.debug("==> ServiceREST.deleteService(" + id + ")");
}
-
+ RangerAdminOpContext opContext = new RangerAdminOpContext();
+ opContext.setBulkModeContext(true);
+ RangerContextHolder.setOpContext(opContext);
RangerPerfTracer perf = null;
try {
@@ -2112,24 +2117,27 @@ public class ServiceREST {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ServiceREST.importPoliciesFromFile()");
}
+ RangerAdminOpContext opContext = new RangerAdminOpContext();
+ opContext.setBulkModeContext(true);
+ RangerContextHolder.setOpContext(opContext);
RangerPerfTracer perf = null;
String metaDataInfo = null;
List<XXTrxLog> trxLogListError = new ArrayList<XXTrxLog>();
XXTrxLog xxTrxLogError = new XXTrxLog();
-
+
try {
if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = RangerPerfTracer.getPerfTracer(PERF_LOG,"ServiceREST.importPoliciesFromFile()");
}
-
+
List<XXTrxLog> trxLogList = new ArrayList<XXTrxLog>();
XXTrxLog xxTrxLog = new XXTrxLog();
xxTrxLog.setAction("IMPORT START");
xxTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_RANGER_POLICY);
- xxTrxLog.setPreviousValue("IMPORT START");
+ xxTrxLog.setPreviousValue("IMPORT START");
trxLogList.add(xxTrxLog);
bizUtil.createTrxLog(trxLogList);
-
+
if (isOverride == null){
isOverride = false;
}
@@ -2398,7 +2406,11 @@ public class ServiceREST {
}
}
}
+ if(totalPolicyCreate % RangerBizUtil.batchSize == 0) {
+ bizUtil.bulkModeOnlyFlushAndClear();
+ }
}
+ bizUtil.bulkModeOnlyFlushAndClear();
if (LOG.isDebugEnabled()) {
LOG.debug("Total Policy Created From Json file : " + totalPolicyCreate);
}
@@ -2419,11 +2431,9 @@ public class ServiceREST {
private RangerExportPolicyList processPolicyInputJsonForMetaData(InputStream uploadedInputStream,
RangerExportPolicyList rangerExportPolicyList) throws Exception {
- Gson gson = new Gson();
String policiesString = IOUtils.toString(uploadedInputStream);
policiesString = policiesString.trim();
if (StringUtils.isNotEmpty(policiesString)) {
- gson.fromJson(policiesString, RangerExportPolicyList.class);
rangerExportPolicyList = JsonUtilsV2.jsonToObj(policiesString, RangerExportPolicyList.class);
} else {
LOG.error("Provided json file is empty!!");
@@ -2644,7 +2654,7 @@ public class ServiceREST {
}
private void deletePoliciesProvidedInServiceMap(List<String> sourceServices, List<String> destinationServices, String zoneName) throws Exception {
- int totalDeletedPilicies = 0;
+ int totalDeletedPolicies = 0;
if (CollectionUtils.isNotEmpty(sourceServices)
&& CollectionUtils.isNotEmpty(destinationServices)) {
RangerPolicyValidator validator = validatorFactory.getPolicyValidator(svcStore);
@@ -2663,26 +2673,29 @@ public class ServiceREST {
ensureAdminAccess(rangerPolicy);
bizUtil.blockAuditorRoleUser();
svcStore.deletePolicy(rangerPolicy, service);
- totalDeletedPilicies = totalDeletedPilicies + 1;
+ totalDeletedPolicies = totalDeletedPolicies + 1;
+ if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) {
+ bizUtil.bulkModeOnlyFlushAndClear();
+ }
if (LOG.isDebugEnabled()) {
- LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully." );
- LOG.debug("TotalDeletedPilicies: " +totalDeletedPilicies);
+ LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully.");
+ LOG.debug("TotalDeletedPilicies: " + totalDeletedPolicies);
}
}
}
- svcStore.createTrxLogsAndHistoryAfterDelete(rangerPolicyList,service);
+ bizUtil.bulkModeOnlyFlushAndClear();
}
}
}
}
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Total Deleted Policy : " + totalDeletedPilicies);
+ LOG.debug("Total Deleted Policy : " + totalDeletedPolicies);
}
}
private void deletePoliciesForResource(List<String> sourceServices, List<String> destinationServices, HttpServletRequest request, List<RangerPolicy> exportPolicies, String zoneName) throws Exception {
- int totalDeletedPilicies = 0;
+ int totalDeletedPolicies = 0;
if (CollectionUtils.isNotEmpty(sourceServices)
&& CollectionUtils.isNotEmpty(destinationServices)) {
Set<String> exportedPolicyNames=new HashSet<String>();
@@ -2733,9 +2746,12 @@ public class ServiceREST {
if (LOG.isDebugEnabled()) {
LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully.");
}
- totalDeletedPilicies = totalDeletedPilicies + 1;
+ totalDeletedPolicies = totalDeletedPolicies + 1;
+ if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) {
+ bizUtil.bulkModeOnlyFlushAndClear();
+ }
}
- svcStore.createTrxLogsAndHistoryAfterDelete(policiesToBeDeleted, service);
+ bizUtil.bulkModeOnlyFlushAndClear();
}
}
}
@@ -4162,7 +4178,7 @@ public class ServiceREST {
private void deleteExactMatchPolicyForResource(List<RangerPolicy> policies, String user, String zoneName) throws Exception {
if (CollectionUtils.isNotEmpty(policies)) {
- List<RangerPolicy> existingMatchedPolicies=new ArrayList<RangerPolicy>();
+ long totalDeletedPolicies = 0;
for (RangerPolicy rangerPolicy : policies) {
RangerPolicy existingPolicy = null ;
try {
@@ -4173,17 +4189,18 @@ public class ServiceREST {
} catch (Exception e) {
existingPolicy=null;
}
- if(existingPolicy!=null) {
+ if (existingPolicy != null) {
svcStore.deletePolicy(existingPolicy, null);
- existingMatchedPolicies.add(existingPolicy);
+ totalDeletedPolicies = totalDeletedPolicies + 1;
+ if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) {
+ bizUtil.bulkModeOnlyFlushAndClear();
+ }
if (LOG.isDebugEnabled()) {
LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully.");
}
}
}
- if (CollectionUtils.isNotEmpty(existingMatchedPolicies)) {
- svcStore.createTrxLogsAndHistoryAfterDelete(existingMatchedPolicies, null);
- }
+ bizUtil.bulkModeOnlyFlushAndClear();
}
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAdminOpContext.java
similarity index 60%
copy from security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java
copy to security-admin/src/main/java/org/apache/ranger/security/context/RangerAdminOpContext.java
index 9884163..a447882 100644
--- a/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java
+++ b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAdminOpContext.java
@@ -17,29 +17,20 @@
* under the License.
*/
- /**
- *
- */
-package org.apache.ranger.security.context;
-
-public class RangerContextHolder {
-
- private static final ThreadLocal<RangerSecurityContext> securityContextThreadLocal = new ThreadLocal<RangerSecurityContext>();
-
- private RangerContextHolder() {
-
- }
-
- public static RangerSecurityContext getSecurityContext(){
- return securityContextThreadLocal.get();
- }
-
- public static void setSecurityContext(RangerSecurityContext context){
- securityContextThreadLocal.set(context);
- }
-
- public static void resetSecurityContext(){
- securityContextThreadLocal.remove();
- }
-
-}
+package org.apache.ranger.security.context;
+
+import java.io.Serializable;
+
+public class RangerAdminOpContext implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private boolean bulkModeContext = false;
+
+ public boolean isBulkModeContext() {
+ return bulkModeContext;
+ }
+
+ public void setBulkModeContext(boolean bulkMode) {
+ this.bulkModeContext = bulkMode;
+ }
+
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java b/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java
index 9884163..865a849 100644
--- a/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java
+++ b/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java
@@ -26,6 +26,8 @@ public class RangerContextHolder {
private static final ThreadLocal<RangerSecurityContext> securityContextThreadLocal = new ThreadLocal<RangerSecurityContext>();
+ private static final ThreadLocal<RangerAdminOpContext> operationContextThreadLocal = new ThreadLocal<RangerAdminOpContext>();
+
private RangerContextHolder() {
}
@@ -41,5 +43,16 @@ public class RangerContextHolder {
public static void resetSecurityContext(){
securityContextThreadLocal.remove();
}
-
+
+ public static RangerAdminOpContext getOpContext() {
+ return operationContextThreadLocal.get();
+ }
+
+ public static void setOpContext(RangerAdminOpContext context) {
+ operationContextThreadLocal.set(context);
+ }
+
+ public static void resetOpContext() {
+ operationContextThreadLocal.remove();
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java
index d75b903..eb40cfd 100644
--- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java
+++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java
@@ -147,6 +147,7 @@ public class RangerSecurityContextFormationFilter extends GenericFilterBean {
} finally {
// [4]remove context from thread-local
RangerContextHolder.resetSecurityContext();
+ RangerContextHolder.resetOpContext();
}
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
index de5d343..2f95517 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java
@@ -227,6 +227,13 @@ public abstract class RangerBaseModelService<T extends XXDBBase, V extends Range
return vObj;
}
+ public V create(V vObj, boolean flush) {
+ T resource = preCreate(vObj);
+ resource = getDao().create(resource, flush);
+ vObj = postCreate(resource);
+ return vObj;
+ }
+
public V read(Long id) {
T resource = getDao().getById(id);
if (resource == null) {
@@ -295,7 +302,7 @@ public abstract class RangerBaseModelService<T extends XXDBBase, V extends Range
tEntityClass.getSimpleName() + ":" + id);
}
try {
- result = getDao().remove(resource, flush);
+ result = getDao().remove(resource);
} catch (Exception e) {
LOG.error("Error deleting " + tEntityClass.getSimpleName()
+ ". Id=" + id, e);
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java
index 019816e..7bd0681 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerDataHistService.java
@@ -53,7 +53,7 @@ public class RangerDataHistService {
public static final String ACTION_UPDATE = "Update";
public static final String ACTION_DELETE = "Delete";
- public void createObjectDataHistory(RangerBaseModelObject baseModelObj, String action, boolean flush) {
+ public void createObjectDataHistory(RangerBaseModelObject baseModelObj, String action) {
if(baseModelObj == null || action == null) {
throw restErrorUtil
.createRESTException("Error while creating DataHistory. "
@@ -108,7 +108,7 @@ public class RangerDataHistService {
xDataHist.setObjectClassType(classType);
xDataHist.setObjectName(objectName);
xDataHist.setContent(content);
- xDataHist = daoMgr.getXXDataHist().create(xDataHist, flush);
+ xDataHist = daoMgr.getXXDataHist().create(xDataHist);
if (ACTION_UPDATE.equalsIgnoreCase(action) || ACTION_DELETE.equalsIgnoreCase(action)) {
XXDataHist prevHist = daoMgr.getXXDataHist().findLatestByObjectClassTypeAndObjectId(classType, objectId);
@@ -122,7 +122,7 @@ public class RangerDataHistService {
prevHist.setUpdateTime(currentDate);
prevHist.setToTime(currentDate);
prevHist.setObjectName(objectName);
- prevHist = daoMgr.getXXDataHist().update(prevHist, flush);
+ prevHist = daoMgr.getXXDataHist().update(prevHist);
}
}
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index 4ea2ad6..3c3d1de 100755
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -863,8 +863,7 @@
<!-- XXDataHist -->
<named-query name="XXDataHist.findLatestByObjectClassTypeAndObjectId">
- <query>select obj from XXDataHist obj where obj.objectId = :objectId
- and obj.objectClassType = :classType ORDER BY obj.id</query>
+ <query>select obj from XXDataHist obj where obj.objectId = :objectId and obj.objectClassType = :classType</query>
</named-query>
<named-query name="XXDataHist.getVersionListOfObject">
@@ -873,8 +872,7 @@
</named-query>
<named-query name="XXDataHist.findObjectByVersionNumber">
- <query>select obj from XXDataHist obj where obj.objectClassType = :classType and
- obj.objectId = :objId and obj.version = :version</query>
+ <query>select obj from XXDataHist obj where obj.objectId = :objId and obj.objectClassType = :classType and obj.version = :version</query>
</named-query>
<!-- XXTrxLog -->
diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
index eb7947a..34e8303 100644
--- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
+++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
@@ -366,6 +366,18 @@
</property>
<property>
+ <name>ranger.jpa.jdbc.batch-clear.enable</name>
+ <value>true</value>
+ <description>property to enable bulk mode optimization</description>
+ </property>
+
+ <property>
+ <name>ranger.jpa.jdbc.batch-clear.size</name>
+ <value>10</value>
+ <description>batch size to flush and clear jdbc statements</description>
+ </property>
+
+ <property>
<name>ranger.jpa.jdbc.credential.alias</name>
<value>ranger.db.password</value>
<description></description>
diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
index b6f13f4..2be3718 100644
--- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
+++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
@@ -1694,7 +1694,7 @@ public class TestServiceDBStore {
Mockito.when(daoManager.getXXPolicy()).thenReturn(xPolicyDao);
- Mockito.when(policyService.create(rangerPolicy)).thenReturn(
+ Mockito.when(policyService.create(rangerPolicy, true)).thenReturn(
rangerPolicy);
Mockito.when(daoManager.getXXPolicy()).thenReturn(xPolicyDao);
@@ -1846,7 +1846,7 @@ public class TestServiceDBStore {
Mockito.when(factory.createPolicyResourceSignature(rangerPolicy))
.thenReturn(signature);
Mockito.when(!bizUtil.hasAccess(xService, null)).thenReturn(true);
- Mockito.when(policyRefUpdater.cleanupRefTables(rangerPolicy, true)).thenReturn(true);
+ Mockito.when(policyRefUpdater.cleanupRefTables(rangerPolicy)).thenReturn(true);
RangerPolicy dbRangerPolicy = serviceDBStore.updatePolicy(rangerPolicy);
@@ -1993,7 +1993,7 @@ public class TestServiceDBStore {
Mockito.when(xPolicyLabelMapDao.findByPolicyId(rangerPolicy.getId())).thenReturn(ListUtils.EMPTY_LIST);
Mockito.when(!bizUtil.hasAccess(xService, null)).thenReturn(true);
- Mockito.when(policyRefUpdater.cleanupRefTables(rangerPolicy, true)).thenReturn(true);
+ Mockito.when(policyRefUpdater.cleanupRefTables(rangerPolicy)).thenReturn(true);
serviceDBStore.deletePolicy(Id);
}
diff --git a/security-admin/src/test/java/org/apache/ranger/service/TestRangerDataHistService.java b/security-admin/src/test/java/org/apache/ranger/service/TestRangerDataHistService.java
index 655e17f..65e273c 100644
--- a/security-admin/src/test/java/org/apache/ranger/service/TestRangerDataHistService.java
+++ b/security-admin/src/test/java/org/apache/ranger/service/TestRangerDataHistService.java
@@ -46,7 +46,7 @@ public class TestRangerDataHistService {
String action = "create";
RangerBaseModelObject baseModelObj = new RangerBaseModelObject();
Mockito.when(daoMgr.getXXDataHist()).thenReturn(XXDataHistDao);
- rangerDataHistService.createObjectDataHistory(baseModelObj, action, true);
+ rangerDataHistService.createObjectDataHistory(baseModelObj, action);
}
}