You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by me...@apache.org on 2018/09/04 12:58:17 UTC
[07/28] ranger git commit: RANGER-2165: Address JPA Cache issue when
policies Create,
Update and Delete are done via REST API in Apache Ranger admin
RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/1a358571
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/1a358571
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/1a358571
Branch: refs/heads/ranger-1.1
Commit: 1a3585717f2c27b100a6dd3dda4198da4ee1ec33
Parents: 4a13ff7
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Jul 31 15:15:33 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:29:46 2018 +0530
----------------------------------------------------------------------
.../ranger/biz/RangerPolicyRetriever.java | 82 ++++++++++++++--
.../apache/ranger/biz/RangerTagDBRetriever.java | 99 ++++++++++++++++++--
.../org/apache/ranger/biz/ServiceDBStore.java | 2 +-
.../java/org/apache/ranger/biz/TagDBStore.java | 8 +-
4 files changed, 175 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index 463957c..3e291d5 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -47,17 +47,36 @@ import org.apache.ranger.plugin.model.RangerValiditySchedule;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator;
import org.apache.ranger.plugin.util.RangerPerfTracer;
import org.apache.ranger.service.RangerPolicyService;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
public class RangerPolicyRetriever {
static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class);
static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever");
- final RangerDaoManager daoMgr;
- final LookupCache lookupCache;
+ private final RangerDaoManager daoMgr;
+ private final LookupCache lookupCache = new LookupCache();
+
+ private final PlatformTransactionManager txManager;
+ private final TransactionTemplate txTemplate;
+
+ public RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) {
+ this.daoMgr = daoMgr;
+ this.txManager = txManager;
+ if (this.txManager != null) {
+ this.txTemplate = new TransactionTemplate(this.txManager);
+ this.txTemplate.setReadOnly(true);
+ } else {
+ this.txTemplate = null;
+ }
+ }
public RangerPolicyRetriever(RangerDaoManager daoMgr) {
this.daoMgr = daoMgr;
- this.lookupCache = new LookupCache();
+ this.txManager = null;
+ this.txTemplate = null;
}
public List<RangerPolicy> getServicePolicies(Long serviceId) {
@@ -96,7 +115,41 @@ public class RangerPolicyRetriever {
return ret;
}
- public List<RangerPolicy> getServicePolicies(XXService xService) {
+ private class PolicyLoaderThread extends Thread {
+ final TransactionTemplate txTemplate;
+ final XXService xService;
+ List<RangerPolicy> policies;
+
+ PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+ this.txTemplate = txTemplate;
+ this.xService = xService;
+ }
+
+ public List<RangerPolicy> getPolicies() { return policies; }
+
+ @Override
+ public void run() {
+ try {
+ policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>() {
+ @Override
+ public List<RangerPolicy> doInTransaction(TransactionStatus status) {
+ try {
+ RetrieverContext ctx = new RetrieverContext(xService);
+ return ctx.getAllPolicies();
+ } catch (Exception ex) {
+ LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex);
+ status.setRollbackOnly();
+ return null;
+ }
+ }
+ });
+ } catch (Throwable ex) {
+ LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex);
+ }
+ }
+ }
+
+ public List<RangerPolicy> getServicePolicies(final XXService xService) {
String serviceName = xService == null ? null : xService.getName();
Long serviceId = xService == null ? null : xService.getId();
@@ -112,9 +165,26 @@ public class RangerPolicyRetriever {
}
if(xService != null) {
- RetrieverContext ctx = new RetrieverContext(xService);
+ if (txTemplate == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction");
+ }
+ RetrieverContext ctx = new RetrieverContext(xService);
+ ret = ctx.getAllPolicies();
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Retrieving policies in a new, read-only transaction");
+ }
- ret = ctx.getAllPolicies();
+ PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService);
+ t.start();
+ try {
+ t.join();
+ ret = t.getPolicies();
+ } catch (InterruptedException ie) {
+ LOG.error("Failed to retrieve policies in a new, read-only thread.", ie);
+ }
+ }
} else {
if(LOG.isDebugEnabled()) {
LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter");
http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index 5550572..789068f 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -38,6 +38,10 @@ import org.apache.ranger.entity.*;
import org.apache.ranger.plugin.model.*;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
public class RangerTagDBRetriever {
static final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class);
@@ -48,6 +52,9 @@ public class RangerTagDBRetriever {
private final XXService xService;
private final LookupCache lookupCache;
+ private final PlatformTransactionManager txManager;
+ private final TransactionTemplate txTemplate;
+
private List<RangerServiceResource> serviceResources;
private Map<Long, RangerTagDef> tagDefs;
private Map<Long, RangerTag> tags;
@@ -55,8 +62,15 @@ public class RangerTagDBRetriever {
private boolean filterForServicePlugin;
- public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xService) {
+ public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) {
this.daoMgr = daoMgr;
+ this.txManager = txManager;
+ if (this.txManager != null) {
+ this.txTemplate = new TransactionTemplate(this.txManager);
+ this.txTemplate.setReadOnly(true);
+ } else {
+ this.txTemplate = null;
+ }
this.xService = xService;
this.lookupCache = new LookupCache();
@@ -70,14 +84,27 @@ public class RangerTagDBRetriever {
}
filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false);
- TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService);
- TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService);
- TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService);
- serviceResources = serviceResourceContext.getAllServiceResources();
- tagDefs = tagDefContext.getAllTagDefs();
- tags = tagContext.getAllTags();
- tagResourceMaps = getAllTagResourceMaps();
+ if (this.txTemplate == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Load Tags in the same thread and using an existing transaction");
+ }
+ if (initializeTagCache(xService) == false) {
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction");
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Load Tags in a separate thread and using a new transaction");
+ }
+
+ TagLoaderThread t = new TagLoaderThread(txTemplate, xService);
+ t.start();
+ try {
+ t.join();
+ } catch (InterruptedException ie) {
+ LOG.error("Failed to get Tags in a separate thread and using a new transaction", ie);
+ }
+ }
RangerPerfTracer.log(perf);
@@ -100,6 +127,30 @@ public class RangerTagDBRetriever {
return tags;
}
+ private boolean initializeTagCache(XXService xService) {
+ boolean ret;
+ try {
+ TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService);
+ TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService);
+ TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService);
+
+ serviceResources = serviceResourceContext.getAllServiceResources();
+ tagDefs = tagDefContext.getAllTagDefs();
+ tags = tagContext.getAllTags();
+
+ tagResourceMaps = getAllTagResourceMaps();
+
+ ret = true;
+ } catch (Exception ex) {
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "]");
+ serviceResources = null;
+ tagDefs = null;
+ tags = null;
+ tagResourceMaps = null;
+ ret = false;
+ }
+ return ret;
+ }
private List<RangerTagResourceMap> getAllTagResourceMaps() {
List<XXTagResourceMap> xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId()) : daoMgr.getXXTagResourceMap().findByServiceId(xService.getId());
@@ -200,6 +251,38 @@ public class RangerTagDBRetriever {
}
}
+ private class TagLoaderThread extends Thread {
+ final TransactionTemplate txTemplate;
+ final XXService xService;
+
+ TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+ this.txTemplate = txTemplate;
+ this.xService = xService;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
+ @Override
+ public Boolean doInTransaction(TransactionStatus status) {
+ boolean ret = initializeTagCache(xService);
+ if (!ret) {
+ status.setRollbackOnly();
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction");
+ }
+ return ret;
+ }
+ });
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("transaction result:[" + result +"]");
+ }
+ } catch (Throwable ex) {
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction", ex);
+ }
+ }
+ }
+
private class TagRetrieverServiceResourceContext {
final XXService service;
http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index e75ea68..0773616 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -2462,7 +2462,7 @@ public class ServiceDBStore extends AbstractServiceStore {
LOG.debug("==> ServiceDBStore.getServicePoliciesFromDb(" + service.getName() + ")");
}
- RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr);
+ RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager);
List<RangerPolicy> ret = policyRetriever.getServicePolicies(service);
http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index 8341a73..d29df93 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -64,7 +64,9 @@ import org.apache.ranger.service.RangerTagResourceMapService;
import org.apache.ranger.service.RangerTagService;
import org.apache.ranger.service.RangerServiceResourceService;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletResponse;
@@ -89,6 +91,10 @@ public class TagDBStore extends AbstractTagStore {
RangerDaoManager daoManager;
@Autowired
+ @Qualifier(value = "transactionManager")
+ PlatformTransactionManager txManager;
+
+ @Autowired
RESTErrorUtil errorUtil;
@Autowired
@@ -991,7 +997,7 @@ public class TagDBStore extends AbstractTagStore {
throw new Exception("service-def does not exist. id=" + xxService.getType());
}
- RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService);
+ RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, txManager, xxService);
Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs();
Map<Long, RangerTag> tagMap = tagDBRetriever.getTags();