You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ab...@apache.org on 2018/09/27 01:53:00 UTC
[1/2] ranger git commit: RANGER-2165: Address JPA Cache issue when
policies Create,
Update and Delete are done via REST API in Apache Ranger admin
Repository: ranger
Updated Branches:
refs/heads/ranger-0.7 475b5290a -> dddcf0155
RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/320169a8
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/320169a8
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/320169a8
Branch: refs/heads/ranger-0.7
Commit: 320169a8a796d2f7bf10a45fff4a39fcb80cec9a
Parents: 475b529
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Jul 31 15:15:33 2018 -0700
Committer: Abhay Kulkarni <ak...@hortonworks.com>
Committed: Wed Sep 26 18:38:05 2018 -0700
----------------------------------------------------------------------
.../ranger/biz/RangerPolicyRetriever.java | 83 ++++++++++++++--
.../apache/ranger/biz/RangerTagDBRetriever.java | 99 ++++++++++++++++++--
.../org/apache/ranger/biz/ServiceDBStore.java | 2 +-
.../java/org/apache/ranger/biz/TagDBStore.java | 8 +-
4 files changed, 175 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/320169a8/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index 1b6f440..e867cf4 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -43,18 +43,36 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator;
import org.apache.ranger.plugin.util.RangerPerfTracer;
-
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
public class RangerPolicyRetriever {
static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class);
static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever");
- final RangerDaoManager daoMgr;
- final LookupCache lookupCache;
+ private final RangerDaoManager daoMgr;
+ private final LookupCache lookupCache = new LookupCache();
+
+ private final PlatformTransactionManager txManager;
+ private final TransactionTemplate txTemplate;
+
+ public RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) {
+ this.daoMgr = daoMgr;
+ this.txManager = txManager;
+ if (this.txManager != null) {
+ this.txTemplate = new TransactionTemplate(this.txManager);
+ this.txTemplate.setReadOnly(true);
+ } else {
+ this.txTemplate = null;
+ }
+ }
public RangerPolicyRetriever(RangerDaoManager daoMgr) {
this.daoMgr = daoMgr;
- this.lookupCache = new LookupCache();
+ this.txManager = null;
+ this.txTemplate = null;
}
public List<RangerPolicy> getServicePolicies(Long serviceId) {
@@ -93,7 +111,41 @@ public class RangerPolicyRetriever {
return ret;
}
- public List<RangerPolicy> getServicePolicies(XXService xService) {
+ private class PolicyLoaderThread extends Thread {
+ final TransactionTemplate txTemplate;
+ final XXService xService;
+ List<RangerPolicy> policies;
+
+ PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+ this.txTemplate = txTemplate;
+ this.xService = xService;
+ }
+
+ public List<RangerPolicy> getPolicies() { return policies; }
+
+ @Override
+ public void run() {
+ try {
+ policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>() {
+ @Override
+ public List<RangerPolicy> doInTransaction(TransactionStatus status) {
+ try {
+ RetrieverContext ctx = new RetrieverContext(xService);
+ return ctx.getAllPolicies();
+ } catch (Exception ex) {
+ LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex);
+ status.setRollbackOnly();
+ return null;
+ }
+ }
+ });
+ } catch (Throwable ex) {
+ LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex);
+ }
+ }
+ }
+
+ public List<RangerPolicy> getServicePolicies(final XXService xService) {
String serviceName = xService == null ? null : xService.getName();
Long serviceId = xService == null ? null : xService.getId();
@@ -109,9 +161,26 @@ public class RangerPolicyRetriever {
}
if(xService != null) {
- RetrieverContext ctx = new RetrieverContext(xService);
+ if (txTemplate == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction");
+ }
+ RetrieverContext ctx = new RetrieverContext(xService);
+ ret = ctx.getAllPolicies();
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Retrieving policies in a new, read-only transaction");
+ }
- ret = ctx.getAllPolicies();
+ PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService);
+ t.start();
+ try {
+ t.join();
+ ret = t.getPolicies();
+ } catch (InterruptedException ie) {
+ LOG.error("Failed to retrieve policies in a new, read-only thread.", ie);
+ }
+ }
} else {
if(LOG.isDebugEnabled()) {
LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter");
http://git-wip-us.apache.org/repos/asf/ranger/blob/320169a8/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index 52c1288..26d8d37 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -35,6 +35,10 @@ import org.apache.ranger.entity.*;
import org.apache.ranger.plugin.model.*;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
public class RangerTagDBRetriever {
@@ -46,6 +50,9 @@ public class RangerTagDBRetriever {
private final XXService xService;
private final LookupCache lookupCache;
+ private final PlatformTransactionManager txManager;
+ private final TransactionTemplate txTemplate;
+
private List<RangerServiceResource> serviceResources;
private Map<Long, RangerTagDef> tagDefs;
private Map<Long, RangerTag> tags;
@@ -53,8 +60,15 @@ public class RangerTagDBRetriever {
private boolean filterForServicePlugin;
- public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xService) {
+ public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) {
this.daoMgr = daoMgr;
+ this.txManager = txManager;
+ if (this.txManager != null) {
+ this.txTemplate = new TransactionTemplate(this.txManager);
+ this.txTemplate.setReadOnly(true);
+ } else {
+ this.txTemplate = null;
+ }
this.xService = xService;
this.lookupCache = new LookupCache();
@@ -68,14 +82,27 @@ public class RangerTagDBRetriever {
}
filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false);
- TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService);
- TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService);
- TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService);
- serviceResources = serviceResourceContext.getAllServiceResources();
- tagDefs = tagDefContext.getAllTagDefs();
- tags = tagContext.getAllTags();
- tagResourceMaps = getAllTagResourceMaps();
+ if (this.txTemplate == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Load Tags in the same thread and using an existing transaction");
+ }
+ if (initializeTagCache(xService) == false) {
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction");
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Load Tags in a separate thread and using a new transaction");
+ }
+
+ TagLoaderThread t = new TagLoaderThread(txTemplate, xService);
+ t.start();
+ try {
+ t.join();
+ } catch (InterruptedException ie) {
+ LOG.error("Failed to get Tags in a separate thread and using a new transaction", ie);
+ }
+ }
RangerPerfTracer.log(perf);
@@ -98,6 +125,30 @@ public class RangerTagDBRetriever {
return tags;
}
+ private boolean initializeTagCache(XXService xService) {
+ boolean ret;
+ try {
+ TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService);
+ TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService);
+ TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService);
+
+ serviceResources = serviceResourceContext.getAllServiceResources();
+ tagDefs = tagDefContext.getAllTagDefs();
+ tags = tagContext.getAllTags();
+
+ tagResourceMaps = getAllTagResourceMaps();
+
+ ret = true;
+ } catch (Exception ex) {
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "]");
+ serviceResources = null;
+ tagDefs = null;
+ tags = null;
+ tagResourceMaps = null;
+ ret = false;
+ }
+ return ret;
+ }
private List<RangerTagResourceMap> getAllTagResourceMaps() {
List<XXTagResourceMap> xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId()) : daoMgr.getXXTagResourceMap().findByServiceId(xService.getId());
@@ -198,6 +249,38 @@ public class RangerTagDBRetriever {
}
}
+ private class TagLoaderThread extends Thread {
+ final TransactionTemplate txTemplate;
+ final XXService xService;
+
+ TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+ this.txTemplate = txTemplate;
+ this.xService = xService;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
+ @Override
+ public Boolean doInTransaction(TransactionStatus status) {
+ boolean ret = initializeTagCache(xService);
+ if (!ret) {
+ status.setRollbackOnly();
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction");
+ }
+ return ret;
+ }
+ });
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("transaction result:[" + result +"]");
+ }
+ } catch (Throwable ex) {
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction", ex);
+ }
+ }
+ }
+
private class TagRetrieverServiceResourceContext {
final XXService service;
http://git-wip-us.apache.org/repos/asf/ranger/blob/320169a8/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index ed6ddac..64cf043 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -2407,7 +2407,7 @@ public class ServiceDBStore extends AbstractServiceStore {
LOG.debug("==> ServiceDBStore.getServicePoliciesFromDb(" + service.getName() + ")");
}
- RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr);
+ RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager);
List<RangerPolicy> ret = policyRetriever.getServicePolicies(service);
http://git-wip-us.apache.org/repos/asf/ranger/blob/320169a8/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index a115bb7..3234be6 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -62,7 +62,9 @@ import org.apache.ranger.service.RangerTagResourceMapService;
import org.apache.ranger.service.RangerTagService;
import org.apache.ranger.service.RangerServiceResourceService;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletResponse;
@@ -87,6 +89,10 @@ public class TagDBStore extends AbstractTagStore {
RangerDaoManager daoManager;
@Autowired
+ @Qualifier(value = "transactionManager")
+ PlatformTransactionManager txManager;
+
+ @Autowired
RESTErrorUtil errorUtil;
@Autowired
@@ -985,7 +991,7 @@ public class TagDBStore extends AbstractTagStore {
throw new Exception("service-def does not exist. id=" + xxService.getType());
}
- RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService);
+ RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, txManager, xxService);
Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs();
Map<Long, RangerTag> tagMap = tagDBRetriever.getTags();
[2/2] ranger git commit: RANGER-2165: Address JPA Cache issue when
policies Create,
Update and Delete are done via REST API in Apache Ranger admin
Posted by ab...@apache.org.
RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/dddcf015
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/dddcf015
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/dddcf015
Branch: refs/heads/ranger-0.7
Commit: dddcf0155f63c35a347755c41e7063ccef93d308
Parents: 320169a
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Aug 14 10:49:34 2018 -0700
Committer: Abhay Kulkarni <ak...@hortonworks.com>
Committed: Wed Sep 26 18:38:30 2018 -0700
----------------------------------------------------------------------
.../main/java/org/apache/ranger/biz/RangerPolicyRetriever.java | 1 +
.../src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/dddcf015/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index e867cf4..7d1e15b 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -126,6 +126,7 @@ public class RangerPolicyRetriever {
@Override
public void run() {
try {
+ txTemplate.setReadOnly(true);
policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>() {
@Override
public List<RangerPolicy> doInTransaction(TransactionStatus status) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/dddcf015/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index 26d8d37..27856ae 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -261,7 +261,8 @@ public class RangerTagDBRetriever {
@Override
public void run() {
try {
- Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
+ txTemplate.setReadOnly(true);
+ Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
boolean ret = initializeTagCache(xService);