You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by me...@apache.org on 2018/09/04 12:58:17 UTC

[07/28] ranger git commit: RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin

RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin


Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/1a358571
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/1a358571
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/1a358571

Branch: refs/heads/ranger-1.1
Commit: 1a3585717f2c27b100a6dd3dda4198da4ee1ec33
Parents: 4a13ff7
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Jul 31 15:15:33 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:29:46 2018 +0530

----------------------------------------------------------------------
 .../ranger/biz/RangerPolicyRetriever.java       | 82 ++++++++++++++--
 .../apache/ranger/biz/RangerTagDBRetriever.java | 99 ++++++++++++++++++--
 .../org/apache/ranger/biz/ServiceDBStore.java   |  2 +-
 .../java/org/apache/ranger/biz/TagDBStore.java  |  8 +-
 4 files changed, 175 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index 463957c..3e291d5 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -47,17 +47,36 @@ import org.apache.ranger.plugin.model.RangerValiditySchedule;
 import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator;
 import org.apache.ranger.plugin.util.RangerPerfTracer;
 import org.apache.ranger.service.RangerPolicyService;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
 
 public class RangerPolicyRetriever {
 	static final Log LOG      = LogFactory.getLog(RangerPolicyRetriever.class);
 	static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever");
 
-	final RangerDaoManager daoMgr;
-	final LookupCache      lookupCache;
+	private final RangerDaoManager  daoMgr;
+	private final LookupCache       lookupCache = new LookupCache();
+
+	private final PlatformTransactionManager  txManager;
+	private final TransactionTemplate         txTemplate;
+
+	public RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) {
+		this.daoMgr     = daoMgr;
+		this.txManager  = txManager;
+		if (this.txManager != null) {
+			this.txTemplate = new TransactionTemplate(this.txManager);
+			this.txTemplate.setReadOnly(true);
+		} else {
+			this.txTemplate = null;
+		}
+	}
 
 	public RangerPolicyRetriever(RangerDaoManager daoMgr) {
 		this.daoMgr      = daoMgr;
-		this.lookupCache = new LookupCache();
+		this.txManager   = null;
+		this.txTemplate  = null;
 	}
 
 	public List<RangerPolicy> getServicePolicies(Long serviceId) {
@@ -96,7 +115,41 @@ public class RangerPolicyRetriever {
 		return ret;
 	}
 
-	public List<RangerPolicy> getServicePolicies(XXService xService) {
+	private class PolicyLoaderThread extends Thread {
+		final TransactionTemplate txTemplate;
+		final XXService           xService;
+		List<RangerPolicy>  policies;
+
+		PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+			this.txTemplate = txTemplate;
+			this.xService   = xService;
+		}
+
+		public List<RangerPolicy> getPolicies() { return policies; }
+
+		@Override
+		public void run() {
+			try {
+				policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>() {
+					@Override
+					public List<RangerPolicy> doInTransaction(TransactionStatus status) {
+						try {
+							RetrieverContext ctx = new RetrieverContext(xService);
+							return ctx.getAllPolicies();
+						} catch (Exception ex) {
+							LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex);
+							status.setRollbackOnly();
+							return null;
+						}
+					}
+				});
+			} catch (Throwable ex) {
+				LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex);
+			}
+		}
+	}
+
+	public List<RangerPolicy> getServicePolicies(final XXService xService) {
 		String serviceName = xService == null ? null : xService.getName();
 		Long   serviceId   = xService == null ? null : xService.getId();
 
@@ -112,9 +165,26 @@ public class RangerPolicyRetriever {
 		}
 
 		if(xService != null) {
-			RetrieverContext ctx = new RetrieverContext(xService);
+			if (txTemplate == null) {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction");
+				}
+				RetrieverContext ctx = new RetrieverContext(xService);
+				ret = ctx.getAllPolicies();
+			} else {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Retrieving policies in a new, read-only transaction");
+				}
 
-			ret = ctx.getAllPolicies();
+				PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService);
+				t.start();
+				try {
+					t.join();
+					ret = t.getPolicies();
+				} catch (InterruptedException ie) {
+					LOG.error("Failed to retrieve policies in a new, read-only thread.", ie);
+				}
+			}
 		} else {
 			if(LOG.isDebugEnabled()) {
 				LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter");

http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index 5550572..789068f 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -38,6 +38,10 @@ import org.apache.ranger.entity.*;
 import org.apache.ranger.plugin.model.*;
 import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
 import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
 
 public class RangerTagDBRetriever {
 	static final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class);
@@ -48,6 +52,9 @@ public class RangerTagDBRetriever {
 	private final XXService xService;
 	private final LookupCache lookupCache;
 
+	private final PlatformTransactionManager  txManager;
+	private final TransactionTemplate         txTemplate;
+
 	private List<RangerServiceResource> serviceResources;
 	private Map<Long, RangerTagDef> tagDefs;
 	private Map<Long, RangerTag> tags;
@@ -55,8 +62,15 @@ public class RangerTagDBRetriever {
 
 	private boolean filterForServicePlugin;
 
-	public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xService) {
+	public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) {
 		this.daoMgr = daoMgr;
+		this.txManager = txManager;
+		if (this.txManager != null) {
+			this.txTemplate = new TransactionTemplate(this.txManager);
+			this.txTemplate.setReadOnly(true);
+		} else {
+			this.txTemplate = null;
+		}
 		this.xService = xService;
 		this.lookupCache = new LookupCache();
 
@@ -70,14 +84,27 @@ public class RangerTagDBRetriever {
 			}
 
 			filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false);
-			TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService);
-			TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService);
-			TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService);
 
-			serviceResources = serviceResourceContext.getAllServiceResources();
-			tagDefs = tagDefContext.getAllTagDefs();
-			tags = tagContext.getAllTags();
-			tagResourceMaps = getAllTagResourceMaps();
+			if (this.txTemplate == null) {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Load Tags in the same thread and using an existing transaction");
+				}
+				if (initializeTagCache(xService) == false) {
+					LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction");
+				}
+			} else {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Load Tags in a separate thread and using a new transaction");
+				}
+
+				TagLoaderThread t = new TagLoaderThread(txTemplate, xService);
+				t.start();
+				try {
+					t.join();
+				} catch (InterruptedException ie) {
+					LOG.error("Failed to get Tags in a separate thread and using a new transaction", ie);
+				}
+			}
 
 			RangerPerfTracer.log(perf);
 
@@ -100,6 +127,30 @@ public class RangerTagDBRetriever {
 		return tags;
 	}
 
+	private boolean initializeTagCache(XXService xService) {
+		boolean ret;
+		try {
+			TagRetrieverServiceResourceContext  serviceResourceContext  = new TagRetrieverServiceResourceContext(xService);
+			TagRetrieverTagDefContext           tagDefContext           = new TagRetrieverTagDefContext(xService);
+			TagRetrieverTagContext              tagContext              = new TagRetrieverTagContext(xService);
+
+			serviceResources    = serviceResourceContext.getAllServiceResources();
+			tagDefs             = tagDefContext.getAllTagDefs();
+			tags                = tagContext.getAllTags();
+
+			tagResourceMaps     = getAllTagResourceMaps();
+
+			ret = true;
+		} catch (Exception ex) {
+			LOG.error("Failed to get tags for service:[" + xService.getName() + "]");
+			serviceResources    = null;
+			tagDefs             = null;
+			tags                = null;
+			tagResourceMaps     = null;
+			ret = false;
+		}
+		return ret;
+	}
 	private List<RangerTagResourceMap> getAllTagResourceMaps() {
 
 		List<XXTagResourceMap> xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId()) : daoMgr.getXXTagResourceMap().findByServiceId(xService.getId());
@@ -200,6 +251,38 @@ public class RangerTagDBRetriever {
 		}
 	}
 
+	private class TagLoaderThread extends Thread {
+		final TransactionTemplate txTemplate;
+		final XXService           xService;
+
+		TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+			this.txTemplate = txTemplate;
+			this.xService   = xService;
+		}
+
+		@Override
+		public void run() {
+			try {
+				 Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
+					@Override
+					public Boolean doInTransaction(TransactionStatus status) {
+						boolean ret = initializeTagCache(xService);
+						if (!ret) {
+							status.setRollbackOnly();
+							LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction");
+						}
+						return ret;
+					}
+				});
+				 if (LOG.isDebugEnabled()) {
+				 	LOG.debug("transaction result:[" + result +"]");
+				 }
+			} catch (Throwable ex) {
+				LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction", ex);
+			}
+		}
+	}
+
 	private class TagRetrieverServiceResourceContext {
 
 		final XXService service;

http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index e75ea68..0773616 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -2462,7 +2462,7 @@ public class ServiceDBStore extends AbstractServiceStore {
 			LOG.debug("==> ServiceDBStore.getServicePoliciesFromDb(" + service.getName() + ")");
 		}
 
-		RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr);
+		RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager);
 
 		List<RangerPolicy> ret = policyRetriever.getServicePolicies(service);
 

http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index 8341a73..d29df93 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -64,7 +64,9 @@ import org.apache.ranger.service.RangerTagResourceMapService;
 import org.apache.ranger.service.RangerTagService;
 import org.apache.ranger.service.RangerServiceResourceService;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
 
 import javax.annotation.PostConstruct;
 import javax.servlet.http.HttpServletResponse;
@@ -89,6 +91,10 @@ public class TagDBStore extends AbstractTagStore {
 	RangerDaoManager daoManager;
 
 	@Autowired
+	@Qualifier(value = "transactionManager")
+	PlatformTransactionManager txManager;
+
+	@Autowired
 	RESTErrorUtil errorUtil;
 
 	@Autowired
@@ -991,7 +997,7 @@ public class TagDBStore extends AbstractTagStore {
 			throw new Exception("service-def does not exist. id=" + xxService.getType());
 		}
 
-		RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService);
+		RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, txManager, xxService);
 
 		Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs();
 		Map<Long, RangerTag> tagMap = tagDBRetriever.getTags();