You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ab...@apache.org on 2020/05/31 22:56:09 UTC
[ranger] branch master updated: RANGER-2839: Assorted improvements
for debugging and handling of thread terminations, clean-up of unused data,
etc
This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 8050020 RANGER-2839: Assorted improvements for debugging and handling of thread terminations, clean-up of unused data, etc
8050020 is described below
commit 8050020935618488ba1406f7568c2f1f654029ec
Author: Abhay Kulkarni <ab...@apache.org>
AuthorDate: Sun May 31 15:55:56 2020 -0700
RANGER-2839: Assorted improvements for debugging and handling of thread terminations, clean-up of unused data, etc
---
.../plugin/contextenricher/RangerTagEnricher.java | 61 +++++++++++++------
.../contextenricher/RangerUserStoreRefresher.java | 28 +++++++--
.../model/validation/RangerServiceDefHelper.java | 6 ++
.../ranger/plugin/policyengine/PolicyEngine.java | 36 ++++-------
.../policyengine/RangerPolicyEngineImpl.java | 8 +--
.../policyengine/RangerPolicyRepository.java | 71 ++++++++++++----------
.../ranger/plugin/service/RangerBasePlugin.java | 14 +++--
.../apache/ranger/plugin/util/PolicyRefresher.java | 40 +++++++++---
.../apache/ranger/biz/RangerPolicyAdminCache.java | 4 +-
.../apache/ranger/biz/RangerPolicyAdminImpl.java | 4 +-
10 files changed, 170 insertions(+), 102 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index 5d6a403..20ea511 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -133,6 +133,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
tagRetriever.init(enricherDef.getEnricherOptions());
tagRefresher = new RangerTagRefresher(tagRetriever, this, -1L, tagDownloadQueue, cacheFile);
+ LOG.info("Created RangerTagRefresher Thread(" + tagRefresher.getName() + ")");
try {
tagRefresher.populateTags();
@@ -212,7 +213,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
* combinations in a service-def will be much smaller than the number of service-resources.
*/
- static private class ResourceHierarchies {
+ static class ResourceHierarchies {
private final Map<Collection<String>, Boolean> accessHierarchies = new HashMap<>();
private final Map<Collection<String>, Boolean> dataMaskHierarchies = new HashMap<>();
private final Map<Collection<String>, Boolean> rowFilterHierarchies = new HashMap<>();
@@ -323,14 +324,18 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
super.preCleanup();
+ Timer tagDownloadTimer = this.tagDownloadTimer;
+ this.tagDownloadTimer = null;
+
if (tagDownloadTimer != null) {
tagDownloadTimer.cancel();
- tagDownloadTimer = null;
}
+ RangerTagRefresher tagRefresher = this.tagRefresher;
+ this.tagRefresher = null;
+
if (tagRefresher != null) {
tagRefresher.cleanup();
- tagRefresher = null;
}
if (LOG.isDebugEnabled()) {
@@ -525,7 +530,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
return ret;
}
- private RangerServiceResourceMatcher createRangerServiceResourceMatcher(RangerServiceResource serviceResource, RangerServiceDefHelper serviceDefHelper, ResourceHierarchies hierarchies) {
+ static public RangerServiceResourceMatcher createRangerServiceResourceMatcher(RangerServiceResource serviceResource, RangerServiceDefHelper serviceDefHelper, ResourceHierarchies hierarchies) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> createRangerServiceResourceMatcher(serviceResource=" + serviceResource + ")");
@@ -554,12 +559,12 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
if (isValidHierarchy) {
RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher();
- matcher.setServiceDef(this.serviceDef);
+ matcher.setServiceDef(serviceDefHelper.getServiceDef());
matcher.setPolicyResources(serviceResource.getResourceElements(), policyType);
if (LOG.isDebugEnabled()) {
LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource
- + ", serviceDef=" + this.serviceDef.getName() + ")");
+ + ", serviceDef=" + serviceDefHelper.getServiceDef() + ")");
}
matcher.setServiceDefHelper(serviceDefHelper);
@@ -833,6 +838,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
} catch(Throwable excp) {
LOG.fatal("failed to create GsonBuilder object", excp);
}
+ setName("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ")-" + getId());
}
public long getLastActivationTimeInMillis() {
@@ -851,22 +857,26 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
}
while (true) {
+ DownloadTrigger trigger = null;
try {
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_CONTEXTENRICHER_INIT_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_CONTEXTENRICHER_INIT_LOG, "RangerTagRefresher.populateTags(serviceName=" + tagRetriever.getServiceName() + ",lastKnownVersion=" + lastKnownVersion + ")");
+ perf = RangerPerfTracer.getPerfTracer(PERF_CONTEXTENRICHER_INIT_LOG, "RangerTagRefresher(" + getName() + ").populateTags(lastKnownVersion=" + lastKnownVersion + ")");
}
- DownloadTrigger trigger = tagDownloadQueue.take();
+ trigger = tagDownloadQueue.take();
populateTags();
- trigger.signalCompletion();
RangerPerfTracer.log(perf);
} catch (InterruptedException excp) {
- LOG.debug("RangerTagRefresher().run() : interrupted! Exiting thread", excp);
+ LOG.info("RangerTagRefresher(" + getName() + ").run(): Interrupted! Exiting thread", excp);
break;
+ } finally {
+ if (trigger != null) {
+ trigger.signalCompletion();
+ }
}
}
@@ -896,14 +906,14 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
if (serviceTags.getIsDelta() && serviceTags.getTagVersion() != -1L) {
saveToCache(tagEnricher.enrichedServiceTags.serviceTags);
}
- LOG.info("RangerTagRefresher.populateTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion="
+ LOG.info("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ").populateTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion="
+ (serviceTags.getTagVersion() == null ? -1L : serviceTags.getTagVersion()));
hasProvidedTagsToReceiver = true;
lastKnownVersion = serviceTags.getTagVersion() == null ? -1L : serviceTags.getTagVersion();
setLastActivationTimeInMillis(System.currentTimeMillis());
} else {
if (LOG.isDebugEnabled()) {
- LOG.debug("RangerTagRefresher.populateTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion);
+ LOG.debug("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ").populateTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion);
}
}
} catch (RangerServiceNotFoundException snfe) {
@@ -919,11 +929,11 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
} catch (InterruptedException interruptedException) {
throw interruptedException;
} catch (Exception e) {
- LOG.error("Encountered unexpected exception. Ignoring", e);
+ LOG.error("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ").populateTags(): Encountered unexpected exception. Ignoring", e);
}
} else {
- LOG.error("RangerTagRefresher.populateTags() - no tag receiver to update tag-cache");
+ LOG.error("RangerTagRefresher(serviceName=" + tagRetriever.getServiceName() + ".populateTags() - no tag receiver to update tag-cache");
}
}
@@ -943,7 +953,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
try {
super.start();
} catch (Exception excp) {
- LOG.error("RangerTagRefresher.startRetriever() - failed to start, exception=" + excp);
+ LOG.error("RangerTagRefresher(" + getName() + ").startRetriever(): Failed to start, exception=" + excp);
}
}
@@ -952,10 +962,21 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
if (super.isAlive()) {
super.interrupt();
- try {
- super.join();
- } catch (InterruptedException excp) {
- LOG.error("RangerTagRefresher(): error while waiting for thread to exit", excp);
+ boolean setInterrupted = false;
+ boolean isJoined = false;
+
+ while (!isJoined) {
+ try {
+ super.join();
+ isJoined = true;
+ } catch (InterruptedException excp) {
+ LOG.warn("RangerTagRefresher(" + getName() + ").stopRefresher(): Error while waiting for thread to exit", excp);
+ LOG.warn("Retrying Thread.join(). Current thread will be marked as 'interrupted' after Thread.join() returns");
+ setInterrupted = true;
+ }
+ }
+ if (setInterrupted) {
+ Thread.currentThread().interrupt();
}
}
}
@@ -1033,7 +1054,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
}
}
} else {
- LOG.info("service-tags is null. Nothing to save in cache");
+ LOG.info("service-tags is null for service=" + tagRetriever.getServiceName() + ". Nothing to save in cache");
}
if (LOG.isDebugEnabled()) {
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerUserStoreRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerUserStoreRefresher.java
index 867ac90..4a9a5b3 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerUserStoreRefresher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerUserStoreRefresher.java
@@ -75,6 +75,7 @@ public class RangerUserStoreRefresher extends Thread {
} catch(Throwable excp) {
LOG.fatal("failed to create GsonBuilder object", excp);
}
+ setName("RangerUserStoreRefresher(serviceName=" + userStoreRetriever.getServiceName() + ")-" + getId());
}
public long getLastActivationTimeInMillis() {
@@ -93,6 +94,7 @@ public class RangerUserStoreRefresher extends Thread {
}
while (true) {
+ DownloadTrigger trigger = null;
try {
RangerPerfTracer perf = null;
@@ -101,15 +103,18 @@ public class RangerUserStoreRefresher extends Thread {
perf = RangerPerfTracer.getPerfTracer(PERF_REFRESHER_INIT_LOG,
"RangerUserStoreRefresher.run(lastKnownVersion=" + lastKnownVersion + ")");
}
- DownloadTrigger trigger = userStoreDownloadQueue.take();
+ trigger = userStoreDownloadQueue.take();
populateUserStoreInfo();
- trigger.signalCompletion();
RangerPerfTracer.log(perf);
} catch (InterruptedException excp) {
LOG.debug("RangerUserStoreRefresher().run() : interrupted! Exiting thread", excp);
break;
+ } finally {
+ if (trigger != null) {
+ trigger.signalCompletion();
+ }
}
}
@@ -225,10 +230,21 @@ public class RangerUserStoreRefresher extends Thread {
if (super.isAlive()) {
super.interrupt();
- try {
- super.join();
- } catch (InterruptedException excp) {
- LOG.error("RangerUserStoreRefresher.stopRefresher(): error while waiting for thread to exit", excp);
+ boolean setInterrupted = false;
+ boolean isJoined = false;
+
+ while (!isJoined) {
+ try {
+ super.join();
+ isJoined = true;
+ } catch (InterruptedException excp) {
+ LOG.warn("RangerUserStoreRefresher(): Error while waiting for thread to exit", excp);
+ LOG.warn("Retrying Thread.join(), interrupted flag will be set after Thread.join() succeeds");
+ setInterrupted = true;
+ }
+ }
+ if (setInterrupted) {
+ Thread.currentThread().interrupt();
}
}
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
index 943b361..0d9a346 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
@@ -163,6 +163,10 @@ public class RangerServiceDefHelper {
_delegate = delegate;
}
+ public RangerServiceDef getServiceDef() {
+ return _delegate._serviceDef;
+ }
+
public void patchServiceDefWithDefaultValues() {
_delegate.patchServiceDefWithDefaultValues();
}
@@ -285,6 +289,7 @@ public class RangerServiceDefHelper {
* Not designed for public access. Package level only for testability.
*/
static class Delegate {
+ final RangerServiceDef _serviceDef;
final Map<Integer, Set<List<RangerResourceDef>>> _hierarchies = new HashMap<>();
final Date _serviceDefFreshnessDate;
final String _serviceName;
@@ -295,6 +300,7 @@ public class RangerServiceDefHelper {
public Delegate(RangerServiceDef serviceDef, boolean checkForCycles) {
// NOTE: we assume serviceDef, its name and update time are can never by null.
+ _serviceDef = serviceDef;
_serviceName = serviceDef.getName();
_serviceDefFreshnessDate = serviceDef.getUpdateTime();
_checkForCycles = checkForCycles;
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
index 297d02f..e6de06f 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
@@ -62,8 +62,6 @@ public class PolicyEngine {
private final Map<String, String> zoneTagServiceMap = new HashMap<>();
private boolean useForwardedIPAddress;
private String[] trustedProxyAddresses;
- private boolean isPreCleaned = false;
-
public boolean getUseForwardedIPAddress() {
return useForwardedIPAddress;
@@ -374,35 +372,27 @@ public class PolicyEngine {
return ret;
}
- public void preCleanup() {
+ public void preCleanup(boolean isForced) {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> PolicyEngine.preCleanup()");
+ LOG.debug("==> PolicyEngine.preCleanup(isForced=" + isForced + ")");
}
- if (!isPreCleaned) {
- if (policyRepository != null) {
- policyRepository.preCleanup();
- }
-
- if (tagPolicyRepository != null) {
- tagPolicyRepository.preCleanup();
- }
+ if (policyRepository != null) {
+ policyRepository.preCleanup(isForced);
+ }
- if (MapUtils.isNotEmpty(this.zonePolicyRepositories)) {
- for (Map.Entry<String, RangerPolicyRepository> entry : this.zonePolicyRepositories.entrySet()) {
- entry.getValue().preCleanup();
- }
- }
+ if (tagPolicyRepository != null) {
+ tagPolicyRepository.preCleanup(isForced);
+ }
- isPreCleaned = true;
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No preCleanup() necessary, policy-engine is already precleaned");
+ if (MapUtils.isNotEmpty(this.zonePolicyRepositories)) {
+ for (Map.Entry<String, RangerPolicyRepository> entry : this.zonePolicyRepositories.entrySet()) {
+ entry.getValue().preCleanup(isForced);
}
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== PolicyEngine.preCleanup()");
+ LOG.debug("<== PolicyEngine.preCleanup(isForced=" + isForced + ")");
}
}
@@ -793,7 +783,7 @@ public class PolicyEngine {
perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerPolicyEngine.cleanUp(hashCode=" + Integer.toHexString(System.identityHashCode(this)) + ")");
}
- preCleanup();
+ preCleanup(false);
if (policyRepository != null) {
policyRepository.cleanup();
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
index b594409..0930e2c 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
@@ -515,15 +515,15 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
return tagPolicyRepository == null ? ListUtils.EMPTY_LIST : tagPolicyRepository.getPolicies();
}
- public void releaseResources() {
+ public void releaseResources(boolean isForced) {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> RangerPolicyEngineImpl.releaseResources()");
+ LOG.debug("==> RangerPolicyEngineImpl.releaseResources(isForced=" + isForced + ")");
}
PolicyEngine policyEngine = this.policyEngine;
if (policyEngine != null) {
- policyEngine.preCleanup();
+ policyEngine.preCleanup(isForced);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Cannot preCleanup policy-engine as it is null!");
@@ -531,7 +531,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== RangerPolicyEngineImpl.releaseResources()");
+ LOG.debug("<== RangerPolicyEngineImpl.releaseResources(isForced=" + isForced + ")");
}
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 197c30f..3886eea 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -445,12 +445,14 @@ public class RangerPolicyRepository {
isContextEnrichersShared = true;
}
- void preCleanup() {
+ void preCleanup(boolean isForced) {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> preCleanup()");
+ LOG.debug("==> preCleanup(isForced=" + isForced + " )");
+ LOG.debug("Repository holds [" + (CollectionUtils.isEmpty(this.contextEnrichers) ? 0 : this.contextEnrichers.size()) + "] enrichers. isPreCleaned=" + isPreCleaned);
}
if (!isPreCleaned) {
- if (CollectionUtils.isNotEmpty(this.contextEnrichers) && !isContextEnrichersShared) {
+ if (CollectionUtils.isNotEmpty(this.contextEnrichers) && (!isContextEnrichersShared || isForced)) {
+ isPreCleaned = true;
if (LOG.isDebugEnabled()) {
LOG.debug("preCleaning context-enrichers");
}
@@ -459,17 +461,16 @@ public class RangerPolicyRepository {
}
} else {
if (LOG.isDebugEnabled()) {
- LOG.debug("No preCleaning of context-enrichers; Context-enrichers are " + (CollectionUtils.isNotEmpty(contextEnrichers) ? " not empty " : "empty ") + ", isContextEnrichersShared=" + isContextEnrichersShared);
+ LOG.debug("No preCleaning of context-enrichers; Context-enrichers are " + (CollectionUtils.isNotEmpty(contextEnrichers) ? " not empty " : "empty ") + ", isContextEnrichersShared=" + isContextEnrichersShared + ", isForced=" + isForced + ")");
}
}
- isPreCleaned = true;
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("preCleanup() already done. No need to do it again");
}
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== preCleanup()");
+ LOG.debug("<== preCleanup(isForced=" + isForced + " )");
}
}
@@ -477,7 +478,7 @@ public class RangerPolicyRepository {
if (LOG.isDebugEnabled()) {
LOG.debug("==> cleanup()");
}
- preCleanup();
+ preCleanup(false);
if (CollectionUtils.isNotEmpty(this.contextEnrichers) && !isContextEnrichersShared) {
for (RangerContextEnricher enricher : this.contextEnrichers) {
@@ -1025,39 +1026,45 @@ public class RangerPolicyRepository {
RangerContextEnricher ret = null;
- RangerPerfTracer perf = null;
+ Map<String, String> enricherDefOptions = enricherDef.getEnricherOptions();
- if(RangerPerfTracer.isPerfTraceEnabled(PERF_CONTEXTENRICHER_INIT_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_CONTEXTENRICHER_INIT_LOG, "RangerContextEnricher.init(appId=" + appId + ",name=" + enricherDef.getName() + ")");
- }
+ String isEnabledAsString = enricherDefOptions.get("IsEnabled");
+
+ if (!StringUtils.equalsIgnoreCase(isEnabledAsString, "false")) {
+ RangerPerfTracer perf = null;
+
+ if (RangerPerfTracer.isPerfTraceEnabled(PERF_CONTEXTENRICHER_INIT_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_CONTEXTENRICHER_INIT_LOG, "RangerContextEnricher.init(appId=" + appId + ",name=" + enricherDef.getName() + ")");
+ }
- String name = enricherDef != null ? enricherDef.getName() : null;
- String clsName = enricherDef != null ? enricherDef.getEnricher() : null;
+ String name = enricherDef != null ? enricherDef.getName() : null;
+ String clsName = enricherDef != null ? enricherDef.getEnricher() : null;
- if(! StringUtils.isEmpty(clsName)) {
- try {
- @SuppressWarnings("unchecked")
- Class<RangerContextEnricher> enricherClass = (Class<RangerContextEnricher>)Class.forName(clsName);
+ if (!StringUtils.isEmpty(clsName)) {
+ try {
+ @SuppressWarnings("unchecked")
+ Class<RangerContextEnricher> enricherClass = (Class<RangerContextEnricher>) Class.forName(clsName);
- ret = enricherClass.newInstance();
- } catch(Exception excp) {
- LOG.error("failed to instantiate context enricher '" + clsName + "' for '" + name + "'", excp);
+ ret = enricherClass.newInstance();
+ } catch (Exception excp) {
+ LOG.error("failed to instantiate context enricher '" + clsName + "' for '" + name + "'", excp);
+ }
}
- }
- if(ret != null) {
- ret.setEnricherDef(enricherDef);
- ret.setServiceName(componentServiceName);
- ret.setServiceDef(componentServiceDef);
- ret.setAppId(appId);
- if (ret instanceof RangerAbstractContextEnricher) {
- RangerAbstractContextEnricher abstractContextEnricher = (RangerAbstractContextEnricher) ret;
- abstractContextEnricher.setPluginContext(pluginContext);
+ if (ret != null) {
+ ret.setEnricherDef(enricherDef);
+ ret.setServiceName(componentServiceName);
+ ret.setServiceDef(componentServiceDef);
+ ret.setAppId(appId);
+ if (ret instanceof RangerAbstractContextEnricher) {
+ RangerAbstractContextEnricher abstractContextEnricher = (RangerAbstractContextEnricher) ret;
+ abstractContextEnricher.setPluginContext(pluginContext);
+ }
+ ret.init();
}
- ret.init();
- }
- RangerPerfTracer.log(perf);
+ RangerPerfTracer.log(perf);
+ }
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerPolicyRepository.buildContextEnricher(" + enricherDef + "): " + ret);
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index 236a4ab..a6ea48d 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -177,6 +177,7 @@ public class RangerBasePlugin {
}
refresher = new PolicyRefresher(this);
+ LOG.info("Created PolicyRefresher Thread(" + refresher.getName() + ")");
refresher.setDaemon(true);
refresher.startRefresher();
}
@@ -283,7 +284,7 @@ public class RangerBasePlugin {
pluginContext.notifyAuthContextChanged();
if (oldPolicyEngine != null) {
- ((RangerPolicyEngineImpl) oldPolicyEngine).releaseResources();
+ ((RangerPolicyEngineImpl) oldPolicyEngine).releaseResources(!isPolicyEngineShared);
}
if (this.refresher != null) {
@@ -305,13 +306,18 @@ public class RangerBasePlugin {
public void cleanup() {
PolicyRefresher refresher = this.refresher;
-
- this.policyEngine = null;
this.refresher = null;
+ RangerPolicyEngine policyEngine = this.policyEngine;
+ this.policyEngine = null;
+
if (refresher != null) {
refresher.stopRefresher();
}
+
+ if (policyEngine != null) {
+ ((RangerPolicyEngineImpl) policyEngine).releaseResources(true);
+ }
}
public void setResultProcessor(RangerAccessResultProcessor resultProcessor) {
@@ -614,7 +620,7 @@ public class RangerBasePlugin {
}
if (LOG.isDebugEnabled()) {
- LOG.info("<== refreshPoliciesAndTags()");
+ LOG.debug("<== refreshPoliciesAndTags()");
}
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
index 63a98f2..497485b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
@@ -96,6 +96,8 @@ public class PolicyRefresher extends Thread {
this.rolesProvider = new RangerRolesProvider(getServiceType(), appId, getServiceName(), rangerAdmin, cacheDir, pluginConfig);
this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000);
+ setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId());
+
if(LOG.isDebugEnabled()) {
LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").PolicyRefresher()");
}
@@ -161,21 +163,35 @@ public class PolicyRefresher extends Thread {
}
public void stopRefresher() {
- super.interrupt();
-
- try {
- super.join();
- } catch (InterruptedException excp) {
- LOG.warn("PolicyRefresher(serviceName=" + serviceName + "): error while waiting for thread to exit", excp);
- }
Timer policyDownloadTimer = this.policyDownloadTimer;
- this.policyDownloadTimer = null;
+ this.policyDownloadTimer = null;
if (policyDownloadTimer != null) {
policyDownloadTimer.cancel();
}
+
+ if (super.isAlive()) {
+ super.interrupt();
+
+ boolean setInterrupted = false;
+ boolean isJoined = false;
+
+ while (!isJoined) {
+ try {
+ super.join();
+ isJoined = true;
+ } catch (InterruptedException excp) {
+ LOG.warn("PolicyRefresher(serviceName=" + serviceName + "): error while waiting for thread to exit", excp);
+ LOG.warn("Retrying Thread.join(). Current thread will be marked as 'interrupted' after Thread.join() returns");
+ setInterrupted = true;
+ }
+ }
+ if (setInterrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
}
public void run() {
@@ -185,14 +201,18 @@ public class PolicyRefresher extends Thread {
}
while(true) {
+ DownloadTrigger trigger = null;
try {
- DownloadTrigger trigger = policyDownloadQueue.take();
+ trigger = policyDownloadQueue.take();
loadRoles();
loadPolicy();
- trigger.signalCompletion();
} catch(InterruptedException excp) {
LOG.info("PolicyRefresher(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp);
break;
+ } finally {
+ if (trigger != null) {
+ trigger.signalCompletion();
+ }
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyAdminCache.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyAdminCache.java
index df3c10d..681698d 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyAdminCache.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyAdminCache.java
@@ -123,6 +123,7 @@ public class RangerPolicyAdminCache {
synchronized(this) {
Boolean hasPolicyDeltas = RangerPolicyDeltaUtil.hasPolicyDeltas(policies);
+ boolean isPolicyEngineShared = false;
if (hasPolicyDeltas != null) {
if (hasPolicyDeltas.equals(Boolean.TRUE)) {
@@ -130,6 +131,7 @@ public class RangerPolicyAdminCache {
ret = RangerPolicyAdminImpl.getPolicyAdmin(oldPolicyAdmin, policies);
if (ret != null) {
ret.setRoles(roles);
+ isPolicyEngineShared = true;
}
} else {
LOG.error("Old policy engine is null! Cannot apply deltas without old policy engine!");
@@ -153,7 +155,7 @@ public class RangerPolicyAdminCache {
}
policyAdminCache.put(policies.getServiceName(), ret);
if (oldPolicyAdmin != null) {
- oldPolicyAdmin.releaseResources();
+ oldPolicyAdmin.releaseResources(!isPolicyEngineShared);
}
} else {
LOG.warn("Could not build new policy-engine. Continuing with the old policy-engine, if any");
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyAdminImpl.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyAdminImpl.java
index 1c63e94..1b5aa9e 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyAdminImpl.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyAdminImpl.java
@@ -307,9 +307,9 @@ public class RangerPolicyAdminImpl implements RangerPolicyAdmin {
return ret;
}
- void releaseResources() {
+ void releaseResources(boolean isForced) {
if (policyEngine != null) {
- policyEngine.preCleanup();
+ policyEngine.preCleanup(isForced);
}
}