You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/10/28 16:59:43 UTC
[07/50] [abbrv] incubator-ranger git commit: RANGER-648: implemented
cleanup of policy-engine/context-enrichers to reclaim resouces like threads
when no longer needed
RANGER-648: implemented cleanup of policy-engine/context-enrichers to reclaim resouces like threads when no longer needed
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/a7453154
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/a7453154
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/a7453154
Branch: refs/heads/master
Commit: a7453154663bc2532894dda78ac43b24b9d27ef1
Parents: 0e47fbf
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Mon Sep 7 16:00:16 2015 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Sep 9 17:38:12 2015 -0700
----------------------------------------------------------------------
.../RangerAbstractContextEnricher.java | 42 +++-
.../RangerAdminTagRetriever.java | 32 +--
.../contextenricher/RangerContextEnricher.java | 21 +-
.../contextenricher/RangerTagEnricher.java | 195 ++++++++++++++++---
.../RangerTagFileStoreRetriever.java | 35 +---
.../contextenricher/RangerTagReceiver.java | 28 ---
.../contextenricher/RangerTagRetriever.java | 29 +--
.../plugin/policyengine/RangerPolicyEngine.java | 5 +
.../policyengine/RangerPolicyEngineImpl.java | 61 +++++-
.../policyengine/RangerPolicyRepository.java | 6 +-
.../ranger/plugin/service/RangerBasePlugin.java | 16 +-
11 files changed, 342 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java
index f869d58..c70fdb4 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java
@@ -40,7 +40,7 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri
protected RangerServiceDef serviceDef;
@Override
- public void setContextEnricherDef(RangerContextEnricherDef enricherDef) {
+ public void setEnricherDef(RangerContextEnricherDef enricherDef) {
this.enricherDef = enricherDef;
}
@@ -56,12 +56,12 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri
}
@Override
- public void setContextServiceName(String serviceName) {
+ public void setServiceName(String serviceName) {
this.serviceName = serviceName;
}
@Override
- public void setContextServiceDef(RangerServiceDef serviceDef) {
+ public void setServiceDef(RangerServiceDef serviceDef) {
this.serviceDef = serviceDef;
}
@@ -70,6 +70,42 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri
this.appId = appId;
}
+
+ @Override
+ public RangerContextEnricherDef getEnricherDef() {
+ return enricherDef;
+ }
+
+ @Override
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ @Override
+ public RangerServiceDef getServiceDef() {
+ return serviceDef;
+ }
+
+ @Override
+ public String getAppId() {
+ return appId;
+ }
+
+
+ @Override
+ public String getName() {
+ return enricherDef == null ? null : enricherDef.getName();
+ }
+
+ @Override
+ public boolean preCleanup() {
+ return true;
+ }
+
+ @Override
+ public void cleanup() {
+ }
+
public String getOption(String name) {
String ret = null;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
index 9db8f0a..7c5b378 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
@@ -36,7 +36,8 @@ public class RangerAdminTagRetriever extends RangerTagRetriever {
@Override
public void init(Map<String, String> options) {
- if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId) && tagReceiver != null) {
+
+ if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) {
String propertyPrefix = "ranger.plugin." + serviceDef.getName();
adminClient = RangerBasePlugin.createAdminClient(serviceName, appId, propertyPrefix);
@@ -47,37 +48,24 @@ public class RangerAdminTagRetriever extends RangerTagRetriever {
}
@Override
- public void retrieveTags() throws InterruptedException {
+ public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException {
+
+ ServiceTags serviceTags = null;
- if (adminClient != null && tagReceiver != null) {
- ServiceTags serviceTags = null;
+ if (adminClient != null) {
try {
serviceTags = adminClient.getServiceTagsIfUpdated(lastKnownVersion);
- }
- catch (InterruptedException interruptedException) {
+ } catch (InterruptedException interruptedException) {
LOG.error("Tag-retriever thread was interrupted");
throw interruptedException;
- }
- catch (ClosedByInterruptException closedByInterruptException) {
+ } catch (ClosedByInterruptException closedByInterruptException) {
LOG.error("Tag-retriever thread was interrupted while blocked on I/O");
throw new InterruptedException();
- }
- catch (Exception exception) {
+ } catch (Exception exception) {
LOG.error("RangerAdminTagRetriever.retrieveTags() - Error retrieving resources, exception=", exception);
}
-
- if (serviceTags != null) {
- tagReceiver.setServiceTags(serviceTags);
- LOG.info("RangerAdminTagRetriever.retrieveTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion());
- setLastKnownVersion(serviceTags.getTagVersion());
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("RangerAdminTagRetriever.retrieveTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion);
- }
- }
- } else {
- LOG.error("RangerAdminTagRetriever.retrieveTags() - No admin client to get tags from or no tag receiver to update tag-cache");
}
+ return serviceTags;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java
index 10fed69..c88cbf1 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java
@@ -25,14 +25,24 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerContextEnricherDef;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
public interface RangerContextEnricher {
- void setContextEnricherDef(RangerContextEnricherDef enricherDef);
+ void setEnricherDef(RangerContextEnricherDef enricherDef);
- void setContextServiceName(String serviceName);
+ void setServiceName(String serviceName);
- void setContextServiceDef(RangerServiceDef serviceDef);
+ void setServiceDef(RangerServiceDef serviceDef);
void setAppId(String appId);
+ RangerContextEnricherDef getEnricherDef();
+
+ String getServiceName();
+
+ RangerServiceDef getServiceDef();
+
+ String getAppId();
+
+ String getName();
+
//void setContextComponentServiceName(String componentServiceName);
//void setContextComponentServiceDef(RangerServiceDef componentServiceDef);
@@ -40,4 +50,9 @@ public interface RangerContextEnricher {
void init();
void enrich(RangerAccessRequest request);
+
+ boolean preCleanup();
+
+ void cleanup();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index 37732a6..6388c78 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -19,11 +19,14 @@
package org.apache.ranger.plugin.contextenricher;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
@@ -32,11 +35,12 @@ import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourc
import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
import org.apache.ranger.plugin.util.ServiceTags;
+import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-public class RangerTagEnricher extends RangerAbstractContextEnricher implements RangerTagReceiver {
+public class RangerTagEnricher extends RangerAbstractContextEnricher {
private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class);
public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval";
@@ -67,8 +71,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
if (StringUtils.isNotBlank(tagRetrieverClassName)) {
- cleanup();
-
try {
@SuppressWarnings("unchecked")
Class<RangerTagRetriever> tagRetriverClass = (Class<RangerTagRetriever>) Class.forName(tagRetrieverClassName);
@@ -86,21 +88,26 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
}
if (tagRetriever != null) {
+ String propertyPrefix = "ranger.plugin." + serviceDef.getName();
+ String cacheDir = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.cache.dir");
+ String cacheFilename = String.format("%s_%s_tag.json", appId, serviceName);
+ cacheFilename = cacheFilename.replace(File.separatorChar, '_');
+ cacheFilename = cacheFilename.replace(File.pathSeparatorChar, '_');
+
+ String cacheFile = cacheDir == null ? null : (cacheDir + File.separator + cacheFilename);
tagRetriever.setServiceName(serviceName);
tagRetriever.setServiceDef(serviceDef);
tagRetriever.setAppId(appId);
- tagRetriever.setLastKnownVersion(lastKnownVersion);
- tagRetriever.setTagReceiver(this);
tagRetriever.init(enricherDef.getEnricherOptions());
+ tagRefresher = new RangerTagRefresher(tagRetriever, this, lastKnownVersion, cacheFile, pollingIntervalMs);
+
try {
- tagRetriever.retrieveTags();
- } catch (Exception exception) {
- // Ignore
+ tagRefresher.populateTags();
+ } catch (Throwable exception) {
+ LOG.error("Exception when retrieving tag for the first time for this enricher", exception);
}
- tagRefresher = new RangerTagRefresher(tagRetriever, pollingIntervalMs);
-
tagRefresher.startRefresher();
}
} else {
@@ -112,14 +119,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
}
}
- public void cleanup() {
-
- if (tagRefresher != null) {
- tagRefresher.cleanup();
- tagRefresher = null;
- }
- }
-
@Override
public void enrich(RangerAccessRequest request) {
if (LOG.isDebugEnabled()) {
@@ -147,7 +146,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
}
}
- @Override
public void setServiceTags(final ServiceTags serviceTags) {
this.serviceTags = serviceTags;
this.lastKnownVersion = serviceTags.getTagVersion();
@@ -181,6 +179,25 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
}
+ @Override
+ public boolean preCleanup() {
+ boolean ret = true;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerTagEnricher.preCleanup()");
+ }
+
+ if (tagRefresher != null) {
+ tagRefresher.cleanup();
+ tagRefresher = null;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerTagEnricher.preCleanup() : result=" + ret);
+ }
+ return ret;
+ }
+
private List<RangerTag> findMatchingTags(final RangerAccessResource resource, final List<RangerServiceResourceMatcher> resourceMatchers) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerTagEnricher.findMatchingTags(" + resource + ")");
@@ -252,16 +269,30 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
private static final Log LOG = LogFactory.getLog(RangerTagRefresher.class);
private final RangerTagRetriever tagRetriever;
+ private final RangerTagEnricher tagEnricher;
+ private long lastKnownVersion = -1L;
private final long pollingIntervalMs;
+ private final String cacheFile;
+ private boolean hasProvidedTagsToReceiver = false;
+ private Gson gson;
+
final long getPollingIntervalMs() {
return pollingIntervalMs;
}
- RangerTagRefresher(RangerTagRetriever tagRetriever, long pollingIntervalMs) {
+ RangerTagRefresher(RangerTagRetriever tagRetriever, RangerTagEnricher tagEnricher, long lastKnownVersion, String cacheFile, long pollingIntervalMs) {
this.tagRetriever = tagRetriever;
+ this.tagEnricher = tagEnricher;
+ this.lastKnownVersion = lastKnownVersion;
+ this.cacheFile = cacheFile;
this.pollingIntervalMs = pollingIntervalMs;
+ try {
+ gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create();
+ } catch(Throwable excp) {
+ LOG.fatal("failed to create GsonBuilder object", excp);
+ }
}
@Override
@@ -275,7 +306,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
try {
- tagRetriever.retrieveTags();
+ populateTags();
if (pollingIntervalMs > 0) {
Thread.sleep(pollingIntervalMs);
@@ -293,8 +324,46 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
}
}
+ private void populateTags() throws InterruptedException {
+
+ if (tagEnricher != null) {
+ ServiceTags serviceTags = null;
+
+ serviceTags = tagRetriever.retrieveTags(lastKnownVersion);
+
+ if (serviceTags == null) {
+ if (!hasProvidedTagsToReceiver) {
+ serviceTags = loadFromCache();
+ }
+ } else {
+ saveToCache(serviceTags);
+ }
+
+ if (serviceTags != null) {
+ tagEnricher.setServiceTags(serviceTags);
+ LOG.info("RangerTagRefresher.populateTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion());
+ lastKnownVersion = serviceTags.getTagVersion();
+ hasProvidedTagsToReceiver = true;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("RangerTagRefresher.populateTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion);
+ }
+ }
+ } else {
+ LOG.error("RangerTagRefresher.populateTags() - no tag receiver to update tag-cache");
+ }
+ }
+
void cleanup() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerTagRefresher.cleanup()");
+ }
+
stopRefresher();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerTagRefresher.cleanup()");
+ }
}
final void startRefresher() {
@@ -317,6 +386,88 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher implements
}
}
}
- }
+
+ final ServiceTags loadFromCache() {
+ ServiceTags serviceTags = null;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").loadFromCache()");
+ }
+
+ File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile);
+
+ if (cacheFile != null && cacheFile.isFile() && cacheFile.canRead()) {
+ Reader reader = null;
+
+ try {
+ reader = new FileReader(cacheFile);
+
+ serviceTags = gson.fromJson(reader, ServiceTags.class);
+
+ if (serviceTags != null) {
+ if (!StringUtils.equals(tagEnricher.getServiceName(), serviceTags.getServiceName())) {
+ LOG.warn("ignoring unexpected serviceName '" + serviceTags.getServiceName() + "' in cache file '" + cacheFile.getAbsolutePath() + "'");
+
+ serviceTags.setServiceName(tagEnricher.getServiceName());
+ }
+ }
+ } catch (Exception excp) {
+ LOG.error("failed to load service-tags from cache file " + cacheFile.getAbsolutePath(), excp);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (Exception excp) {
+ LOG.error("error while closing opened cache file " + cacheFile.getAbsolutePath(), excp);
+ }
+ }
+ }
+ } else {
+ LOG.warn("cache file does not exist or not readble '" + (cacheFile == null ? null : cacheFile.getAbsolutePath()) + "'");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").loadFromCache()");
+ }
+
+ return serviceTags;
+ }
+
+ final void saveToCache(ServiceTags serviceTags) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").saveToCache()");
+ }
+
+ if (serviceTags != null) {
+ File cacheFile = StringUtils.isEmpty(this.cacheFile) ? null : new File(this.cacheFile);
+
+ if (cacheFile != null) {
+ Writer writer = null;
+
+ try {
+ writer = new FileWriter(cacheFile);
+
+ gson.toJson(serviceTags, writer);
+ } catch (Exception excp) {
+ LOG.error("failed to save service-tags to cache file '" + cacheFile.getAbsolutePath() + "'", excp);
+ } finally {
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (Exception excp) {
+ LOG.error("error while closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp);
+ }
+ }
+ }
+ }
+ } else {
+ LOG.info("service-tags is null. Nothing to save in cache");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerTagRetriever(serviceName=" + tagEnricher.getServiceName() + ").saveToCache()");
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java
index 0259bdf..1ee00d3 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java
@@ -34,12 +34,10 @@ public class RangerTagFileStoreRetriever extends RangerTagRetriever {
private TagStore tagStore;
- public RangerTagFileStoreRetriever() {
- }
-
@Override
public void init(Map<String, String> options) {
- if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId) && tagReceiver != null) {
+
+ if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) {
tagStore = TagFileStore.getInstance();
@@ -49,37 +47,24 @@ public class RangerTagFileStoreRetriever extends RangerTagRetriever {
}
@Override
- public void retrieveTags() throws InterruptedException {
+ public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException {
- if (tagStore != null && tagReceiver != null) {
- ServiceTags serviceTags = null;
+ ServiceTags serviceTags = null;
+
+ if (tagStore != null) {
try {
serviceTags = tagStore.getServiceTagsIfUpdated(serviceName, lastKnownVersion);
- }
- catch (InterruptedException interruptedException) {
+ } catch (InterruptedException interruptedException) {
LOG.error("Tag-retriever thread was interrupted");
throw interruptedException;
- }
- catch (ClosedByInterruptException closedByInterruptException) {
+ } catch (ClosedByInterruptException closedByInterruptException) {
LOG.error("Tag-retriever thread was interrupted while blocked on I/O");
throw new InterruptedException();
- }
- catch (Exception exception) {
+ } catch (Exception exception) {
LOG.error("RangerTagFileStoreRetriever.retrieveTags() - Error retrieving resources, exception=", exception);
}
-
- if (serviceTags != null) {
- tagReceiver.setServiceTags(serviceTags);
- LOG.info("RangerTagFileStoreRetriever.retrieveTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion());
- setLastKnownVersion(serviceTags.getTagVersion());
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("RangerTagFileStoreRetriever.retrieveTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion);
- }
- }
- } else {
- LOG.error("RangerTagFileStoreRetriever.retrieveTags() - No tag-store to get tags from or no tag receiver to update tag-cache...");
}
+ return serviceTags;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java
deleted file mode 100644
index 47db707..0000000
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ranger.plugin.contextenricher;
-
-import org.apache.ranger.plugin.util.ServiceTags;
-
-import java.util.List;
-
-public interface RangerTagReceiver {
- void setServiceTags(final ServiceTags serviceTags);
-}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
index 4967cbe..374e5c1 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java
@@ -19,21 +19,27 @@
package org.apache.ranger.plugin.contextenricher;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.util.ServiceTags;
+import java.io.*;
import java.util.Map;
public abstract class RangerTagRetriever {
+ private static final Log LOG = LogFactory.getLog(RangerTagRetriever.class);
protected String serviceName;
protected RangerServiceDef serviceDef;
protected String appId;
- protected long lastKnownVersion;
- protected RangerTagReceiver tagReceiver;
public abstract void init(Map<String, String> options);
- public abstract void retrieveTags() throws InterruptedException;
+ public abstract ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException;
public String getServiceName() {
return serviceName;
@@ -58,21 +64,4 @@ public abstract class RangerTagRetriever {
public void setAppId(String appId) {
this.appId = appId;
}
-
- public long getLastKnownVersion() {
- return lastKnownVersion;
- }
-
- public void setLastKnownVersion(long lastKnownVersion) {
- this.lastKnownVersion = lastKnownVersion;
- }
-
- public RangerTagReceiver getTagReceiver() {
- return tagReceiver;
- }
-
- public void setTagReceiver(RangerTagReceiver tagReceiver) {
- this.tagReceiver = tagReceiver;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
index 055a2db..29080b7 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
@@ -57,4 +57,9 @@ public interface RangerPolicyEngine {
RangerPolicy getExactMatchPolicy(RangerAccessResource resource);
List<RangerPolicy> getAllowedPolicies(String user, Set<String> userGroups, String accessType);
+
+ boolean preCleanup();
+
+ void cleanup();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
index 3cccde0..5d1140b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
@@ -40,7 +40,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
private final RangerPolicyRepository policyRepository;
private final RangerPolicyRepository tagPolicyRepository;
- private final List<RangerContextEnricher> allContextEnrichers;
+ private List<RangerContextEnricher> allContextEnrichers;
public RangerPolicyEngineImpl(String appId, ServicePolicies servicePolicies, RangerPolicyEngineOptions options) {
if (LOG.isDebugEnabled()) {
@@ -484,6 +484,65 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
}
@Override
+ public boolean preCleanup() {
+
+ boolean ret = true;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerPolicyEngineImpl.preCleanup()");
+ }
+
+ if (CollectionUtils.isNotEmpty(allContextEnrichers)) {
+ for (RangerContextEnricher contextEnricher : allContextEnrichers) {
+ boolean notReadyForCleanup = contextEnricher.preCleanup();
+ if (!notReadyForCleanup) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("contextEnricher.preCleanup() failed for contextEnricher=" + contextEnricher.getName());
+ }
+ ret = false;
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerPolicyEngineImpl.preCleanup() : result=" + ret);
+ }
+
+ return ret;
+ }
+
+ @Override
+ public void cleanup() {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerPolicyEngineImpl.cleanup()");
+ }
+
+ preCleanup();
+
+ if (CollectionUtils.isNotEmpty(allContextEnrichers)) {
+ for (RangerContextEnricher contextEnricher : allContextEnrichers) {
+ contextEnricher.cleanup();
+ }
+ }
+
+ this.allContextEnrichers = null;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerPolicyEngineImpl.cleanup()");
+ }
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ cleanup();
+ }
+ finally {
+ super.finalize();
+ }
+ }
+
+ @Override
public String toString( ) {
StringBuilder sb = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 8519860..37b1ea6 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -336,9 +336,9 @@ public class RangerPolicyRepository {
}
if(ret != null) {
- ret.setContextEnricherDef(enricherDef);
- ret.setContextServiceName(componentServiceName);
- ret.setContextServiceDef(componentServiceDef);
+ ret.setEnricherDef(enricherDef);
+ ret.setServiceName(componentServiceName);
+ ret.setServiceDef(componentServiceDef);
ret.setAppId(appId);
ret.init();
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a7453154/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index c857484..9d9987b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -30,6 +30,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.ranger.admin.client.RangerAdminClient;
import org.apache.ranger.admin.client.RangerAdminRESTClient;
import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.policyengine.*;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
@@ -107,21 +108,34 @@ public class RangerBasePlugin {
}
public void setPolicies(ServicePolicies policies) {
+ RangerPolicyEngine oldPolicyEngine = this.policyEngine;
+
RangerPolicyEngine policyEngine = new RangerPolicyEngineImpl(appId, policies, policyEngineOptions);
this.policyEngine = policyEngine;
+
+ if (oldPolicyEngine != null && !oldPolicyEngine.preCleanup()) {
+ LOG.error("preCleanup() failed on the previous policy engine instance !!");
+ }
}
public void cleanup() {
+
PolicyRefresher refresher = this.refresher;
+ RangerPolicyEngine policyEngine = this.policyEngine;
+
this.serviceName = null;
this.policyEngine = null;
this.refresher = null;
- if(refresher != null) {
+ if (refresher != null) {
refresher.stopRefresher();
}
+
+ if (policyEngine != null) {
+ policyEngine.cleanup();
+ }
}
public void setResultProcessor(RangerAccessResultProcessor resultProcessor) {