You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2016/02/11 02:25:03 UTC
incubator-ranger git commit: RANGER-836: Optimize policy download to
plugins
Repository: incubator-ranger
Updated Branches:
refs/heads/master 6ca739b04 -> 4e2e83eef
RANGER-836: Optimize policy download to plugins
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/4e2e83ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/4e2e83ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/4e2e83ee
Branch: refs/heads/master
Commit: 4e2e83eef660fae5287d4d1dc7bbea68015445ee
Parents: 6ca739b
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Mon Feb 1 12:07:41 2016 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Feb 10 17:24:34 2016 -0800
----------------------------------------------------------------------
.../policyengine/RangerPolicyEngineImpl.java | 24 ++
.../policyengine/RangerPolicyEngineOptions.java | 3 +-
.../policyengine/RangerPolicyRepository.java | 47 ++-
.../policyevaluator/RangerPolicyEvaluator.java | 2 +-
.../ranger/plugin/service/RangerBasePlugin.java | 2 +-
.../plugin/store/AbstractServiceStore.java | 11 +
.../ranger/plugin/store/ServiceStore.java | 4 +
.../apache/ranger/plugin/store/TagStore.java | 4 +
.../plugin/store/file/ServiceFileStore.java | 5 +-
.../ranger/plugin/store/file/TagFileStore.java | 119 +++++---
.../plugin/store/rest/ServiceRESTStore.java | 5 +
.../apache/ranger/plugin/util/ServiceTags.java | 1 -
.../org/apache/ranger/biz/ServiceDBStore.java | 141 +++++----
.../java/org/apache/ranger/biz/TagDBStore.java | 121 +++++---
.../common/RangerServicePoliciesCache.java | 301 ++++++++++++++++++
.../ranger/common/RangerServiceTagsCache.java | 306 +++++++++++++++++++
.../org/apache/ranger/rest/ServiceREST.java | 3 +-
17 files changed, 949 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
index 6222f47..022f5a7 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
@@ -23,6 +23,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.contextenricher.RangerContextEnricher;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerTag;
@@ -43,6 +44,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
private static final Log PERF_POLICYENGINE_AUDIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.audit");
private static final Log PERF_CONTEXTENRICHER_REQUEST_LOG = RangerPerfTracer.getPerfLogger("contextenricher.request");
+ private static final int MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR = 500;
+
private final RangerPolicyRepository policyRepository;
private final RangerPolicyRepository tagPolicyRepository;
@@ -63,6 +66,27 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
options = new RangerPolicyEngineOptions();
}
+ if(StringUtils.isBlank(options.evaluatorType) || StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO)) {
+
+ String serviceType = servicePolicies.getServiceDef().getName();
+ String propertyName = "ranger.plugin." + serviceType + ".policyengine.evaluator.auto.maximum.policycount.for.cache.type";
+
+ int thresholdForUsingOptimizedEvaluator = RangerConfiguration.getInstance().getInt(propertyName, MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR);
+
+ int servicePoliciesCount = servicePolicies.getPolicies().size() + (servicePolicies.getTagPolicies() != null ? servicePolicies.getTagPolicies().getPolicies().size() : 0);
+
+ if (servicePoliciesCount > thresholdForUsingOptimizedEvaluator) {
+ options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED;
+ } else {
+ options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED;
+ }
+ } else if (StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED)) {
+ options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED;
+ } else {
+ // All other cases
+ options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED;
+ }
+
policyRepository = new RangerPolicyRepository(appId, servicePolicies, options);
ServicePolicies.TagPolicies tagPolicies = servicePolicies.getTagPolicies();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
index dcd1b3c..805f5a5 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
@@ -23,9 +23,10 @@ import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
public class RangerPolicyEngineOptions {
- public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED;
+ public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO;
public boolean cacheAuditResults = true;
public boolean disableContextEnrichers = false;
public boolean disableCustomConditions = false;
public boolean disableTagPolicyEvaluation = true;
+ public boolean evaluateDelegateAdminOnly = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 641320f..e79b5cd 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -270,11 +270,48 @@ public class RangerPolicyRepository {
return policyItems;
}
+ public static boolean isDelegateAdminPolicy(RangerPolicy policy) {
+ boolean ret = false;
+
+ ret = hasDelegateAdminItems(policy.getPolicyItems())
+ || hasDelegateAdminItems(policy.getDenyPolicyItems())
+ || hasDelegateAdminItems(policy.getAllowExceptions())
+ || hasDelegateAdminItems(policy.getDenyExceptions());
+
+ return ret;
+ }
+
+ private static boolean hasDelegateAdminItems(List<RangerPolicy.RangerPolicyItem> items) {
+ boolean ret = false;
+
+ if (CollectionUtils.isNotEmpty(items)) {
+ for (RangerPolicy.RangerPolicyItem item : items) {
+ if(item.getDelegateAdmin()) {
+ ret = true;
+
+ break;
+ }
+ }
+ }
+ return ret;
+ }
+
+ private static boolean skipBuildingPolicyEvaluator(RangerPolicy policy, RangerPolicyEngineOptions options) {
+ boolean ret = false;
+ if (!policy.getIsEnabled()) {
+ ret = true;
+ } else if (options.evaluateDelegateAdminOnly && !isDelegateAdminPolicy(policy)) {
+ ret = true;
+ }
+ return ret;
+ }
+
private void init(RangerPolicyEngineOptions options) {
List<RangerPolicyEvaluator> policyEvaluators = new ArrayList<RangerPolicyEvaluator>();
+
for (RangerPolicy policy : policies) {
- if (!policy.getIsEnabled()) {
+ if (skipBuildingPolicyEvaluator(policy, options)) {
continue;
}
@@ -367,12 +404,10 @@ public class RangerPolicyRepository {
RangerPolicyEvaluator ret;
- if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_DEFAULT)) {
- ret = new RangerOptimizedPolicyEvaluator();
- } else if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED)) {
- ret = new RangerOptimizedPolicyEvaluator();
- } else {
+ if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED)) {
ret = new RangerCachedPolicyEvaluator();
+ } else {
+ ret = new RangerOptimizedPolicyEvaluator();
}
ret.init(policy, serviceDef, options);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
index e5f34a2..9cb90f4 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
@@ -32,7 +32,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResource;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> {
- public static final String EVALUATOR_TYPE_DEFAULT = "default";
+ public static final String EVALUATOR_TYPE_AUTO = "auto";
public static final String EVALUATOR_TYPE_OPTIMIZED = "optimized";
public static final String EVALUATOR_TYPE_CACHED = "cached";
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index 9d9987b..24749bb 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -95,7 +95,7 @@ public class RangerBasePlugin {
serviceName = RangerConfiguration.getInstance().get(propertyPrefix + ".service.name");
- policyEngineOptions.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED);
+ policyEngineOptions.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO);
policyEngineOptions.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true);
policyEngineOptions.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false);
policyEngineOptions.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
index e9aae60..0dead8a 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
@@ -137,6 +137,17 @@ public abstract class AbstractServiceStore implements ServiceStore {
return ret == null ? null : ret.getName();
}
+ @Override
+ public Long getServicePolicyVersion(String serviceName) {
+ RangerService service = null;
+ try {
+ service = getServiceByName(serviceName);
+ } catch (Exception exception) {
+ LOG.error("Failed to get service object for service:" + serviceName);
+ }
+ return service != null ? service.getPolicyVersion() : null;
+ }
+
protected void preCreate(RangerBaseModelObject obj) throws Exception {
obj.setId(0L);
if(obj.getGuid() == null) {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
index 8013262..febe640 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
@@ -89,6 +89,10 @@ public interface ServiceStore {
ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception;
+ Long getServicePolicyVersion(String serviceName);
+
+ ServicePolicies getServicePolicies(String serviceName) throws Exception;
+
RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId);
RangerPolicy getPolicyForVersionNumber(Long policyId, Integer versionNo);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
index 104459d..69bd628 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
@@ -124,5 +124,9 @@ public interface TagStore {
ServiceTags getServiceTagsIfUpdated(String serviceName, Long lastKnownVersion) throws Exception;
+ ServiceTags getServiceTags(String serviceName) throws Exception;
+
+ Long getTagVersion(String serviceName);
+
void deleteAllTagObjectsForService(String serviceName, boolean isResourePrivateTag) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java
index 8509f63..a68a6ed 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java
@@ -779,6 +779,10 @@ public class ServiceFileStore extends AbstractServiceStore {
return ret;
}
+ @Override
+ public ServicePolicies getServicePolicies(String serviceName) throws Exception {
+ return getServicePoliciesIfUpdated(serviceName, -1L);
+ }
private void handleServiceRename(RangerService service, String oldName) throws Exception {
List<RangerPolicy> policies = getAllPolicies();
@@ -787,7 +791,6 @@ public class ServiceFileStore extends AbstractServiceStore {
for(RangerPolicy policy : policies) {
if(StringUtils.equalsIgnoreCase(policy.getService(), oldName)) {
policy.setService(service.getName());
-
preUpdate(policy);
fileStoreUtil.saveToFile(policy, FILE_PREFIX_POLICY, service.getId(), true);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
index 14b0d73..6f15fd1 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
@@ -1076,9 +1076,7 @@ public class TagFileStore extends AbstractTagStore {
LOG.debug("==> TagFileStore.getServiceTagsIfUpdated(" + serviceName + ", " + lastKnownVersion + ")");
}
- ServiceTags ret = new ServiceTags();
- ret.setOp(ServiceTags.OP_ADD_OR_UPDATE);
- ret.setTagModel(ServiceTags.TAGMODEL_SHARED);
+ ServiceTags ret = null;
boolean tagsChanged = true;
@@ -1086,7 +1084,6 @@ public class TagFileStore extends AbstractTagStore {
try {
service = svcStore.getServiceByName(serviceName);
- ret.setServiceName(serviceName);
} catch (Exception exception) {
LOG.error("Cannot find service for serviceName=" + serviceName);
tagsChanged = false;
@@ -1094,55 +1091,21 @@ public class TagFileStore extends AbstractTagStore {
if (lastKnownVersion != null
&& service != null && service.getTagVersion() != null
- && lastKnownVersion.compareTo(service.getTagVersion()) >= 0 ) {
+ && lastKnownVersion.equals(service.getTagVersion())) {
tagsChanged = false;
}
if (tagsChanged) {
- SearchFilter filter = new SearchFilter();
-
- filter.setParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME, serviceName);
-
- List<RangerServiceResource> serviceResources = getServiceResources(filter);
- List<RangerServiceResource> filteredServiceResources = new ArrayList<RangerServiceResource>();
-
- Map<Long, RangerTag> tagsMap = new HashMap<Long, RangerTag>();
- Map<Long, List<Long>> resourceToTagIdsMap = new HashMap<Long, List<Long>>();
-
- for (RangerServiceResource serviceResource : serviceResources) {
- List<RangerTag> tagList = getTagsForServiceResourceObject(serviceResource);
-
- if (CollectionUtils.isNotEmpty(tagList)) {
- List<Long> tagIdList = new ArrayList<Long>();
- for (RangerTag tag : tagList) {
- tagsMap.put(tag.getId(), tag);
- tagIdList.add(tag.getId());
- }
- resourceToTagIdsMap.put(serviceResource.getId(), tagIdList);
- filteredServiceResources.add(serviceResource);
- }
- }
-
- ret.setServiceResources(filteredServiceResources);
- ret.setResourceToTagIds(resourceToTagIdsMap);
- ret.setTags(tagsMap);
-
- if (service != null && service.getTagVersion() != null) {
- ret.setTagVersion(service.getTagVersion());
- }
- if (service != null && service.getTagUpdateTime() != null) {
- ret.setTagUpdateTime(service.getTagUpdateTime());
- }
if (LOG.isDebugEnabled()) {
LOG.debug("Changes to tagVersion detected, tagVersion in service=" + (service == null ? null : service.getTagVersion())
+ ", Plugin-provided lastKnownVersion=" + lastKnownVersion);
}
+ ret = getServiceTags(serviceName);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("No changes to tagVersion detected, tagVersion in service=" + (service == null ? null : service.getTagVersion())
- + ", Plugin-provided lastKnownVersion=" + lastKnownVersion);
+ + ", Plugin-provided lastKnownVersion=" + lastKnownVersion);
}
- ret.setTagVersion(lastKnownVersion);
}
if (LOG.isDebugEnabled()) {
@@ -1150,7 +1113,81 @@ public class TagFileStore extends AbstractTagStore {
}
return ret;
+ }
+
+ @Override
+ public ServiceTags getServiceTags(String serviceName) throws Exception {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> TagFileStore.getServiceTags(" + serviceName + ")");
+ }
+
+ ServiceTags ret = new ServiceTags();
+
+ RangerService service = null;
+
+ try {
+ service = svcStore.getServiceByName(serviceName);
+ } catch (Exception exception) {
+ LOG.error("Cannot find service for serviceName=" + serviceName);
+ }
+
+ SearchFilter filter = new SearchFilter();
+
+ filter.setParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME, serviceName);
+
+ List<RangerServiceResource> serviceResources = getServiceResources(filter);
+ List<RangerServiceResource> filteredServiceResources = new ArrayList<RangerServiceResource>();
+
+ Map<Long, RangerTag> tagsMap = new HashMap<Long, RangerTag>();
+ Map<Long, List<Long>> resourceToTagIdsMap = new HashMap<Long, List<Long>>();
+
+ for (RangerServiceResource serviceResource : serviceResources) {
+ List<RangerTag> tagList = getTagsForServiceResourceObject(serviceResource);
+
+ if (CollectionUtils.isNotEmpty(tagList)) {
+ List<Long> tagIdList = new ArrayList<Long>();
+ for (RangerTag tag : tagList) {
+ tagsMap.put(tag.getId(), tag);
+ tagIdList.add(tag.getId());
+ }
+ resourceToTagIdsMap.put(serviceResource.getId(), tagIdList);
+ filteredServiceResources.add(serviceResource);
+ }
+ }
+
+ ret.setServiceName(serviceName);
+ ret.setServiceResources(filteredServiceResources);
+ ret.setResourceToTagIds(resourceToTagIdsMap);
+ ret.setTags(tagsMap);
+
+ if (service != null && service.getTagVersion() != null) {
+ ret.setTagVersion(service.getTagVersion());
+ }
+ if (service != null && service.getTagUpdateTime() != null) {
+ ret.setTagUpdateTime(service.getTagUpdateTime());
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== TagFileStore.getServiceTags(" + serviceName + "): " + ret);
+ }
+
+ return ret;
+
+ }
+
+ @Override
+ public Long getTagVersion(String serviceName) {
+
+ RangerService service = null;
+
+ try {
+ service = svcStore.getServiceByName(serviceName);
+ } catch (Exception exception) {
+ LOG.error("Cannot find service for serviceName=" + serviceName);
+ }
+ return service != null ? service.getTagVersion() : null;
}
private List<RangerTag> getTagsForServiceResourceObject(RangerServiceResource serviceResource) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java
index 5886cf5..2717551 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java
@@ -600,6 +600,11 @@ public class ServiceRESTStore extends AbstractServiceStore {
return ret;
}
+ @Override
+ public ServicePolicies getServicePolicies(String serviceName) throws Exception {
+ return getServicePoliciesIfUpdated(serviceName, -1L);
+ }
+
private WebResource createWebResource(String url) {
return createWebResource(url, null);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
index c3fb4cf..738a947 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
@@ -202,5 +202,4 @@ public class ServiceTags implements java.io.Serializable {
return sb;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index 38091bd..68e64c3 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -19,12 +19,7 @@
package org.apache.ranger.biz;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
import javax.annotation.PostConstruct;
@@ -34,17 +29,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
-import org.apache.ranger.common.AppConstants;
-import org.apache.ranger.common.ContextUtil;
-import org.apache.ranger.common.DateUtil;
-import org.apache.ranger.common.MessageEnums;
-import org.apache.ranger.common.PasswordUtils;
-import org.apache.ranger.common.RESTErrorUtil;
-import org.apache.ranger.common.RangerCommonEnums;
-import org.apache.ranger.common.RangerConstants;
-import org.apache.ranger.common.RangerFactory;
-import org.apache.ranger.common.StringUtil;
-import org.apache.ranger.common.UserSessionBase;
+import org.apache.ranger.common.*;
import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.db.XXAccessTypeDefDao;
import org.apache.ranger.db.XXAccessTypeDefGrantsDao;
@@ -192,7 +177,8 @@ public class ServiceDBStore extends AbstractServiceStore {
public static final String CONFIG_KEY_PASSWORD = "password";
private ServicePredicateUtil predicateUtil = null;
-
+
+
@Override
public void init() throws Exception {
if (LOG.isDebugEnabled()) {
@@ -1576,7 +1562,7 @@ public class ServiceDBStore extends AbstractServiceStore {
bizUtil.createTrxLog(trxLogList);
- LOG.info("Policy Deleted Successfully. PolicyName : " +policyName);
+ LOG.info("Policy Deleted Successfully. PolicyName : " + policyName);
}
@Override
@@ -1736,7 +1722,7 @@ public class ServiceDBStore extends AbstractServiceStore {
@Override
public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception {
- if(LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
LOG.debug("==> ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ")");
}
@@ -1744,60 +1730,100 @@ public class ServiceDBStore extends AbstractServiceStore {
XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName);
- if(serviceDbObj == null) {
+ if (serviceDbObj == null) {
throw new Exception("service does not exist. name=" + serviceName);
}
- if(lastKnownVersion == null || serviceDbObj.getPolicyVersion() == null || !lastKnownVersion.equals(serviceDbObj.getPolicyVersion())) {
- RangerServiceDef serviceDef = getServiceDef(serviceDbObj.getType());
+ if (lastKnownVersion == null || serviceDbObj.getPolicyVersion() == null || !lastKnownVersion.equals(serviceDbObj.getPolicyVersion())) {
+ ret = RangerServicePoliciesCache.getInstance().getServicePolicies(serviceName, this);
+ }
- if(serviceDef == null) {
- throw new Exception("service-def does not exist. id=" + serviceDbObj.getType());
- }
+ if (ret != null && lastKnownVersion != null && lastKnownVersion.equals(ret.getPolicyVersion())) {
+ // ServicePolicies are not changed
+ ret = null;
+ }
- List<RangerPolicy> policies = null;
- ServicePolicies.TagPolicies tagPolicies = null;
+ if (LOG.isDebugEnabled()) {
+ RangerServicePoliciesCache.getInstance().dump();
+ }
- if (serviceDbObj.getIsenabled()) {
- if (serviceDbObj.getTagService() != null) {
- XXService tagServiceDbObj = daoMgr.getXXService().getById(serviceDbObj.getTagService());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
+ }
+
+ return ret;
+ }
- if (tagServiceDbObj != null && tagServiceDbObj.getIsenabled()) {
- RangerServiceDef tagServiceDef = getServiceDef(tagServiceDbObj.getType());
+ @Override
+ public Long getServicePolicyVersion(String serviceName) {
- if (tagServiceDef == null) {
- throw new Exception("service-def does not exist. id=" + tagServiceDbObj.getType());
- }
+ XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName);
+
+ return serviceDbObj != null ? serviceDbObj.getPolicyVersion() : null;
+ }
- tagPolicies = new ServicePolicies.TagPolicies();
+ @Override
+ public ServicePolicies getServicePolicies(String serviceName) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ServiceDBStore.getServicePolicies(" + serviceName + ")");
+ }
- tagPolicies.setServiceId(tagServiceDbObj.getId());
- tagPolicies.setServiceName(tagServiceDbObj.getName());
- tagPolicies.setPolicyVersion(tagServiceDbObj.getPolicyVersion());
- tagPolicies.setPolicyUpdateTime(tagServiceDbObj.getPolicyUpdateTime());
- tagPolicies.setPolicies(getServicePolicies(tagServiceDbObj));
- tagPolicies.setServiceDef(tagServiceDef);
+ ServicePolicies ret = null;
+
+ XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName);
+
+ if(serviceDbObj == null) {
+ throw new Exception("service does not exist. name=" + serviceName);
+ }
+
+ RangerServiceDef serviceDef = getServiceDef(serviceDbObj.getType());
+
+ if (serviceDef == null) {
+ throw new Exception("service-def does not exist. id=" + serviceDbObj.getType());
+ }
+ List<RangerPolicy> policies = null;
+ ServicePolicies.TagPolicies tagPolicies = null;
+
+ if (serviceDbObj.getIsenabled()) {
+ if (serviceDbObj.getTagService() != null) {
+ XXService tagServiceDbObj = daoMgr.getXXService().getById(serviceDbObj.getTagService());
+
+ if (tagServiceDbObj != null && tagServiceDbObj.getIsenabled()) {
+ RangerServiceDef tagServiceDef = getServiceDef(tagServiceDbObj.getType());
+
+ if (tagServiceDef == null) {
+ throw new Exception("service-def does not exist. id=" + tagServiceDbObj.getType());
}
- }
- policies = getServicePolicies(serviceDbObj);
- } else {
- policies = new ArrayList<RangerPolicy>();
+ tagPolicies = new ServicePolicies.TagPolicies();
+
+ tagPolicies.setServiceId(tagServiceDbObj.getId());
+ tagPolicies.setServiceName(tagServiceDbObj.getName());
+ tagPolicies.setPolicyVersion(tagServiceDbObj.getPolicyVersion());
+ tagPolicies.setPolicyUpdateTime(tagServiceDbObj.getPolicyUpdateTime());
+ tagPolicies.setPolicies(getServicePolicies(tagServiceDbObj));
+ tagPolicies.setServiceDef(tagServiceDef);
+ }
}
- ret = new ServicePolicies();
+ policies = getServicePolicies(serviceDbObj);
- ret.setServiceId(serviceDbObj.getId());
- ret.setServiceName(serviceDbObj.getName());
- ret.setPolicyVersion(serviceDbObj.getPolicyVersion());
- ret.setPolicyUpdateTime(serviceDbObj.getPolicyUpdateTime());
- ret.setPolicies(policies);
- ret.setServiceDef(serviceDef);
- ret.setTagPolicies(tagPolicies);
+ } else {
+ policies = new ArrayList<RangerPolicy>();
}
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
+ ret = new ServicePolicies();
+
+ ret.setServiceId(serviceDbObj.getId());
+ ret.setServiceName(serviceDbObj.getName());
+ ret.setPolicyVersion(serviceDbObj.getPolicyVersion());
+ ret.setPolicyUpdateTime(serviceDbObj.getPolicyUpdateTime());
+ ret.setPolicies(policies);
+ ret.setServiceDef(serviceDef);
+ ret.setTagPolicies(tagPolicies);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceDBStore.getServicePolicies(" + serviceName + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
}
return ret;
@@ -2346,4 +2372,5 @@ public class ServiceDBStore extends AbstractServiceStore {
return ret;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index f89a434..bf56afa 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -32,6 +32,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.ranger.common.GUIDUtil;
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.RESTErrorUtil;
+import org.apache.ranger.common.RangerServiceTagsCache;
import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.entity.XXDBBase;
import org.apache.ranger.entity.XXResourceDef;
@@ -827,7 +828,7 @@ public class TagDBStore extends AbstractTagStore {
@Override
public ServiceTags getServiceTagsIfUpdated(String serviceName, Long lastKnownVersion) throws Exception {
- if(LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
LOG.debug("==> TagDBStore.getServiceTagsIfUpdated(" + serviceName + ", " + lastKnownVersion + ")");
}
@@ -835,65 +836,105 @@ public class TagDBStore extends AbstractTagStore {
XXService xxService = daoManager.getXXService().findByName(serviceName);
- if(xxService == null) {
+ if (xxService == null) {
throw new Exception("service does not exist. name=" + serviceName);
}
- if(lastKnownVersion == null || xxService.getTagVersion() == null || !lastKnownVersion.equals(xxService.getTagVersion())) {
- RangerServiceDef serviceDef = svcStore.getServiceDef(xxService.getType());
+ if (lastKnownVersion == null || xxService.getTagVersion() == null || !lastKnownVersion.equals(xxService.getTagVersion())) {
+ ret = RangerServiceTagsCache.getInstance().getServiceTags(serviceName, this);
+ }
- if(serviceDef == null) {
- throw new Exception("service-def does not exist. id=" + xxService.getType());
- }
+ if (ret != null && lastKnownVersion != null && lastKnownVersion.equals(ret.getTagVersion())) {
+ // ServiceTags are not changed
+ ret = null;
+ }
- RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService);
+ if (LOG.isDebugEnabled()) {
+ RangerServiceTagsCache.getInstance().dump();
+ }
- Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs();
- Map<Long, RangerTag> tagMap = tagDBRetriever.getTags();
- List<RangerServiceResource> resources = tagDBRetriever.getServiceResources();
- List<RangerTagResourceMap> tagResourceMaps = tagDBRetriever.getTagResourceMaps();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== TagDBStore.getServiceTagsIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getTags() == null) ? 0 : ret.getTags().size()));
+ }
- Map<Long, List<Long>> resourceToTagIds = new HashMap<Long, List<Long>>();
+ return ret;
+ }
- if(CollectionUtils.isNotEmpty(tagResourceMaps)) {
- Long resourceId = null;
- List<Long> tagIds = null;
+ @Override
+ public Long getTagVersion(String serviceName) {
+
+ XXService serviceDbObj = daoManager.getXXService().findByName(serviceName);
+
+ return serviceDbObj != null ? serviceDbObj.getTagVersion() : null;
+ }
+
+ @Override
+ public ServiceTags getServiceTags(String serviceName) throws Exception {
- for(RangerTagResourceMap tagResourceMap : tagResourceMaps) {
- if(! tagResourceMap.getResourceId().equals(resourceId)) {
- if(resourceId != null) {
- resourceToTagIds.put(resourceId, tagIds);
- }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> TagDBStore.getServiceTags(" + serviceName + ")");
+ }
+
+ ServiceTags ret = null;
- resourceId = tagResourceMap.getResourceId();
- tagIds = new ArrayList<Long>();
+ XXService xxService = daoManager.getXXService().findByName(serviceName);
+
+ if (xxService == null) {
+ throw new Exception("service does not exist. name=" + serviceName);
+ }
+
+ RangerServiceDef serviceDef = svcStore.getServiceDef(xxService.getType());
+
+ if (serviceDef == null) {
+ throw new Exception("service-def does not exist. id=" + xxService.getType());
+ }
+
+ RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService);
+
+ Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs();
+ Map<Long, RangerTag> tagMap = tagDBRetriever.getTags();
+ List<RangerServiceResource> resources = tagDBRetriever.getServiceResources();
+ List<RangerTagResourceMap> tagResourceMaps = tagDBRetriever.getTagResourceMaps();
+
+ Map<Long, List<Long>> resourceToTagIds = new HashMap<Long, List<Long>>();
+
+ if (CollectionUtils.isNotEmpty(tagResourceMaps)) {
+ Long resourceId = null;
+ List<Long> tagIds = null;
+
+ for (RangerTagResourceMap tagResourceMap : tagResourceMaps) {
+ if (!tagResourceMap.getResourceId().equals(resourceId)) {
+ if (resourceId != null) {
+ resourceToTagIds.put(resourceId, tagIds);
}
- tagIds.add(tagResourceMap.getTagId());
- }
-
- if(resourceId != null) {
- resourceToTagIds.put(resourceId, tagIds);
+ resourceId = tagResourceMap.getResourceId();
+ tagIds = new ArrayList<Long>();
}
+
+ tagIds.add(tagResourceMap.getTagId());
}
- ret = new ServiceTags();
- ret.setOp(ServiceTags.OP_ADD_OR_UPDATE);
- ret.setTagModel(ServiceTags.TAGMODEL_SHARED);
- ret.setServiceName(xxService.getName());
- ret.setTagVersion(xxService.getTagVersion());
- ret.setTagUpdateTime(xxService.getTagUpdateTime());
- ret.setTagDefinitions(tagDefMap);
- ret.setTags(tagMap);
- ret.setServiceResources(resources);
- ret.setResourceToTagIds(resourceToTagIds);
+ if (resourceId != null) {
+ resourceToTagIds.put(resourceId, tagIds);
+ }
}
+ ret = new ServiceTags();
+
+ ret.setServiceName(xxService.getName());
+ ret.setTagVersion(xxService.getTagVersion());
+ ret.setTagUpdateTime(xxService.getTagUpdateTime());
+ ret.setTagDefinitions(tagDefMap);
+ ret.setTags(tagMap);
+ ret.setServiceResources(resources);
+ ret.setResourceToTagIds(resourceToTagIds);
+
if(LOG.isDebugEnabled()) {
- LOG.debug("<== TagDBStore.getServiceTagsIfUpdated(" + serviceName + ", " + lastKnownVersion + ")");
+ LOG.debug("<== TagDBStore.getServiceTags(" + serviceName + ")");
}
-
return ret;
+
}
private List<XXTagAttributeDef> createTagAttributeDefs(Long tagDefId, List<RangerTagAttributeDef> tagAttrDefList) {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
new file mode 100644
index 0000000..f4afa3e
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.common;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.store.ServiceStore;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.util.ServicePolicies;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class RangerServicePoliciesCache {
+ private static final Log LOG = LogFactory.getLog(RangerServicePoliciesCache.class);
+
+ private static volatile RangerServicePoliciesCache sInstance = null;
+ private final boolean useServicePoliciesCache;
+ private final int waitTimeInSeconds;
+
+ private final Map<String, ServicePoliciesWrapper> servicePoliciesMap = new HashMap<String, ServicePoliciesWrapper>();
+
+ public static RangerServicePoliciesCache getInstance() {
+ if (sInstance == null) {
+ synchronized (RangerServicePoliciesCache.class) {
+ if (sInstance == null) {
+ sInstance = new RangerServicePoliciesCache();
+ }
+ }
+ }
+ return sInstance;
+ }
+
+ private RangerServicePoliciesCache() {
+ useServicePoliciesCache = RangerConfiguration.getInstance().getBoolean("ranger.admin.policy.download.usecache", true);
+ waitTimeInSeconds = RangerConfiguration.getInstance().getInt("ranger.admin.policy.download.cache.max.waittime.for.update", 20);
+ }
+
+ public void dump() {
+
+ if (useServicePoliciesCache) {
+ Set<String> serviceNames = null;
+
+ synchronized (this) {
+ serviceNames = servicePoliciesMap.keySet();
+ }
+
+ if (CollectionUtils.isNotEmpty(serviceNames)) {
+ ServicePoliciesWrapper cachedServicePoliciesWrapper = null;
+
+ for (String serviceName : serviceNames) {
+ cachedServicePoliciesWrapper = servicePoliciesMap.get(serviceName);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("serviceName:" + serviceName + ", Cached-MetaData:" + cachedServicePoliciesWrapper);
+ }
+ }
+ }
+ }
+ }
+
+ public ServicePolicies getServicePolicies(String serviceName) {
+
+ ServicePolicies ret = null;
+
+ if (useServicePoliciesCache && StringUtils.isNotBlank(serviceName)) {
+ ServicePoliciesWrapper cachedServicePoliciesWrapper = null;
+ synchronized (this) {
+ cachedServicePoliciesWrapper = servicePoliciesMap.get(serviceName);
+ }
+ if (cachedServicePoliciesWrapper != null) {
+ ret = cachedServicePoliciesWrapper.getServicePolicies();
+ }
+ }
+
+ return ret;
+ }
+
+ public ServicePolicies getServicePolicies(String serviceName, ServiceStore serviceStore) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerServicePoliciesCache.getServicePolicies(" + serviceName + ")");
+ }
+
+ ServicePolicies ret = null;
+
+ if (StringUtils.isNotBlank(serviceName)) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("useServicePoliciesCache=" + useServicePoliciesCache);
+ }
+
+ ServicePolicies servicePolicies = null;
+
+ if (!useServicePoliciesCache) {
+ if (serviceStore != null) {
+ try {
+ servicePolicies = serviceStore.getServicePolicies(serviceName);
+ } catch (Exception exception) {
+ LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies from service-store", exception);
+ }
+ } else {
+ LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies as service-store is null!");
+ }
+ } else {
+ ServicePoliciesWrapper servicePoliciesWrapper = null;
+
+ synchronized (this) {
+ servicePoliciesWrapper = servicePoliciesMap.get(serviceName);
+
+ if (servicePoliciesWrapper == null) {
+ servicePoliciesWrapper = new ServicePoliciesWrapper();
+ servicePoliciesMap.put(serviceName, servicePoliciesWrapper);
+ }
+ }
+
+ if (serviceStore != null) {
+ boolean refreshed = servicePoliciesWrapper.getLatestOrCached(serviceName, serviceStore);
+ LOG.info("tryRefreshFromStore returned " + refreshed);
+ } else {
+ LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies as service-store is null!");
+ }
+
+ servicePolicies = servicePoliciesWrapper.getServicePolicies();
+ }
+
+ ret = servicePolicies;
+
+ } else {
+ LOG.error("getServicePolicies() failed to get policies as serviceName is null or blank!");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerServicePoliciesCache.getServicePolicies(" + serviceName + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
+ }
+
+ return ret;
+ }
+
+ private class ServicePoliciesWrapper {
+ ServicePolicies servicePolicies;
+ Date updateTime = null;
+ long longestDbLoadTimeInMs = -1;
+
+ ReentrantLock lock = new ReentrantLock();
+
+ ServicePoliciesWrapper() {
+ servicePolicies = null;
+ }
+
+ ServicePolicies getServicePolicies() {
+ return servicePolicies;
+ }
+
+ Date getUpdateTime() {
+ return updateTime;
+ }
+
+ long getLongestDbLoadTimeInMs() {
+ return longestDbLoadTimeInMs;
+ }
+
+ boolean getLatestOrCached(String serviceName, ServiceStore serviceStore) {
+ boolean ret = false;
+
+ try {
+ ret = lock.tryLock(waitTimeInSeconds, TimeUnit.SECONDS);
+ if (ret) {
+ getLatest(serviceName, serviceStore);
+ }
+ } catch (InterruptedException exception) {
+ LOG.error("tryRefreshFromStore:lock got interrupted..", exception);
+ } finally {
+ if (ret) {
+ lock.unlock();
+ }
+ }
+
+ return ret;
+ }
+
+ void getLatest(String serviceName, ServiceStore serviceStore) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ServicePoliciesWrapper.getLatest(" + serviceName + ")");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Found ServicePolicies in-cache : " + (servicePolicies != null));
+ }
+
+ Long servicePolicyVersionInDb = serviceStore.getServicePolicyVersion(serviceName);
+
+
+ if (servicePolicies == null || servicePolicyVersionInDb == null || !servicePolicyVersionInDb.equals(servicePolicies.getPolicyVersion())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("loading servicePolicies from db ... cachedServicePoliciesVersion=" + (servicePolicies != null ? servicePolicies.getPolicyVersion() : null) + ", servicePolicyVersionInDb=" + servicePolicyVersionInDb);
+ }
+
+ ServicePolicies servicePoliciesFromDb = null;
+
+ try {
+ long startTimeMs = System.currentTimeMillis();
+
+ servicePoliciesFromDb = serviceStore.getServicePolicies(serviceName);
+
+ long dbLoadTime = System.currentTimeMillis() - startTimeMs;
+
+ if (dbLoadTime > longestDbLoadTimeInMs) {
+ longestDbLoadTimeInMs = dbLoadTime;
+ }
+ updateTime = new Date();
+ } catch (Exception exception) {
+ LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies from service-store", exception);
+ }
+
+ if (servicePoliciesFromDb != null) {
+ if (servicePoliciesFromDb.getPolicyVersion() == null) {
+ servicePoliciesFromDb.setPolicyVersion(0L);
+ }
+ servicePolicies = servicePoliciesFromDb;
+ pruneUnusedAttributes();
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServicePoliciesWrapper.getLatest(" + serviceName + ")");
+ }
+ }
+
+ private void pruneUnusedAttributes() {
+ if (servicePolicies != null) {
+ pruneUnusedPolicyAttributes(servicePolicies.getPolicies());
+ if (servicePolicies.getTagPolicies() != null) {
+ pruneUnusedPolicyAttributes(servicePolicies.getTagPolicies().getPolicies());
+ }
+ }
+ }
+
+ private void pruneUnusedPolicyAttributes(List<RangerPolicy> policies) {
+
+ // Null out attributes not required by plug-ins
+ if (CollectionUtils.isNotEmpty(policies)) {
+ for (RangerPolicy policy : policies) {
+ policy.setCreatedBy(null);
+ policy.setCreateTime(null);
+ policy.setUpdatedBy(null);
+ policy.setUpdateTime(null);
+ policy.setGuid(null);
+ policy.setName(null);
+ policy.setDescription(null);
+ policy.setResourceSignature(null);
+ }
+ }
+ }
+
+ StringBuilder toString(StringBuilder sb) {
+ sb.append("RangerServicePoliciesWrapper={");
+
+ sb.append("updateTime=").append(updateTime)
+ .append(", longestDbLoadTimeInMs=").append(longestDbLoadTimeInMs)
+ .append(", Service-Version:").append(servicePolicies != null ? servicePolicies.getPolicyVersion() : "null")
+ .append(", Number-Of-Policies:").append(servicePolicies != null ? servicePolicies.getPolicies().size() : 0);
+
+ sb.append("} ");
+
+ return sb;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ toString(sb);
+
+ return sb.toString();
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
new file mode 100644
index 0000000..e20cba8
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.common;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+import org.apache.ranger.plugin.model.RangerServiceResource;
+import org.apache.ranger.plugin.model.RangerTag;
+import org.apache.ranger.plugin.store.TagStore;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.util.ServiceTags;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+public class RangerServiceTagsCache {
+ private static final Log LOG = LogFactory.getLog(RangerServiceTagsCache.class);
+
+ private static volatile RangerServiceTagsCache sInstance = null;
+ private final boolean useServiceTagsCache;
+ private final int waitTimeInSeconds;
+
+ private final Map<String, ServiceTagsWrapper> serviceTagsMap = new HashMap<String, ServiceTagsWrapper>();
+
+ public static RangerServiceTagsCache getInstance() {
+ if (sInstance == null) {
+ synchronized (RangerServiceTagsCache.class) {
+ if (sInstance == null) {
+ sInstance = new RangerServiceTagsCache();
+ }
+ }
+ }
+ return sInstance;
+ }
+
+ private RangerServiceTagsCache() {
+ useServiceTagsCache = RangerConfiguration.getInstance().getBoolean("ranger.admin.tag.download.usecache", true);
+ waitTimeInSeconds = RangerConfiguration.getInstance().getInt("ranger.admin.tag.download.cache.max.waittime.for.update", 20);
+ }
+
+ public void dump() {
+
+ if (useServiceTagsCache) {
+ Set<String> serviceNames = null;
+
+ synchronized (this) {
+ serviceNames = serviceTagsMap.keySet();
+ }
+
+ if (CollectionUtils.isNotEmpty(serviceNames)) {
+ ServiceTagsWrapper cachedServiceTagsWrapper = null;
+
+ for (String serviceName : serviceNames) {
+ cachedServiceTagsWrapper = serviceTagsMap.get(serviceName);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("serviceName:" + serviceName + ", Cached-MetaData:" + cachedServiceTagsWrapper);
+ }
+ }
+ }
+ }
+ }
+
+ public ServiceTags getServiceTags(String serviceName) {
+
+ ServiceTags ret = null;
+
+ if (useServiceTagsCache && StringUtils.isNotBlank(serviceName)) {
+ ServiceTagsWrapper cachedServiceTagsWrapper = null;
+ synchronized (this) {
+ cachedServiceTagsWrapper = serviceTagsMap.get(serviceName);
+ }
+ if (cachedServiceTagsWrapper != null) {
+ ret = cachedServiceTagsWrapper.getServiceTags();
+ }
+ }
+
+ return ret;
+ }
+
+ public ServiceTags getServiceTags(String serviceName, TagStore tagStore) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerServiceTagsCache.getServiceTags(" + serviceName + ")");
+ }
+
+ ServiceTags ret = null;
+
+ if (StringUtils.isNotBlank(serviceName)) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("useServiceTagsCache=" + useServiceTagsCache);
+ }
+
+ ServiceTags serviceTags = null;
+
+ if (!useServiceTagsCache) {
+ if (tagStore != null) {
+ try {
+ serviceTags = tagStore.getServiceTags(serviceName);
+ } catch (Exception exception) {
+ LOG.error("getServiceTags(" + serviceName + "): failed to get latest tags from tag-store", exception);
+ }
+ } else {
+ LOG.error("getServiceTags(" + serviceName + "): failed to get latest tags as tag-store is null!");
+ }
+ } else {
+ ServiceTagsWrapper serviceTagsWrapper = null;
+
+ synchronized (this) {
+ serviceTagsWrapper = serviceTagsMap.get(serviceName);
+
+ if (serviceTagsWrapper == null) {
+ serviceTagsWrapper = new ServiceTagsWrapper();
+ serviceTagsMap.put(serviceName, serviceTagsWrapper);
+ }
+ }
+
+ if (tagStore != null) {
+ boolean refreshed = serviceTagsWrapper.getLatestOrCached(serviceName, tagStore);
+ LOG.info("tryRefreshFromStore returned " + refreshed);
+ } else {
+ LOG.error("getServiceTags(" + serviceName + "): failed to get latest tags as tag-store is null!");
+ }
+
+ serviceTags = serviceTagsWrapper.getServiceTags();
+ }
+
+ ret = serviceTags;
+
+ } else {
+ LOG.error("getServiceTags() failed to get tags as serviceName is null or blank!");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerServiceTagsCache.getServiceTags(" + serviceName + "): count=" + ((ret == null || ret.getTags() == null) ? 0 : ret.getTags().size()));
+ }
+
+ return ret;
+ }
+
+ private class ServiceTagsWrapper {
+ ServiceTags serviceTags;
+ Date updateTime = null;
+ long longestDbLoadTimeInMs = -1;
+
+ ReentrantLock lock = new ReentrantLock();
+
+ ServiceTagsWrapper() {
+ serviceTags = null;
+ }
+
+ ServiceTags getServiceTags() {
+ return serviceTags;
+ }
+
+ Date getUpdateTime() {
+ return updateTime;
+ }
+
+ long getLongestDbLoadTimeInMs() {
+ return longestDbLoadTimeInMs;
+ }
+
+ boolean getLatestOrCached(String serviceName, TagStore tagStore) {
+ boolean ret = false;
+
+ try {
+ ret = lock.tryLock(waitTimeInSeconds, TimeUnit.SECONDS);
+ if (ret) {
+ getLatest(serviceName, tagStore);
+ }
+ } catch (InterruptedException exception) {
+ LOG.error("tryRefreshFromStore:lock got interrupted..", exception);
+ } finally {
+ if (ret) {
+ lock.unlock();
+ }
+ }
+
+ return ret;
+ }
+
+ void getLatest(String serviceName, TagStore tagStore) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ServiceTagsWrapper.getLatest(" + serviceName + ")");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Found ServiceTags in-cache : " + (serviceTags != null));
+ }
+
+ Long tagVersionInDb = tagStore.getTagVersion(serviceName);
+
+
+ if (serviceTags == null || tagVersionInDb == null || !tagVersionInDb.equals(serviceTags.getTagVersion())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("loading serviceTags from db ... cachedServiceTagsVersion=" + (serviceTags != null ? serviceTags.getTagVersion() : null) + ", tagVersionInDb=" + tagVersionInDb);
+ }
+
+ ServiceTags serviceTagsFromDb = null;
+
+ try {
+ long startTimeMs = System.currentTimeMillis();
+
+ serviceTagsFromDb = tagStore.getServiceTags(serviceName);
+
+ long dbLoadTime = System.currentTimeMillis() - startTimeMs;
+
+ if (dbLoadTime > longestDbLoadTimeInMs) {
+ longestDbLoadTimeInMs = dbLoadTime;
+ }
+ updateTime = new Date();
+ } catch (Exception exception) {
+ LOG.error("getServiceTags(" + serviceName + "): failed to get latest tags from tag-store", exception);
+ }
+
+ if (serviceTagsFromDb != null) {
+ if (serviceTagsFromDb.getTagVersion() == null) {
+ serviceTagsFromDb.setTagVersion(0L);
+ }
+ serviceTags = serviceTagsFromDb;
+ pruneUnusedAttributes();
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceTagsWrapper.getLatest(" + serviceName + ")");
+ }
+ }
+
+ private void pruneUnusedAttributes() {
+ if (serviceTags != null) {
+ serviceTags.setOp(null);
+ serviceTags.setTagModel(null);
+ serviceTags.setTagUpdateTime(null);
+
+ serviceTags.setTagDefinitions(null);
+
+ for (Map.Entry<Long, RangerTag> entry : serviceTags.getTags().entrySet()) {
+ RangerTag tag = entry.getValue();
+ tag.setCreatedBy(null);
+ tag.setCreateTime(null);
+ tag.setUpdatedBy(null);
+ tag.setUpdateTime(null);
+ tag.setGuid(null);
+ }
+
+ for (RangerServiceResource serviceResource : serviceTags.getServiceResources()) {
+ serviceResource.setCreatedBy(null);
+ serviceResource.setCreateTime(null);
+ serviceResource.setUpdatedBy(null);
+ serviceResource.setUpdateTime(null);
+ serviceResource.setGuid(null);
+
+ serviceResource.setServiceName(null);
+ serviceResource.setResourceSignature(null);
+ }
+ }
+ }
+
+ StringBuilder toString(StringBuilder sb) {
+ sb.append("RangerServiceTagsWrapper={");
+
+ sb.append("updateTime=").append(updateTime)
+ .append(", longestDbLoadTimeInMs=").append(longestDbLoadTimeInMs)
+ .append(", Service-Version:").append(serviceTags != null ? serviceTags.getTagVersion() : "null")
+ .append(", Number-Of-Tags:").append(serviceTags != null ? serviceTags.getTags().size() : 0);
+
+ sb.append("} ");
+
+ return sb;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ toString(sb);
+
+ return sb.toString();
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
index 2464063..174a5ee 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
@@ -1927,10 +1927,11 @@ public class ServiceREST {
String propertyPrefix = "ranger.admin";
- options.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED);
+ options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED;
options.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", false);
options.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true);
options.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true);
+ options.evaluateDelegateAdminOnly = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.evaluate.delegateadmin.only", true);
RangerPolicyEngineCache.getInstance().setPolicyEngineOptions(options);;
}