You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2024/09/09 22:04:00 UTC
(ranger) 01/04: RANGER-4922: Reduce time to find tags associated with multi-level resource
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.6
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 0447404215fe9f2aad310af4a189222c1cfe34bb
Author: Abhay Kulkarni <ak...@cloudera.com>
AuthorDate: Thu Sep 5 14:34:31 2024 -0700
RANGER-4922: Reduce time to find tags associated with multi-level resource
(cherry picked from commit 7aa06586039a53c58d0d9b07572168b0fd6ae0fa)
---
.../RangerServiceResourceMatcher.java | 12 +-
.../plugin/contextenricher/RangerTagEnricher.java | 145 ++++++++++++++++++---
.../model/validation/RangerServiceDefHelper.java | 30 +++++
.../validation/RangerZoneResourceMatcher.java | 4 +
.../plugin/policyengine/RangerResourceTrie.java | 103 ++++++++++-----
.../RangerAbstractPolicyEvaluator.java | 4 +
.../RangerResourceEvaluator.java | 2 +
.../util/RangerResourceEvaluatorsRetriever.java | 16 ++-
.../apache/ranger/plugin/util/ServiceDefUtil.java | 28 +++-
.../plugin/contextenricher/TestTagEnricher.java | 1 +
.../plugin/policyengine/TestPathResourceTrie.java | 3 +
distro/src/main/assembly/ranger-tools.xml | 4 +
ranger-tools/conf/logback-mem-sizing.xml | 21 +++
ranger-tools/pom.xml | 20 +++
14 files changed, 338 insertions(+), 55 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
index e696db518..44cf3dcdf 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
@@ -19,8 +19,9 @@
package org.apache.ranger.plugin.contextenricher;
+import org.apache.commons.lang.StringUtils;
import org.apache.ranger.plugin.model.RangerPolicy;
-import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope;
import org.apache.ranger.plugin.policyengine.RangerAccessResource;
@@ -38,12 +39,12 @@ public class RangerServiceResourceMatcher implements RangerResourceEvaluator {
private final RangerServiceResource serviceResource;
private final RangerPolicyResourceMatcher policyResourceMatcher;
- private RangerServiceDef.RangerResourceDef leafResourceDef;
+ private final RangerResourceDef leafResourceDef;
public RangerServiceResourceMatcher(final RangerServiceResource serviceResource, RangerPolicyResourceMatcher policyResourceMatcher) {
this.serviceResource = serviceResource;
this.policyResourceMatcher = policyResourceMatcher;
- this.leafResourceDef = ServiceDefUtil.getLeafResourceDef(policyResourceMatcher.getServiceDef(), getPolicyResource());
+ this.leafResourceDef = ServiceDefUtil.getLeafResourceDef(policyResourceMatcher.getServiceDef(), getPolicyResource(), true);
}
public RangerServiceResource getServiceResource() { return serviceResource; }
@@ -67,7 +68,7 @@ public class RangerServiceResourceMatcher implements RangerResourceEvaluator {
}
@Override
- public boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef) {
+ public boolean isAncestorOf(RangerResourceDef resourceDef) {
return ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(), leafResourceDef, resourceDef);
}
@@ -86,4 +87,7 @@ public class RangerServiceResourceMatcher implements RangerResourceEvaluator {
public String toString() {
return String.valueOf(getId());
}
+
+ @Override
+ public boolean isLeaf(String resourceName) { return StringUtils.equals(resourceName, leafResourceDef.getName()); }
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index a8fbc0215..92d2a7848 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -21,17 +21,28 @@ package org.apache.ranger.plugin.contextenricher;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.collections.Predicate;
import org.apache.commons.lang.StringUtils;
import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;
import org.apache.ranger.authorization.utils.JsonUtils;
import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
-import org.apache.ranger.plugin.policyengine.*;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceMatchingScope;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
+import org.apache.ranger.plugin.policyengine.RangerAccessResource;
+import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
+import org.apache.ranger.plugin.policyengine.RangerPluginContext;
+import org.apache.ranger.plugin.policyengine.RangerResourceTrie;
import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
+import org.apache.ranger.plugin.policyresourcematcher.RangerResourceEvaluator;
import org.apache.ranger.plugin.util.DownloadTrigger;
import org.apache.ranger.plugin.util.DownloaderTask;
import org.apache.ranger.plugin.service.RangerAuthContext;
@@ -42,6 +53,7 @@ import org.apache.ranger.plugin.util.RangerReadWriteLock;
import org.apache.ranger.plugin.util.RangerResourceEvaluatorsRetriever;
import org.apache.ranger.plugin.util.RangerServiceNotFoundException;
import org.apache.ranger.plugin.util.RangerServiceTagsDeltaUtil;
+import org.apache.ranger.plugin.util.ServiceDefUtil;
import org.apache.ranger.plugin.util.ServiceTags;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -83,12 +95,13 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
private EnrichedServiceTags enrichedServiceTags;
private boolean disableCacheIfServiceNotFound = true;
private boolean dedupStrings = true;
+ private Timer tagDownloadTimer;
+ private RangerServiceDefHelper serviceDefHelper;
private final BlockingQueue<DownloadTrigger> tagDownloadQueue = new LinkedBlockingQueue<>();
- private Timer tagDownloadTimer;
+ private final RangerReadWriteLock lock = new RangerReadWriteLock(false);
+ private final CachedResourceEvaluators cache = new CachedResourceEvaluators();
- private RangerServiceDefHelper serviceDefHelper;
- private RangerReadWriteLock lock = new RangerReadWriteLock(false);
@Override
public void init() {
@@ -347,6 +360,8 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
}
setEnrichedServiceTagsInPlugin();
+ cache.clearCache();
+
RangerPerfTracer.logAlways(perf);
}
@@ -451,7 +466,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
if (!disableTrieLookupPrefilter) {
serviceResourceTrie = new HashMap<>();
- for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
+ for (RangerResourceDef resourceDef : serviceDef.getResources()) {
serviceResourceTrie.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, resourceMatchers, getPolicyEngineOptions().optimizeTagTrieForRetrieval, getPolicyEngineOptions().optimizeTagTrieForSpace, null));
}
}
@@ -483,8 +498,8 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
RangerServiceResourceMatcher resourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies, getPluginContext());
if (resourceMatcher != null) {
- for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
- RangerPolicy.RangerPolicyResource policyResource = serviceResource.getResourceElements().get(resourceDef.getName());
+ for (RangerResourceDef resourceDef : serviceDef.getResources()) {
+ RangerPolicyResource policyResource = serviceResource.getResourceElements().get(resourceDef.getName());
RangerResourceTrie<RangerServiceResourceMatcher> trie = serviceResourceTrie.get(resourceDef.getName());
if (LOG.isDebugEnabled()) {
@@ -543,7 +558,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
RangerAccessResourceImpl accessResource = new RangerAccessResourceImpl();
- for (Map.Entry<String, RangerPolicy.RangerPolicyResource> entry : serviceResource.getResourceElements().entrySet()) {
+ for (Map.Entry<String, RangerPolicyResource> entry : serviceResource.getResourceElements().entrySet()) {
accessResource.setValue(entry.getKey(), entry.getValue().getValues());
}
@@ -583,7 +598,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
}
for (RangerServiceResourceMatcher matcher : oldMatchers) {
- for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
+ for (RangerResourceDef resourceDef : serviceDef.getResources()) {
String resourceDefName = resourceDef.getName();
RangerResourceTrie<RangerServiceResourceMatcher> trie = resourceTries.get(resourceDefName);
@@ -624,7 +639,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
if (isValidHierarchy == null) { // hierarchy not yet validated
isValidHierarchy = Boolean.FALSE;
- for (List<RangerServiceDef.RangerResourceDef> hierarchy : serviceDefHelper.getResourceHierarchies(policyType)) {
+ for (List<RangerResourceDef> hierarchy : serviceDefHelper.getResourceHierarchies(policyType)) {
if (serviceDefHelper.hierarchyHasAllResources(hierarchy, resourceKeys)) {
isValidHierarchy = Boolean.TRUE;
@@ -716,7 +731,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
if (request.isAccessTypeAny()) {
isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE;
- } else if (request.getResourceMatchingScope() == RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) {
+ } else if (request.getResourceMatchingScope() == ResourceMatchingScope.SELF_OR_DESCENDANTS) {
isMatched = matchType != RangerPolicyResourceMatcher.MatchType.NONE;
} else {
isMatched = matchType == RangerPolicyResourceMatcher.MatchType.SELF || matchType == RangerPolicyResourceMatcher.MatchType.SELF_AND_ALL_DESCENDANTS || matchType == RangerPolicyResourceMatcher.MatchType.ANCESTOR;
@@ -752,14 +767,42 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
return ret;
}
+ private static class CachedResourceEvaluators {
+ private final Map<String, Map<Map<String, ResourceElementMatchingScope>, Collection<RangerServiceResourceMatcher>>> cache = new HashMap<>();
+ private final RangerReadWriteLock cacheLock = new RangerReadWriteLock(true);
+
+ CachedResourceEvaluators() {}
+
+ Collection<RangerServiceResourceMatcher> getEvaluators(String resourceKey, Map<String, ResourceElementMatchingScope> scopes) {
+ Collection<RangerServiceResourceMatcher> ret;
+
+ try (RangerReadWriteLock.RangerLock ignored = cacheLock.getReadLock()) {
+ ret = cache.getOrDefault(resourceKey, Collections.emptyMap()).get(scopes);
+ }
+
+ return ret;
+ }
+
+ void cacheEvaluators(String resource, Map<String, ResourceElementMatchingScope> scopes, Collection<RangerServiceResourceMatcher> evaluators) {
+ try (RangerReadWriteLock.RangerLock ignored = cacheLock.getWriteLock()) {
+ cache.computeIfAbsent(resource, k -> new HashMap<>()).put(scopes, evaluators);
+ }
+ }
+
+ void clearCache() {
+ try (RangerReadWriteLock.RangerLock ignored = cacheLock.getWriteLock()) {
+ cache.clear();
+ }
+ }
+ }
+
private Collection<RangerServiceResourceMatcher> getEvaluators(RangerAccessRequest request, EnrichedServiceTags enrichedServiceTags) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerTagEnricher.getEvaluators(request=" + request + ")");
}
- Collection<RangerServiceResourceMatcher> ret;
-
- RangerAccessResource resource = request.getResource();
+ Collection<RangerServiceResourceMatcher> ret = null;
+ final RangerAccessResource resource = request.getResource();
final Map<String, RangerResourceTrie<RangerServiceResourceMatcher>> serviceResourceTrie = enrichedServiceTags.getServiceResourceTrie();
if (resource == null || resource.getKeys() == null || resource.getKeys().isEmpty() || serviceResourceTrie == null) {
@@ -771,7 +814,27 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerTagEnricher.getEvaluators(resource=" + resource.getAsString() + ")");
}
- ret = RangerResourceEvaluatorsRetriever.getEvaluators(serviceResourceTrie, resource.getAsMap(), request.getResourceElementMatchingScopes());
+ final Predicate predicate = excludeDescendantMatches(request) ? new SelfOrAncestorPredicate(serviceDefHelper.getResourceDef(resource.getLeafName())) : null;
+
+ if (predicate != null) {
+ ret = cache.getEvaluators(resource.getCacheKey(), request.getResourceElementMatchingScopes());
+ }
+
+ if (ret == null) {
+ ret = RangerResourceEvaluatorsRetriever.getEvaluators(serviceResourceTrie, resource.getAsMap(), request.getResourceElementMatchingScopes(), predicate);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Found [" + ret.size() + "] service-resource-matchers for service-resource [" + resource.getAsString() + "]");
+ }
+
+ if (predicate != null) {
+ cache.cacheEvaluators(resource.getCacheKey(), request.getResourceElementMatchingScopes(), ret);
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Found [" + ret.size() + "] service-resource-matchers for service-resource [" + resource.getAsString() + "] in the cache");
+ }
+ }
RangerPerfTracer.logAlways(perf);
}
@@ -786,6 +849,39 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
return ret;
}
+ private boolean excludeDescendantMatches(RangerAccessRequest request) {
+ final boolean ret;
+
+ if (request.isAccessTypeAny() || RangerAccessRequestUtil.getIsAnyAccessInContext(request.getContext())) {
+ ret = false;
+ } else {
+ RangerAccessResource resource = request.getResource();
+ String leafName = resource.getLeafName();
+
+ if (StringUtils.isNotEmpty(leafName)) {
+ RangerServiceDefHelper helper = new RangerServiceDefHelper(getServiceDef());
+ Set<List<RangerResourceDef>> hierarchies = helper.getResourceHierarchies(RangerPolicy.POLICY_TYPE_ACCESS, resource.getKeys());
+
+ // skip caching if the leaf of accessed resource is the deepest in the only applicable hierarchy
+ if (hierarchies.size() == 1) {
+ List<RangerResourceDef> theHierarchy = hierarchies.iterator().next();
+ RangerResourceDef leafOfHierarchy = theHierarchy.get(theHierarchy.size() - 1);
+
+ if (StringUtils.equals(leafOfHierarchy.getName(), leafName)) {
+ ret = false;
+ } else {
+ ret = true;
+ }
+ } else {
+ ret = true;
+ }
+ } else {
+ ret = false;
+ }
+ }
+ return ret;
+ }
+
private static Set<RangerTagForEval> getTagsForServiceResource(Date accessTime, final ServiceTags serviceTags, final RangerServiceResource serviceResource, final RangerPolicyResourceMatcher.MatchType matchType) {
Set<RangerTagForEval> ret = new HashSet<>();
@@ -1141,4 +1237,23 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
}
}
}
+
+ private static class SelfOrAncestorPredicate implements Predicate {
+ private final RangerServiceDef.RangerResourceDef leafResourceDef;
+
+ public SelfOrAncestorPredicate(RangerServiceDef.RangerResourceDef leafResourceDef) {
+ this.leafResourceDef = leafResourceDef;
+ }
+
+ @Override
+ public boolean evaluate(Object o) {
+ if (o instanceof RangerResourceEvaluator) {
+ RangerResourceEvaluator evaluator = (RangerResourceEvaluator) o;
+
+ return evaluator.isLeaf(leafResourceDef.getName()) || evaluator.isAncestorOf(leafResourceDef);
+ }
+
+ return false;
+ }
+ }
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
index d14c1ec78..aee6203d8 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
@@ -172,6 +172,14 @@ public class RangerServiceDefHelper {
_delegate.patchServiceDefWithDefaultValues();
}
+ public RangerResourceDef getResourceDef(String resourceName) {
+ return _delegate.getResourceDef(resourceName, RangerPolicy.POLICY_TYPE_ACCESS);
+ }
+
+ public RangerResourceDef getResourceDef(String resourceName, Integer policyType) {
+ return _delegate.getResourceDef(resourceName, policyType);
+ }
+
/**
* for a resource definition as follows:
*
@@ -428,6 +436,28 @@ public class RangerServiceDefHelper {
}
}
+ public RangerResourceDef getResourceDef(String resourceName, Integer policyType) {
+ RangerResourceDef ret = null;
+
+ if (policyType == null) {
+ policyType = RangerPolicy.POLICY_TYPE_ACCESS;
+ }
+
+ List<RangerResourceDef> resourceDefs = this.getResourceDefs(_serviceDef, policyType);
+
+ if (resourceDefs != null) {
+ for (RangerResourceDef resourceDef : resourceDefs) {
+ if (StringUtils.equals(resourceName, resourceDef.getName())) {
+ ret = resourceDef;
+
+ break;
+ }
+ }
+ }
+
+ return ret;
+ }
+
public Set<List<RangerResourceDef>> getResourceHierarchies(Integer policyType) {
if(policyType == null) {
policyType = RangerPolicy.POLICY_TYPE_ACCESS;
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
index 1a8a867a0..27ed49b0b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
@@ -19,6 +19,7 @@
package org.apache.ranger.plugin.model.validation;
+import org.apache.commons.lang.StringUtils;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.policyengine.RangerPluginContext;
@@ -109,6 +110,9 @@ public class RangerZoneResourceMatcher implements RangerResourceEvaluator {
return ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(), leafResourceDef, resourceDef);
}
+ @Override
+ public boolean isLeaf(String resourceName) { return StringUtils.equals(resourceName, leafResourceDef.getName()); }
+
@Override
public String toString() {
return "{security-zone-name:[" + securityZoneName + "], policyResource=[" + policyResource +"]}";
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
index abfc8f0aa..3a3a80e53 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.policyengine;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.Predicate;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
@@ -36,15 +37,7 @@ import org.apache.ranger.plugin.util.ServiceDefUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Stack;
+import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -194,7 +187,11 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
}
public Set<T> getEvaluatorsForResource(Object resource, ResourceElementMatchingScope scope) {
- EvalCollector<T> ret = new EvalCollector<>();
+ return getEvaluatorsForResource(resource, scope, (Predicate) null);
+ }
+
+ public Set<T> getEvaluatorsForResource(Object resource, ResourceElementMatchingScope scope, Predicate predicate) {
+ EvalCollector<T> ret = new EvalCollector<>(predicate);
traverse(resource, scope, ret);
@@ -202,7 +199,11 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
}
public Set<T> getEvaluatorsForResource(Object resource, ResourceElementMatchingScope scope, Set<T> filter) {
- EvalSubsetCollector<T> ret = new EvalSubsetCollector<>(filter);
+ return getEvaluatorsForResource(resource, scope, filter, null);
+ }
+
+ public Set<T> getEvaluatorsForResource(Object resource, ResourceElementMatchingScope scope, Set<T> filter, Predicate predicate) {
+ EvalSubsetCollector<T> ret = new EvalSubsetCollector<>(filter, predicate);
traverse(resource, scope, ret);
@@ -210,7 +211,11 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
}
public int getEvaluatorsCountForResource(Object resource, ResourceElementMatchingScope scope) {
- EvalCountCollector<T> ret = new EvalCountCollector<>();
+ return getEvaluatorsCountForResource(resource, scope, null);
+ }
+
+ public int getEvaluatorsCountForResource(Object resource, ResourceElementMatchingScope scope, Predicate predicate) {
+ EvalCountCollector<T> ret = new EvalCountCollector<>(predicate);
traverse(resource, scope, ret);
@@ -1321,18 +1326,20 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
}
}
- public interface TraverseMatchHandler<T> {
+ public interface TraverseMatchHandler<T extends RangerResourceEvaluator> {
// return: true - stop traverse, processing is complete
// false - continue traverse, processing is not complete yet
boolean process(Set<T> evaluators);
}
- public static class EvalCollector<T> implements TraverseMatchHandler<T> {
- private Set<T> result;
- private boolean isOwnedResult = false;
+ public static class EvalCollector<T extends RangerResourceEvaluator> implements TraverseMatchHandler<T> {
+ private final Predicate predicate;
+ private Set<T> result;
+ private boolean isOwnedResult = false;
- public EvalCollector() {
- this.result = null;
+ public EvalCollector(Predicate predicate) {
+ this.predicate = predicate;
+ this.result = null;
}
public Set<T> getResult() {
@@ -1343,7 +1350,18 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
public boolean process(Set<T> evaluators) {
if (evaluators != null && !evaluators.isEmpty()) {
if (result == null) {
- result = evaluators;
+ if (predicate == null) {
+ result = evaluators;
+ } else {
+ result = new HashSet<>();
+ isOwnedResult = true;
+
+ for (T evaluator : evaluators) {
+ if (predicate.evaluate(evaluator)) {
+ result.add(evaluator);
+ }
+ }
+ }
} else {
if (!isOwnedResult) {
result = new HashSet<>(result);
@@ -1351,7 +1369,15 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
isOwnedResult = true;
}
- result.addAll(evaluators);
+ if (predicate == null) {
+ result.addAll(evaluators);
+ } else {
+ for (T evaluator : evaluators) {
+ if (predicate.evaluate(evaluator)) {
+ result.add(evaluator);
+ }
+ }
+ }
}
}
@@ -1359,13 +1385,15 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
}
}
- public static class EvalSubsetCollector<T> implements TraverseMatchHandler<T> {
- private final Set<T> filter;
- private Set<T> result;
+ public static class EvalSubsetCollector<T extends RangerResourceEvaluator> implements TraverseMatchHandler<T> {
+ private final Predicate predicate;
+ private final Set<T> filter;
+ private Set<T> result;
- public EvalSubsetCollector(Set<T> filter) {
- this.filter = filter == null ? Collections.emptySet() : filter;
- this.result = null;
+ public EvalSubsetCollector(Set<T> filter, Predicate predicate) {
+ this.predicate = predicate;
+ this.filter = filter == null ? Collections.emptySet() : filter;
+ this.result = null;
}
public Set<T> getResult() {
@@ -1380,6 +1408,10 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
}
intersect(filter, evaluators, result);
+
+ if (predicate != null) {
+ result.removeIf(evaluator -> !predicate.evaluate(evaluator));
+ }
}
return result != null && (filter.size() == result.size()); // stop traverse once the result includes all entries in the filter
@@ -1393,8 +1425,13 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
}
}
- public static class EvalCountCollector<T> implements TraverseMatchHandler<T> {
- private int result = 0;
+ public static class EvalCountCollector<T extends RangerResourceEvaluator> implements TraverseMatchHandler<T> {
+ private final Predicate predicate;
+ private int result = 0;
+
+ public EvalCountCollector(Predicate predicate) {
+ this.predicate = predicate;
+ }
public int getResult() {
return result;
@@ -1403,7 +1440,15 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
@Override
public boolean process(Set<T> evaluators) {
if (evaluators != null) {
- result += evaluators.size();
+ if (predicate == null) {
+ result += evaluators.size();
+ } else {
+ for (T evaluator : evaluators) {
+ if (predicate.evaluate(evaluator)) {
+ result++;
+ }
+ }
+ }
}
return false; // continue traverse
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
index a72656287..d1a35a37a 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
@@ -20,6 +20,7 @@
package org.apache.ranger.plugin.policyevaluator;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
@@ -609,5 +610,8 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
return ServiceDefUtil.isAncestorOf(serviceDef, leafResourceDef, resourceDef);
}
}
+
+ @Override
+ public boolean isLeaf(String resourceName) { return StringUtils.equals(resourceName, leafResourceDef.getName()); }
}
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerResourceEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerResourceEvaluator.java
index 014bdd528..7544f1bdd 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerResourceEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerResourceEvaluator.java
@@ -36,4 +36,6 @@ public interface RangerResourceEvaluator {
RangerResourceMatcher getResourceMatcher(String resourceName);
boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef);
+
+ boolean isLeaf(String resourceName);
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceEvaluatorsRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceEvaluatorsRetriever.java
index 69cb4417d..7f39dc2a2 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceEvaluatorsRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceEvaluatorsRetriever.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.util;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.collections.Predicate;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope;
import org.apache.ranger.plugin.policyengine.RangerResourceTrie;
import org.apache.ranger.plugin.policyresourcematcher.RangerResourceEvaluator;
@@ -40,6 +41,10 @@ public class RangerResourceEvaluatorsRetriever {
}
public static <T extends RangerResourceEvaluator> Collection<T> getEvaluators(Map<String, RangerResourceTrie<T>> resourceTrie, Map<String, ?> resource, Map<String, ResourceElementMatchingScope> scopes) {
+ return getEvaluators(resourceTrie, resource, scopes, null);
+ }
+
+ public static <T extends RangerResourceEvaluator> Collection<T> getEvaluators(Map<String, RangerResourceTrie<T>> resourceTrie, Map<String, ?> resource, Map<String, ResourceElementMatchingScope> scopes, Predicate predicate) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyResourceEvaluatorsRetriever.getEvaluators(" + resource + ")");
}
@@ -65,13 +70,18 @@ public class RangerResourceEvaluatorsRetriever {
Object resourceValues = resource.get(resourceDefName);
- int evalCount = trie.getEvaluatorsCountForResource(resourceValues, scopes.get(resourceDefName));
+ int evalCount = trie.getEvaluatorsCountForResource(resourceValues, scopes.get(resourceDefName), predicate);
if (resourceWithMinEvals == null || (evalCount < minEvalCount)) {
resourceWithMinEvals = resourceDefName;
minEvalCount = evalCount;
}
}
+
+ if (minEvalCount == 0) {
+ resourceWithMinEvals = null;
+ ret = Collections.emptySet();
+ }
} else if (resourceKeys.size() == 1) { // skip getEvaluatorsCountForResource() when there is only one resource
String resourceKey = resourceKeys.iterator().next();
RangerResourceTrie<T> trie = resourceTrie.get(resourceKey);
@@ -84,7 +94,7 @@ public class RangerResourceEvaluatorsRetriever {
if (resourceWithMinEvals != null) {
RangerResourceTrie<T> trie = resourceTrie.get(resourceWithMinEvals);
- ret = trie.getEvaluatorsForResource(resource.get(resourceWithMinEvals), scopes.get(resourceWithMinEvals));
+ ret = trie.getEvaluatorsForResource(resource.get(resourceWithMinEvals), scopes.get(resourceWithMinEvals), predicate);
for (String resourceDefName : resourceKeys) {
if (resourceWithMinEvals.equals(resourceDefName)) {
@@ -97,7 +107,7 @@ public class RangerResourceEvaluatorsRetriever {
continue;
}
- Set<T> evaluators = trie.getEvaluatorsForResource(resource.get(resourceDefName), scopes.get(resourceDefName), ret);
+ Set<T> evaluators = trie.getEvaluatorsForResource(resource.get(resourceDefName), scopes.get(resourceDefName), ret, predicate);
if (CollectionUtils.isEmpty(evaluators)) {
ret = Collections.emptySet();
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
index e3690bf04..e43458644 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
@@ -42,6 +42,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.policyengine.RangerPluginContext;
import org.apache.ranger.plugin.policyengine.RangerRequestScriptEvaluator;
+import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher;
import org.apache.ranger.plugin.store.AbstractServiceStore;
import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
import org.apache.ranger.plugin.util.ServicePolicies.SecurityZoneInfo;
@@ -185,6 +186,10 @@ public class ServiceDefUtil {
}
public static RangerResourceDef getLeafResourceDef(RangerServiceDef serviceDef, Map<String, RangerPolicyResource> policyResource) {
+ return getLeafResourceDef(serviceDef, policyResource, false);
+ }
+
+ public static RangerResourceDef getLeafResourceDef(RangerServiceDef serviceDef, Map<String, RangerPolicyResource> policyResource, boolean excludeWildcardLeaves) {
RangerResourceDef ret = null;
if(serviceDef != null && policyResource != null) {
@@ -194,10 +199,12 @@ public class ServiceDefUtil {
RangerResourceDef resourceDef = ServiceDefUtil.getResourceDef(serviceDef, resource);
if (resourceDef != null && resourceDef.getLevel() != null) {
- if (ret == null) {
- ret = resourceDef;
- } else if(ret.getLevel() < resourceDef.getLevel()) {
- ret = resourceDef;
+ if (ret == null || ret.getLevel() < resourceDef.getLevel()) {
+ if (StringUtils.isEmpty(resourceDef.getParent())) {
+ ret = resourceDef;
+ } else if (!excludeWildcardLeaves || !hasWildcardValue(entry.getValue().getValues())) {
+ ret = resourceDef;
+ }
}
}
}
@@ -207,6 +214,19 @@ public class ServiceDefUtil {
return ret;
}
+ private static boolean hasWildcardValue(List<String> values) {
+ boolean ret = false;
+
+ for (String strValue : values) {
+ ret = StringUtils.equals(strValue, RangerAbstractResourceMatcher.WILDCARD_ASTERISK);
+ if (ret) {
+ break;
+ }
+ }
+
+ return ret;
+ }
+
public static boolean isAncestorOf(RangerServiceDef serviceDef, RangerResourceDef ancestor, RangerResourceDef descendant) {
boolean ret = false;
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
index 86985177a..3e0812a0f 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
@@ -128,6 +128,7 @@ public class TestTagEnricher {
for (TestData test : testCase.tests) {
RangerAccessRequestImpl request = new RangerAccessRequestImpl(test.resource, test.accessType, "testUser", null, null);
+ ((RangerMutableResource)request.getResource()).setServiceDef(testCase.serviceDef);
tagEnricher.enrich(request);
List<RangerTag> expected = test.result;
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPathResourceTrie.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPathResourceTrie.java
index d23509f86..24e1a80cc 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPathResourceTrie.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPathResourceTrie.java
@@ -191,6 +191,9 @@ public class TestPathResourceTrie {
return false;
}
+ @Override
+ public boolean isLeaf(String resourceName) { return true; }
+
@Override
public String toString() {
return "id=" + id + ", resource=" + policyResource;
diff --git a/distro/src/main/assembly/ranger-tools.xml b/distro/src/main/assembly/ranger-tools.xml
index 6893366d3..f12159ff9 100644
--- a/distro/src/main/assembly/ranger-tools.xml
+++ b/distro/src/main/assembly/ranger-tools.xml
@@ -72,6 +72,10 @@
<include>net.java.dev.jna:jna-platform:jar:${jna-platform.version}</include>
<include>org.slf4j:slf4j-api</include>
<include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include>
+ <include>org.slf4j:log4j-over-slf4j:jar:${slf4j.version}</include>
+ <include>ch.qos.logback:logback-classic:jar:${logback.version}</include>
+ <include>ch.qos.logback:logback-core:jar:${logback.version}</include>
+ <include>com.google.code.gson:gson:jar:${gson.version}</include>
</includes>
</dependencySet>
</dependencySets>
diff --git a/ranger-tools/conf/logback-mem-sizing.xml b/ranger-tools/conf/logback-mem-sizing.xml
index c3b601451..700fcb538 100644
--- a/ranger-tools/conf/logback-mem-sizing.xml
+++ b/ranger-tools/conf/logback-mem-sizing.xml
@@ -39,6 +39,27 @@
<level>INFO</level>
</filter>
</appender>
+ <appender name="perf_appender" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <!--See http://logback.qos.ch/manual/appenders.html#RollingFileAppender-->
+ <!--and http://logback.qos.ch/manual/appenders.html#TimeBasedRollingPolicy-->
+ <!--for further documentation-->
+ <file>./ranger-mem-sizing-perf.log</file>
+ <append>true</append>
+ <encoder>
+ <pattern>%d [%t] %msg%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>./ranger-mem-sizing-perf.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>15</maxHistory>
+ <cleanHistoryOnStart>true</cleanHistoryOnStart>
+ </rollingPolicy>
+ </appender>
+ <logger name="org.apache.ranger.perf.tagenricher.tags.retrieval" additivity="false" level="info">
+ <appender-ref ref="perf_appender"/>
+ </logger>
+ <logger name="org.apache.ranger.perf.policyresourcematcher.gethierarchy" additivity="false" level="info">
+ <appender-ref ref="perf_appender"/>
+ </logger>
<root level="INFO">
<appender-ref ref="file"/>
</root>
diff --git a/ranger-tools/pom.xml b/ranger-tools/pom.xml
index 0f6363a4d..7e05a14ec 100644
--- a/ranger-tools/pom.xml
+++ b/ranger-tools/pom.xml
@@ -60,6 +60,26 @@
<artifactId>chart</artifactId>
<version>2.2.0</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
<!-- Test -->
<dependency>