You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ab...@apache.org on 2019/10/17 01:11:00 UTC
[ranger] branch master updated: RANGER-2510: Support for
Incremental tag updates to improve performance - part 3
This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 74839e3 RANGER-2510: Support for Incremental tag updates to improve performance - part 3
74839e3 is described below
commit 74839e3ed04125b312c09543953b100fb103a8bb
Author: Abhay Kulkarni <ab...@apache.org>
AuthorDate: Wed Oct 16 17:45:10 2019 -0700
RANGER-2510: Support for Incremental tag updates to improve performance - part 3
---
.../RangerServiceResourceMatcher.java | 13 ++----
.../plugin/contextenricher/RangerTagEnricher.java | 51 +++++++++++-----------
.../validation/RangerZoneResourceMatcher.java | 8 ++--
.../policyengine/RangerPolicyRepository.java | 12 +++--
.../RangerAbstractPolicyEvaluator.java | 8 ++--
.../RangerPolicyResourceEvaluator.java | 3 +-
.../ranger/plugin/util/RangerResourceTrie.java | 35 +++++++++------
.../apache/ranger/plugin/util/ServiceDefUtil.java | 18 ++++++++
8 files changed, 87 insertions(+), 61 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
index f9bbb12..7b02dd6 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
@@ -37,12 +37,12 @@ public class RangerServiceResourceMatcher implements RangerPolicyResourceEvaluat
private final RangerServiceResource serviceResource;
private final RangerPolicyResourceMatcher policyResourceMatcher;
- private final Integer leafResourceLevel;
+ private RangerServiceDef.RangerResourceDef leafResourceDef;
public RangerServiceResourceMatcher(final RangerServiceResource serviceResource, RangerPolicyResourceMatcher policyResourceMatcher) {
this.serviceResource = serviceResource;
this.policyResourceMatcher = policyResourceMatcher;
- this.leafResourceLevel = ServiceDefUtil.getLeafResourceLevel(getServiceDef(), getPolicyResource());
+ this.leafResourceDef = ServiceDefUtil.getLeafResourceDef(policyResourceMatcher.getServiceDef(), getPolicyResource());
}
public RangerServiceResource getServiceResource() { return serviceResource; }
@@ -66,18 +66,13 @@ public class RangerServiceResourceMatcher implements RangerPolicyResourceEvaluat
}
@Override
- public Integer getLeafResourceLevel() {
- return leafResourceLevel;
+ public boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef) {
+ return ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(), leafResourceDef, resourceDef);
}
-
-
public RangerPolicyResourceMatcher.MatchType getMatchType(RangerAccessResource requestedResource, Map<String, Object> evalContext) {
return policyResourceMatcher != null ? policyResourceMatcher.getMatchType(requestedResource, evalContext) : RangerPolicyResourceMatcher.MatchType.NONE;
}
- RangerServiceDef getServiceDef() {
- return policyResourceMatcher != null ? policyResourceMatcher.getServiceDef() : null;
- }
static class IdComparator implements Comparator<RangerServiceResourceMatcher>, Serializable {
@Override
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index 7434ec9..4e56f5c 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -467,41 +467,26 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
for (RangerServiceResource serviceResource : changedServiceResources) {
- if (removeOldServiceResource(serviceResource, resourceMatchers, serviceResourceTrie)) {
+ final boolean removedOldServiceResource = MapUtils.isEmpty(serviceResource.getResourceElements()) || removeOldServiceResource(serviceResource, resourceMatchers, serviceResourceTrie);
+ if (removedOldServiceResource) {
if (!StringUtils.isEmpty(serviceResource.getResourceSignature())) {
RangerServiceResourceMatcher resourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies);
if (resourceMatcher != null) {
- for (String resourceDefName : serviceResource.getResourceElements().keySet()) {
-
- RangerResourceTrie<RangerServiceResourceMatcher> trie = serviceResourceTrie.get(resourceDefName);
-
- if (trie == null) {
- List<RangerServiceDef.RangerResourceDef> resourceDefs = serviceDef.getResources();
- RangerServiceDef.RangerResourceDef found = null;
- for (RangerServiceDef.RangerResourceDef resourceDef : resourceDefs) {
- if (StringUtils.equals(resourceDef.getName(), resourceDefName)) {
- found = resourceDef;
- break;
- }
- }
- if (found != null) {
- trie = new RangerResourceTrie<>(found, new ArrayList<>());
- serviceResourceTrie.put(resourceDefName, trie);
- }
- }
+ for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
+
+ RangerResourceTrie<RangerServiceResourceMatcher> trie = serviceResourceTrie.get(resourceDef.getName());
if (trie != null) {
- trie.add(serviceResource.getResourceElements().get(resourceDefName), resourceMatcher);
+ trie.add(serviceResource.getResourceElements().get(resourceDef.getName()), resourceMatcher);
if (LOG.isDebugEnabled()) {
LOG.debug("Added resource-matcher for service-resource:[" + serviceResource + "]");
}
} else {
- LOG.error("Could not create resource-matcher for resource: [" + serviceResource + "]. Should NOT happen!!");
- LOG.error("Setting tagVersion to -1 to ensure that in the next download all tags are downloaded");
- isInError = true;
+ trie = new RangerResourceTrie<>(resourceDef, Collections.singletonList(resourceMatcher));
+ serviceResourceTrie.put(resourceDef.getName(), trie);
}
}
resourceMatchers.add(resourceMatcher);
@@ -526,6 +511,9 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
LOG.error("Error in processing tag-deltas. Will continue to use old tags");
deltas.setTagVersion(-1L);
} else {
+ for (Map.Entry<String, RangerResourceTrie<RangerServiceResourceMatcher>> entry : serviceResourceTrie.entrySet()) {
+ entry.getValue().wrapUpUpdate();
+ }
enrichedServiceTags = new EnrichedServiceTags(allServiceTags, resourceMatchers, serviceResourceTrie);
}
@@ -804,10 +792,13 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
private static Set<RangerTagForEval> getTagsForServiceResource(final ServiceTags serviceTags, final RangerServiceResource serviceResource, final RangerPolicyResourceMatcher.MatchType matchType) {
Set<RangerTagForEval> ret = new HashSet<>();
- final Long resourceId = serviceResource.getId();
-
+ final Long resourceId = serviceResource.getId();
final Map<Long, List<Long>> resourceToTagIds = serviceTags.getResourceToTagIds();
- final Map<Long, RangerTag> tags = serviceTags.getTags();
+ final Map<Long, RangerTag> tags = serviceTags.getTags();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Looking for tags for resource-id:[" + resourceId + "] in serviceTags:[" + serviceTags + "]");
+ }
if (resourceId != null && MapUtils.isNotEmpty(resourceToTagIds) && MapUtils.isNotEmpty(tags)) {
@@ -823,6 +814,14 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
ret.add(new RangerTagForEval(tag, matchType));
}
}
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No tags mapping found for resource:[" + resourceId + "]");
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("resourceId is null or resourceToTagTds mapping is null or tags mapping is null!");
}
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
index c7f5bc4..2b570f6 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
@@ -39,7 +39,7 @@ public class RangerZoneResourceMatcher implements RangerPolicyResourceEvaluator
private final String securityZoneName;
private final Map<String, RangerPolicy.RangerPolicyResource> policyResource;
private final RangerPolicyResourceMatcher policyResourceMatcher;
- private final Integer leafResourceLevel;
+ private RangerServiceDef.RangerResourceDef leafResourceDef;
public RangerZoneResourceMatcher(final String securityZoneName, final Map<String, RangerPolicy.RangerPolicyResource> policyResource, final RangerServiceDef serviceDef) {
@@ -77,7 +77,7 @@ public class RangerZoneResourceMatcher implements RangerPolicyResourceEvaluator
this.securityZoneName = securityZoneName;
this.policyResourceMatcher = matcher;
this.policyResource = policyResource;
- this.leafResourceLevel = ServiceDefUtil.getLeafResourceLevel(serviceDef, policyResource);
+ this.leafResourceDef = ServiceDefUtil.getLeafResourceDef(serviceDef, getPolicyResource());
}
public String getSecurityZoneName() { return securityZoneName; }
@@ -101,8 +101,8 @@ public class RangerZoneResourceMatcher implements RangerPolicyResourceEvaluator
}
@Override
- public Integer getLeafResourceLevel() {
- return leafResourceLevel;
+ public boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef) {
+ return ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(), leafResourceDef, resourceDef);
}
@Override
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index abc57df..065120f 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -1118,6 +1118,14 @@ class RangerPolicyRepository {
RangerResourceTrie<RangerPolicyEvaluator> trie = trieMap.get(resourceDefName);
+ if (trie == null) {
+ if (RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE == policyDeltaType || RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE == policyDeltaType) {
+ LOG.warn("policyDeltaType is not for POLICY_CREATE and trie for resourceDef:[" + resourceDefName + "] was null! Should not have happened!!");
+ }
+ trie = new RangerResourceTrie<>(resourceDef, new ArrayList<>(), RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR, true);
+ trieMap.put(resourceDefName, trie);
+ }
+
if (policyDeltaType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE) {
addEvaluatorToTrie(newEvaluator, trie, resourceDefName);
} else if (policyDeltaType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
@@ -1137,9 +1145,7 @@ class RangerPolicyRepository {
private void addEvaluatorToTrie(RangerPolicyEvaluator newEvaluator, RangerResourceTrie<RangerPolicyEvaluator> trie, String resourceDefName) {
if (newEvaluator != null) {
RangerPolicy.RangerPolicyResource resource = newEvaluator.getPolicyResource().get(resourceDefName);
- if (resource != null) {
- trie.add(resource, newEvaluator);
- }
+ trie.add(resource, newEvaluator);
}
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
index 4d6962f..689985c 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
@@ -37,7 +37,7 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
private RangerPolicy policy;
private RangerServiceDef serviceDef;
- private Integer leafResourceLevel;
+ private RangerServiceDef.RangerResourceDef leafResourceDef;
private int evalOrder;
protected long usageCount;
protected boolean usageCountMutable = true;
@@ -51,7 +51,7 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
this.policy = policy;
this.serviceDef = serviceDef;
- this.leafResourceLevel = ServiceDefUtil.getLeafResourceLevel(serviceDef, getPolicyResource());
+ this.leafResourceDef = ServiceDefUtil.getLeafResourceDef(serviceDef, getPolicyResource());
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerAbstractPolicyEvaluator.init(" + policy + ", " + serviceDef + ")");
@@ -84,8 +84,8 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
}
@Override
- public Integer getLeafResourceLevel() {
- return leafResourceLevel;
+ public boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef) {
+ return ServiceDefUtil.isAncestorOf(serviceDef, leafResourceDef, resourceDef);
}
public boolean hasAllow() {
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
index 7d43b4b..9da9fac 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.policyresourcematcher;
import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
import java.util.Map;
@@ -34,5 +35,5 @@ public interface RangerPolicyResourceEvaluator {
RangerResourceMatcher getResourceMatcher(String resourceName);
- Integer getLeafResourceLevel();
+ boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef);
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index 0cba882..1afd07d 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -50,7 +50,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
private static final String DEFAULT_WILDCARD_CHARS = "*?";
private static final String TRIE_BUILDER_THREAD_COUNT = "ranger.policyengine.trie.builder.thread.count";
- private final String resourceName;
+ private final RangerServiceDef.RangerResourceDef resourceDef;
private final boolean optIgnoreCase;
private final boolean optWildcard;
private final String wildcardChars;
@@ -99,7 +99,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
tokenReplaceSpecialChars += delimiterEscape;
}
- this.resourceName = resourceDef.getName();
+ this.resourceDef = resourceDef;
this.optIgnoreCase = RangerAbstractResourceMatcher.getOptionIgnoreCase(matcherOptions);
this.optWildcard = RangerAbstractResourceMatcher.getOptionWildCard(matcherOptions);
this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS + tokenReplaceSpecialChars : "" + tokenReplaceSpecialChars;
@@ -123,7 +123,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
if (TRACE_LOG.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
root.toString("", sb);
- TRACE_LOG.trace("Trie Dump from RangerResourceTrie.init(name=" + resourceName + "):\n{" + sb.toString() + "}");
+ TRACE_LOG.trace("Trie Dump from RangerResourceTrie.init(name=" + resourceDef.getName() + "):\n{" + sb.toString() + "}");
}
if(LOG.isDebugEnabled()) {
@@ -132,7 +132,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
public String getResourceName() {
- return resourceName;
+ return resourceDef.getName();
}
public List<T> getEvaluatorsForResource(Object resource) {
@@ -160,12 +160,18 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.add(name=" + resource + ")");
}
- if (resource.getIsExcludes()) {
- root.addWildcardEvaluator(evaluator);
+ if (resource == null) {
+ if (evaluator.isAncestorOf(resourceDef)) {
+ root.addWildcardEvaluator(evaluator);
+ }
} else {
- if (CollectionUtils.isNotEmpty(resource.getValues())) {
- for (String value : resource.getValues()) {
- insert(root, value, resource.getIsRecursive(), evaluator);
+ if (resource.getIsExcludes()) {
+ root.addWildcardEvaluator(evaluator);
+ } else {
+ if (CollectionUtils.isNotEmpty(resource.getValues())) {
+ for (String value : resource.getValues()) {
+ insert(root, value, resource.getIsRecursive(), evaluator);
+ }
}
}
}
@@ -372,10 +378,10 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.copyTrie(name=" + other.resourceName + ")");
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.copyTrie(name=" + other.resourceDef.getName() + ")");
}
- this.resourceName = other.resourceName;
+ this.resourceDef = other.resourceDef;
this.optIgnoreCase = other.optIgnoreCase;
this.optWildcard = other.optWildcard;
this.wildcardChars = other.wildcardChars;
@@ -391,7 +397,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
if (TRACE_LOG.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
root.toString("", sb);
- TRACE_LOG.trace("Trie Dump from RangerResourceTrie.copyTrie(name=" + other.resourceName + "):\n{" + sb.toString() + "}");
+ TRACE_LOG.trace("Trie Dump from RangerResourceTrie.copyTrie(name=" + other.resourceDef.getName() + "):\n{" + sb.toString() + "}");
}
}
@@ -410,6 +416,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
final boolean isMultiThreaded = builderThreadCount > 1;
final List<ResourceTrieBuilderThread> builderThreads;
final Map<Character, Integer> builderThreadMap;
+ final String resourceName = resourceDef.getName();
int lastUsedThreadIndex = 0;
if (isMultiThreaded) {
@@ -431,7 +438,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null;
if (policyResource == null) {
- if (evaluator.getLeafResourceLevel() != null && resourceDef.getLevel() != null && evaluator.getLeafResourceLevel() < resourceDef.getLevel()) {
+ if (evaluator.isAncestorOf(resourceDef)) {
ret.addWildcardEvaluator(evaluator);
}
@@ -763,7 +770,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
TrieData trieData = getTrieData();
- sb.append("resourceName=").append(resourceName);
+ sb.append("resourceName=").append(resourceDef.getName());
sb.append("; optIgnoreCase=").append(optIgnoreCase);
sb.append("; optWildcard=").append(optWildcard);
sb.append("; wildcardChars=").append(wildcardChars);
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
index f383241..f82f65f 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
@@ -128,6 +128,24 @@ public class ServiceDefUtil {
return ret;
}
+ public static boolean isAncestorOf(RangerServiceDef serviceDef, RangerResourceDef ancestor, RangerResourceDef descendant) {
+
+ boolean ret = false;
+
+ if (ancestor != null && descendant != null) {
+ final String ancestorName = ancestor.getName();
+
+ for (RangerResourceDef node = descendant; node != null; node = ServiceDefUtil.getResourceDef(serviceDef, node.getParent())) {
+ if (StringUtils.equalsIgnoreCase(ancestorName, node.getParent())) {
+ ret = true;
+ break;
+ }
+ }
+ }
+
+ return ret;
+ }
+
public static boolean isEmpty(RangerPolicy.RangerPolicyResource policyResource) {
boolean ret = true;
if (policyResource != null) {