You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2023/02/23 22:01:50 UTC
[ranger] branch ranger-2.4 updated: RANGER-4100: Efficient computation of the smallest set of evaluators returned by search of multiple Trie trees - #2
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.4
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.4 by this push:
new 0f930c905 RANGER-4100: Efficient computation of the smallest set of evaluators returned by search of multiple Trie trees - #2
0f930c905 is described below
commit 0f930c9053d080555544f5e515059e8d292cc22d
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Wed Feb 22 18:59:47 2023 -0800
RANGER-4100: Efficient computation of the smallest set of evaluators returned by search of multiple Trie trees - #2
(cherry picked from commit 93e888f7c0d12bd18426a49719fa996f8874d162)
---
.../policyengine/RangerPolicyRepository.java | 153 +++++++--------------
.../plugin/policyengine/RangerResourceTrie.java | 4 +-
.../plugin/policyengine/TestPolicyEngine.java | 6 +-
3 files changed, 52 insertions(+), 111 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 85a3afd01..fce2b1052 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -42,6 +42,7 @@ import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator.RangerPolicyResourceEvaluator;
import org.apache.ranger.plugin.store.AbstractServiceStore;
import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.apache.ranger.plugin.util.RangerResourceEvaluatorsRetriever;
import org.apache.ranger.plugin.util.ServiceDefUtil;
import org.apache.ranger.plugin.util.ServicePolicies;
import org.slf4j.Logger;
@@ -72,30 +73,30 @@ public class RangerPolicyRepository {
AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT
}
- private final String serviceName;
- private final String zoneName;
- private final String appId;
- private final RangerPolicyEngineOptions options;
- private final RangerPluginContext pluginContext;
- private final RangerServiceDef serviceDef;
- private /*final*/ List<RangerPolicy> policies;
- private final long policyVersion;
- private /*final*/ List<RangerContextEnricher> contextEnrichers;
- private final AuditModeEnum auditModeEnum;
- private final Map<String, AuditInfo> accessAuditCache;
- private final String componentServiceName;
- private final RangerServiceDef componentServiceDef;
- private final Map<String, RangerResourceTrie> policyResourceTrie;
- private final Map<String, RangerResourceTrie> dataMaskResourceTrie;
- private final Map<String, RangerResourceTrie> rowFilterResourceTrie;
- private final Map<String, RangerResourceTrie> auditFilterResourceTrie;
- private List<RangerPolicyEvaluator> policyEvaluators;
- private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators;
- private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators;
- private final List<RangerPolicyEvaluator> auditPolicyEvaluators;
- private Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
- private boolean isContextEnrichersShared = false;
- private boolean isPreCleaned = false;
+ private final String serviceName;
+ private final String zoneName;
+ private final String appId;
+ private final RangerPolicyEngineOptions options;
+ private final RangerPluginContext pluginContext;
+ private final RangerServiceDef serviceDef;
+ private /*final*/ List<RangerPolicy> policies;
+ private final long policyVersion;
+ private /*final*/ List<RangerContextEnricher> contextEnrichers;
+ private final AuditModeEnum auditModeEnum;
+ private final Map<String, AuditInfo> accessAuditCache;
+ private final String componentServiceName;
+ private final RangerServiceDef componentServiceDef;
+ private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> policyResourceTrie;
+ private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> dataMaskResourceTrie;
+ private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> rowFilterResourceTrie;
+ private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> auditFilterResourceTrie;
+ private List<RangerPolicyEvaluator> policyEvaluators;
+ private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators;
+ private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators;
+ private final List<RangerPolicyEvaluator> auditPolicyEvaluators;
+ private Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
+ private boolean isContextEnrichersShared = false;
+ private boolean isPreCleaned = false;
RangerPolicyRepository(final RangerPolicyRepository other, final List<RangerPolicyDelta> deltas, long policyVersion) {
this.serviceName = other.serviceName;
@@ -117,8 +118,8 @@ public class RangerPolicyRepository {
if (other.policyResourceTrie != null) {
this.policyResourceTrie = new HashMap<>();
- for (Map.Entry<String, RangerResourceTrie> entry : other.policyResourceTrie.entrySet()) {
- policyResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
+ for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.policyResourceTrie.entrySet()) {
+ policyResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
}
} else {
this.policyResourceTrie = null;
@@ -127,8 +128,8 @@ public class RangerPolicyRepository {
if (other.dataMaskResourceTrie != null) {
this.dataMaskResourceTrie = new HashMap<>();
- for (Map.Entry<String, RangerResourceTrie> entry : other.dataMaskResourceTrie.entrySet()) {
- dataMaskResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
+ for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.dataMaskResourceTrie.entrySet()) {
+ dataMaskResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
}
} else {
this.dataMaskResourceTrie = null;
@@ -137,8 +138,8 @@ public class RangerPolicyRepository {
if (other.rowFilterResourceTrie != null) {
this.rowFilterResourceTrie = new HashMap<>();
- for (Map.Entry<String, RangerResourceTrie> entry : other.rowFilterResourceTrie.entrySet()) {
- rowFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
+ for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.rowFilterResourceTrie.entrySet()) {
+ rowFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
}
} else {
this.rowFilterResourceTrie = null;
@@ -147,8 +148,8 @@ public class RangerPolicyRepository {
if (other.auditFilterResourceTrie != null) {
this.auditFilterResourceTrie = new HashMap<>();
- for (Map.Entry<String, RangerResourceTrie> entry : other.auditFilterResourceTrie.entrySet()) {
- auditFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
+ for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.auditFilterResourceTrie.entrySet()) {
+ auditFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
}
} else {
this.auditFilterResourceTrie = null;
@@ -157,7 +158,7 @@ public class RangerPolicyRepository {
if (other.accessAuditCache != null) {
int auditResultCacheSize = other.accessAuditCache.size();
- this.accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize));
+ this.accessAuditCache = Collections.synchronizedMap(new CacheMap<>(auditResultCacheSize));
} else {
this.accessAuditCache = null;
}
@@ -217,7 +218,7 @@ public class RangerPolicyRepository {
final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64 * 1024;
int auditResultCacheSize = pluginContext.getConfig().getInt(propertyName, RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE);
- accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize));
+ accessAuditCache = Collections.synchronizedMap(new CacheMap<>(auditResultCacheSize));
} else {
accessAuditCache = null;
}
@@ -587,7 +588,7 @@ public class RangerPolicyRepository {
if (CollectionUtils.isNotEmpty(tags) && getServiceDef() != null) {
- ret = new ArrayList<PolicyEvaluatorForTag>();
+ ret = new ArrayList<>();
for (RangerTagForEval tag : tags) {
if (tag.isApplicable(accessTime)) {
@@ -691,7 +692,7 @@ public class RangerPolicyRepository {
return auditFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getAuditPolicyEvaluators() : getLikelyMatchPolicyEvaluators(auditFilterResourceTrie, request);
}
- private List<RangerPolicyEvaluator> getLikelyMatchPolicyEvaluators(Map<String, RangerResourceTrie> resourceTrie, RangerAccessRequest request) {
+ private List<RangerPolicyEvaluator> getLikelyMatchPolicyEvaluators(Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> resourceTrie, RangerAccessRequest request) {
List<RangerPolicyEvaluator> ret = Collections.EMPTY_LIST;
RangerAccessResource resource = request.getResource();
@@ -702,67 +703,7 @@ public class RangerPolicyRepository {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerPolicyRepository.getLikelyMatchEvaluators(resource=" + resource.getAsString() + ")");
}
- List<String> resourceKeys = resource == null ? null : options.getServiceDefHelper().getOrderedResourceNames(resource.getKeys());
- Set<RangerPolicyResourceEvaluator> smallestList = null;
-
- if (CollectionUtils.isNotEmpty(resourceKeys)) {
-
- for (String resourceName : resourceKeys) {
- RangerResourceTrie<RangerPolicyResourceEvaluator> trie = resourceTrie.get(resourceName);
-
- if (trie == null) { // if no trie exists for this resource level, ignore and continue to next level
- continue;
- }
-
- Set<RangerPolicyResourceEvaluator> serviceResourceMatchersForResource = trie.getEvaluatorsForResource(resource.getValue(resourceName), request.getResourceMatchingScope());
- Set<RangerPolicyResourceEvaluator> inheritedResourceMatchers = trie.getInheritedEvaluators();
-
- if (smallestList != null) {
- if (CollectionUtils.isEmpty(inheritedResourceMatchers) && CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
- smallestList = null;
- } else if (CollectionUtils.isEmpty(inheritedResourceMatchers)) {
- smallestList.retainAll(serviceResourceMatchersForResource);
- } else if (CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
- smallestList.retainAll(inheritedResourceMatchers);
- } else {
- Set<RangerPolicyResourceEvaluator> smaller, bigger;
- if (serviceResourceMatchersForResource.size() < inheritedResourceMatchers.size()) {
- smaller = serviceResourceMatchersForResource;
- bigger = inheritedResourceMatchers;
- } else {
- smaller = inheritedResourceMatchers;
- bigger = serviceResourceMatchersForResource;
- }
- Set<RangerPolicyResourceEvaluator> tmp = new HashSet<>();
- if (smallestList.size() < smaller.size()) {
- smallestList.stream().filter(smaller::contains).forEach(tmp::add);
- smallestList.stream().filter(bigger::contains).forEach(tmp::add);
- } else {
- smaller.stream().filter(smallestList::contains).forEach(tmp::add);
- if (smallestList.size() < bigger.size()) {
- smallestList.stream().filter(bigger::contains).forEach(tmp::add);
- } else {
- bigger.stream().filter(smallestList::contains).forEach(tmp::add);
- }
- }
- smallestList = tmp;
- }
- } else {
- if (CollectionUtils.isEmpty(inheritedResourceMatchers) || CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
- Set<RangerPolicyResourceEvaluator> tmp = CollectionUtils.isEmpty(inheritedResourceMatchers) ? serviceResourceMatchersForResource : inheritedResourceMatchers;
- smallestList = resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp);
- } else {
- smallestList = new HashSet<>(serviceResourceMatchersForResource);
- smallestList.addAll(inheritedResourceMatchers);
- }
- }
-
- if (CollectionUtils.isEmpty(smallestList)) {// no tags for this resource, bail out
- smallestList = null;
- break;
- }
- }
- }
+ Collection<RangerPolicyResourceEvaluator> smallestList = RangerResourceEvaluatorsRetriever.getEvaluators(resourceTrie, resource.getAsMap(), request.getResourceMatchingScope());
if (smallestList != null) {
if (smallestList.size() == 0) {
@@ -1224,8 +1165,8 @@ public class RangerPolicyRepository {
return ret;
}
- private Map<String, RangerResourceTrie> createResourceTrieMap(List<? extends RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval, boolean optimizeTrieForSpace) {
- final Map<String, RangerResourceTrie> ret;
+ private Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> createResourceTrieMap(List<? extends RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval, boolean optimizeTrieForSpace) {
+ final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> ret;
if (serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) {
ret = new HashMap<>();
@@ -1240,7 +1181,7 @@ public class RangerPolicyRepository {
return ret;
}
- private void updateTrie(Map<String, RangerResourceTrie> trieMap, Integer policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator newEvaluator) {
+ private void updateTrie(Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trieMap, Integer policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator newEvaluator) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyRepository.updateTrie(policyDeltaType=" + policyDeltaType + "): ");
}
@@ -1436,7 +1377,7 @@ public class RangerPolicyRepository {
break;
}
- Map<String, RangerResourceTrie> trieMap = getTrie(policyType);
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trieMap = getTrie(policyType);
if (trieMap != null) {
updateTrie(trieMap, changeType, currentEvaluator, newEvaluator);
@@ -1457,8 +1398,8 @@ public class RangerPolicyRepository {
return ret;
}
- Map<String, RangerResourceTrie> getTrie(final int policyType) {
- final Map<String, RangerResourceTrie> ret;
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> getTrie(final int policyType) {
+ final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> ret;
switch (policyType) {
case RangerPolicy.POLICY_TYPE_ACCESS:
ret = policyResourceTrie;
@@ -1583,10 +1524,10 @@ public class RangerPolicyRepository {
for (int policyType = 0; policyType < flags.length; policyType++) {
if (flags[policyType]) {
- Map<String, RangerResourceTrie> trie = getTrie(policyType);
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trie = getTrie(policyType);
if (trie != null) {
- for (Map.Entry<String, RangerResourceTrie> entry : trie.entrySet()) {
+ for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : trie.entrySet()) {
entry.getValue().wrapUpUpdate();
}
}
@@ -1594,7 +1535,7 @@ public class RangerPolicyRepository {
}
if (auditFilterResourceTrie != null) {
- for (Map.Entry<String, RangerResourceTrie> entry : auditFilterResourceTrie.entrySet()) {
+ for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : auditFilterResourceTrie.entrySet()) {
entry.getValue().wrapUpUpdate();
}
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
index 504acd3a2..f89d51e35 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
@@ -105,7 +105,7 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
this(resourceDef, evaluators, isOptimizedForRetrieval, false, pluginContext);
}
- public RangerResourceTrie(RangerResourceDef resourceDef, List<T> evaluators, boolean isOptimizedForRetrieval, boolean isOptimizedForSpace, RangerPluginContext pluginContext) {
+ public <T extends RangerResourceEvaluator, E> RangerResourceTrie(RangerResourceDef resourceDef, List<E> evaluators, boolean isOptimizedForRetrieval, boolean isOptimizedForSpace, RangerPluginContext pluginContext) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + ", isOptimizedForSpace=" + isOptimizedForSpace + ")");
}
@@ -154,7 +154,7 @@ public class RangerResourceTrie<T extends RangerResourceEvaluator> {
this.isOptimizedForRetrieval = !isOptimizedForSpace && isOptimizedForRetrieval; // isOptimizedForSpace takes precedence
this.separatorChar = ServiceDefUtil.getCharOption(matcherOptions, OPTION_PATH_SEPARATOR, DEFAULT_PATH_SEPARATOR_CHAR);
- TrieNode<T> tmpRoot = buildTrie(resourceDef, evaluators, builderThreadCount);
+ final TrieNode tmpRoot = buildTrie(resourceDef, evaluators, builderThreadCount);
if (builderThreadCount > 1 && tmpRoot == null) { // if multi-threaded trie-creation failed, build using a single thread
this.root = buildTrie(resourceDef, evaluators, 1);
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
index 632d7cf7e..f4f7aaafb 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
@@ -995,13 +995,13 @@ public class TestPolicyEngine {
public static boolean compareTrie(final int policyType, RangerPolicyRepository me, RangerPolicyRepository other) {
boolean ret;
- Map<String, RangerResourceTrie> myTrie = me.getTrie(policyType);
- Map<String, RangerResourceTrie> otherTrie = other.getTrie(policyType);
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> myTrie = me.getTrie(policyType);
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> otherTrie = other.getTrie(policyType);
ret = myTrie.size() == otherTrie.size();
if (ret) {
- for (Map.Entry<String, RangerResourceTrie> entry : myTrie.entrySet()) {
+ for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : myTrie.entrySet()) {
RangerResourceTrie myResourceTrie = entry.getValue();
RangerResourceTrie otherResourceTrie = otherTrie.get(entry.getKey());