You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ab...@apache.org on 2021/11/21 17:52:57 UTC
[ranger] branch master updated: RANGER-3519: Provide an option to optimize space needed by Trie objects
This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 71888f2 RANGER-3519: Provide an option to optimize space needed by Trie objects
71888f2 is described below
commit 71888f243d38ae7cff5e0406c7d54a386d269664
Author: Abhay Kulkarni <ab...@apache.org>
AuthorDate: Sun Nov 21 09:38:11 2021 -0800
RANGER-3519: Provide an option to optimize space needed by Trie objects
---
.../plugin/contextenricher/RangerTagEnricher.java | 4 +-
.../policyengine/RangerPolicyEngineOptions.java | 32 +++++++++++-
.../policyengine/RangerPolicyRepository.java | 22 ++++-----
.../plugin/policyengine/RangerResourceTrie.java | 57 +++++++++++++++++++---
distro/src/main/assembly/ranger-tools.xml | 6 +++
ranger-tools/scripts/create_requests.py | 42 ++++++++++++++++
.../ranger/policyengine/PerfTestConfiguration.java | 31 ++++++++++++
.../apache/ranger/policyengine/PerfTestEngine.java | 7 +++
.../ranger/policyengine/PerfTestOptions.java | 11 +++++
.../policyengine/RangerPolicyenginePerfTester.java | 27 ++++++----
.../src/test/resources/testdata/ranger-config.xml | 9 ++++
.../resources/testdata/test_requests_hive.json | 4 +-
ranger-tools/testdata/ranger-config.xml | 11 ++++-
13 files changed, 227 insertions(+), 36 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index c8346d3..6b0451e 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -447,7 +447,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
serviceResourceTrie = new HashMap<>();
for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
- serviceResourceTrie.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, resourceMatchers, getPolicyEngineOptions().optimizeTrieForRetrieval, null));
+ serviceResourceTrie.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, resourceMatchers, getPolicyEngineOptions().optimizeTagTrieForRetrieval, getPolicyEngineOptions().optimizeTagTrieForSpace, null));
}
}
enrichedServiceTags = new EnrichedServiceTags(serviceTags, resourceMatchers, serviceResourceTrie);
@@ -491,7 +491,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
LOG.debug("Added resource-matcher for service-resource:[" + serviceResource + "]");
}
} else {
- trie = new RangerResourceTrie<>(resourceDef, Collections.singletonList(resourceMatcher));
+ trie = new RangerResourceTrie<>(resourceDef, Collections.singletonList(resourceMatcher), getPolicyEngineOptions().optimizeTagTrieForRetrieval, getPolicyEngineOptions().optimizeTagTrieForSpace, null);
serviceResourceTrie.put(resourceDef.getName(), trie);
}
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
index 07d0a39..2afa755 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
@@ -37,6 +37,10 @@ public class RangerPolicyEngineOptions {
public boolean disableAccessEvaluationWithPolicyACLSummary = true;
public boolean optimizeTrieForRetrieval = false;
public boolean disableRoleResolution = true;
+ public boolean optimizeTrieForSpace = false;
+ public boolean optimizeTagTrieForRetrieval = false;
+ public boolean optimizeTagTrieForSpace = false;
+
private RangerServiceDefHelper serviceDefHelper;
@@ -56,6 +60,9 @@ public class RangerPolicyEngineOptions {
this.optimizeTrieForRetrieval = other.optimizeTrieForRetrieval;
this.disableRoleResolution = other.disableRoleResolution;
this.serviceDefHelper = null;
+ this.optimizeTrieForSpace = other.optimizeTrieForSpace;
+ this.optimizeTagTrieForRetrieval = other.optimizeTagTrieForRetrieval;
+ this.optimizeTagTrieForSpace = other.optimizeTagTrieForSpace;
}
public void configureForPlugin(Configuration conf, String propertyPrefix) {
@@ -76,6 +83,9 @@ public class RangerPolicyEngineOptions {
disableAccessEvaluationWithPolicyACLSummary = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.access.evaluation.with.policy.acl.summary", true);
optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false);
disableRoleResolution = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.role.resolution", true);
+ optimizeTrieForSpace = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.space", false);
+ optimizeTagTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.tag.trie.for.retrieval", false);
+ optimizeTagTrieForSpace = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.tag.trie.for.space", false);
}
@@ -125,6 +135,10 @@ public class RangerPolicyEngineOptions {
cacheAuditResults = false;
evaluateDelegateAdminOnly = false;
enableTagEnricherWithLocalRefresher = true;
+
+ optimizeTrieForSpace = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.space", false);
+ optimizeTagTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.tag.trie.for.retrieval", false);
+ optimizeTagTrieForSpace = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.tag.trie.for.space", true);
}
public RangerServiceDefHelper getServiceDefHelper() {
@@ -155,7 +169,11 @@ public class RangerPolicyEngineOptions {
&& this.evaluateDelegateAdminOnly == that.evaluateDelegateAdminOnly
&& this.enableTagEnricherWithLocalRefresher == that.enableTagEnricherWithLocalRefresher
&& this.optimizeTrieForRetrieval == that.optimizeTrieForRetrieval
- && this.disableRoleResolution == that.disableRoleResolution;
+ && this.disableRoleResolution == that.disableRoleResolution
+ && this.optimizeTrieForSpace == that.optimizeTrieForSpace
+ && this.optimizeTagTrieForRetrieval == that.optimizeTagTrieForRetrieval
+ && this.optimizeTagTrieForSpace == that.optimizeTagTrieForSpace
+ ;
}
return ret;
}
@@ -184,7 +202,14 @@ public class RangerPolicyEngineOptions {
ret += optimizeTrieForRetrieval ? 1 : 0;
ret *= 2;
ret += disableRoleResolution ? 1 : 0;
- ret *= 2; return ret;
+ ret *= 2;
+ ret += optimizeTrieForSpace ? 1 : 0;
+ ret *= 2;
+ ret += optimizeTagTrieForRetrieval ? 1 : 0;
+ ret *= 2;
+ ret += optimizeTagTrieForSpace ? 1 : 0;
+ ret *= 2;
+ return ret;
}
@Override
@@ -202,6 +227,9 @@ public class RangerPolicyEngineOptions {
", optimizeTrieForRetrieval: " + optimizeTrieForRetrieval +
", cacheAuditResult: " + cacheAuditResults +
", disableRoleResolution: " + disableRoleResolution +
+ ", optimizeTrieForSpace: " + optimizeTrieForSpace +
+ ", optimizeTagTrieForRetrieval: " + optimizeTagTrieForRetrieval +
+ ", optimizeTagTrieForSpace: " + optimizeTagTrieForSpace +
" }";
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 008ee77..e998944 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -242,10 +242,10 @@ public class RangerPolicyRepository {
rowFilterResourceTrie = null;
auditFilterResourceTrie = null;
} else {
- policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval);
- dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval);
- rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval);
- auditFilterResourceTrie = createResourceTrieMap(auditPolicyEvaluators, options.optimizeTrieForRetrieval);
+ policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
+ dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
+ rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
+ auditFilterResourceTrie = createResourceTrieMap(auditPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
}
}
@@ -296,10 +296,10 @@ public class RangerPolicyRepository {
rowFilterResourceTrie = null;
auditFilterResourceTrie = null;
} else {
- policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval);
- dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval);
- rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval);
- auditFilterResourceTrie = createResourceTrieMap(auditPolicyEvaluators, options.optimizeTrieForRetrieval);
+ policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
+ dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTagTrieForSpace);
+ rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
+ auditFilterResourceTrie = createResourceTrieMap(auditPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
}
}
@@ -1188,14 +1188,14 @@ public class RangerPolicyRepository {
return ret;
}
- private Map<String, RangerResourceTrie> createResourceTrieMap(List<? extends RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval) {
+ private Map<String, RangerResourceTrie> createResourceTrieMap(List<? extends RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval, boolean optimizeTrieForSpace) {
final Map<String, RangerResourceTrie> ret;
if (serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) {
ret = new HashMap<>();
for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
- ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators, optimizeTrieForRetrieval, pluginContext));
+ ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators, optimizeTrieForRetrieval, optimizeTrieForSpace, pluginContext));
}
} else {
ret = null;
@@ -1218,7 +1218,7 @@ public class RangerPolicyRepository {
if (RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE == policyDeltaType || RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE == policyDeltaType) {
LOG.warn("policyDeltaType is not for POLICY_CREATE and trie for resourceDef:[" + resourceDefName + "] was null! Should not have happened!!");
}
- trie = new RangerResourceTrie<>(resourceDef, new ArrayList<>(), true, pluginContext);
+ trie = new RangerResourceTrie<>(resourceDef, new ArrayList<>(), options.optimizeTrieForRetrieval, options.optimizeTrieForSpace, pluginContext);
trieMap.put(resourceDefName, trie);
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
index 7c37e05..80ed569 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
@@ -60,6 +60,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
private final boolean optWildcard;
private final String wildcardChars;
private final boolean isOptimizedForRetrieval;
+ private final boolean isOptimizedForSpace;
private final Character separatorChar;
private Set<T> inheritedEvaluators;
private final TrieNode<T> root;
@@ -79,6 +80,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
this.optIgnoreCase = other.optIgnoreCase;
this.optWildcard = other.optWildcard;
this.wildcardChars = other.wildcardChars;
+ this.isOptimizedForSpace = other.isOptimizedForSpace;
this.isOptimizedForRetrieval = false;
this.separatorChar = other.separatorChar;
this.inheritedEvaluators = other.inheritedEvaluators != null ? new HashSet<>(other.inheritedEvaluators) : null;
@@ -98,8 +100,12 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
public RangerResourceTrie(RangerResourceDef resourceDef, List<T> evaluators, boolean isOptimizedForRetrieval, RangerPluginContext pluginContext) {
+ this(resourceDef, evaluators, isOptimizedForRetrieval, false, pluginContext);
+ }
+
+ public RangerResourceTrie(RangerResourceDef resourceDef, List<T> evaluators, boolean isOptimizedForRetrieval, boolean isOptimizedForSpace, RangerPluginContext pluginContext) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + ")");
+ LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + ", isOptimizedForSpace=" + isOptimizedForSpace + ")");
}
RangerPerfTracer perf = null;
@@ -137,7 +143,8 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
this.optIgnoreCase = RangerAbstractResourceMatcher.getOptionIgnoreCase(matcherOptions);
this.optWildcard = RangerAbstractResourceMatcher.getOptionWildCard(matcherOptions);
this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS + tokenReplaceSpecialChars : "" + tokenReplaceSpecialChars;
- this.isOptimizedForRetrieval = isOptimizedForRetrieval;
+ this.isOptimizedForSpace = isOptimizedForSpace;
+ this.isOptimizedForRetrieval = !isOptimizedForSpace && isOptimizedForRetrieval; // isOptimizedForSpace takes precedence
this.separatorChar = ServiceDefUtil.getCharOption(matcherOptions, OPTION_PATH_SEPARATOR, DEFAULT_PATH_SEPARATOR_CHAR);
TrieNode<T> tmpRoot = buildTrie(resourceDef, evaluators, builderThreadCount);
@@ -163,7 +170,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + "): " + toString());
+ LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + this.isOptimizedForRetrieval + ", isOptimizedForSpace=" + this.isOptimizedForSpace + "): " + toString());
}
}
@@ -565,9 +572,15 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
final int len = resource.length();
int i = 0;
+ Set<T> accumulatedEvaluators = new HashSet<>();
+
while (i < len) {
- if (!isOptimizedForRetrieval) {
+ if (!isOptimizedForSpace) {
curr.setupIfNeeded(parent);
+ } else {
+ if (curr.getWildcardEvaluators() != null) {
+ accumulatedEvaluators.addAll(curr.getWildcardEvaluators());
+ }
}
child = curr.getChild(getLookupChar(resource, i));
@@ -587,12 +600,28 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
i += childStr.length();
}
- if (!isOptimizedForRetrieval) {
+ if (!isOptimizedForSpace) {
curr.setupIfNeeded(parent);
+ } else {
+ if (curr.getWildcardEvaluators() != null) {
+ accumulatedEvaluators.addAll(curr.getWildcardEvaluators());
+ }
+ }
+
+ boolean isSelfMatch = (i == len);
+ Set<T> ret;
+
+ if (!isOptimizedForSpace) {
+ ret = isSelfMatch ? curr.getEvaluators() : curr.getWildcardEvaluators();
+ } else {
+ if (isSelfMatch) {
+ if (curr.getEvaluators() != null) {
+ accumulatedEvaluators.addAll(curr.getEvaluators());
+ }
+ }
+ ret = accumulatedEvaluators;
}
- boolean isSelfMatch = (i == len);
- Set<T> ret = isSelfMatch ? curr.getEvaluators() : curr.getWildcardEvaluators();
boolean includeEvaluatorsOfChildResources = scope == RangerAccessRequest.ResourceMatchingScope.SELF_OR_CHILD;
if (includeEvaluatorsOfChildResources) {
@@ -1149,17 +1178,29 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
void collectChildEvaluators(Character sep, int startIdx, Set<U> childEvaluators) {
- setupIfNeeded(getParent());
+ if (!isOptimizedForSpace) {
+ setupIfNeeded(getParent());
+ }
final int sepPos = startIdx < str.length() ? str.indexOf(sep, startIdx) : -1;
if (sepPos == -1) { // ex: startIdx=5, path(str)=/tmp/test, path(a child) could be: /tmp/test.txt, /tmp/test/, /tmp/test/a, /tmp/test/a/b
+ if (isOptimizedForSpace) {
+ if (this.wildcardEvaluators != null) {
+ childEvaluators.addAll(this.wildcardEvaluators);
+ }
+ }
if (this.evaluators != null) {
childEvaluators.addAll(this.evaluators);
}
children.values().stream().forEach(c -> c.collectChildEvaluators(sep, 0, childEvaluators));
} else if (sepPos == (str.length() - 1)) { // ex: str=/tmp/test/, startIdx=5
+ if (isOptimizedForSpace) {
+ if (this.wildcardEvaluators != null) {
+ childEvaluators.addAll(this.wildcardEvaluators);
+ }
+ }
if (this.evaluators != null) {
childEvaluators.addAll(this.evaluators);
}
diff --git a/distro/src/main/assembly/ranger-tools.xml b/distro/src/main/assembly/ranger-tools.xml
index 3ae890c..5fdf3a5 100644
--- a/distro/src/main/assembly/ranger-tools.xml
+++ b/distro/src/main/assembly/ranger-tools.xml
@@ -140,5 +140,11 @@
<destName>README.txt</destName>
<fileMode>644</fileMode>
</file>
+ <file>
+ <source>${project.parent.basedir}/ranger-tools/scripts/create_requests.py</source>
+ <outputDirectory></outputDirectory>
+ <destName>create_requests.py</destName>
+ <fileMode>644</fileMode>
+ </file>
</files>
</assembly>
diff --git a/ranger-tools/scripts/create_requests.py b/ranger-tools/scripts/create_requests.py
new file mode 100755
index 0000000..51efc05
--- /dev/null
+++ b/ranger-tools/scripts/create_requests.py
@@ -0,0 +1,42 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+import json
+
+# Opening JSON file
+f = open("test_servicetags_hive.json")
+
+# returns JSON object as a dictionary
+data = json.load(f)
+
+final_list = []
+
+# Iterating through the json list
+for i in data['serviceResources']:
+ resource_id = i['id']
+ # dictionary with table, database, or column
+ resource_elements = i['resourceElements']
+ temp = {'name': "request-" + str(resource_id), 'request': {'resource': {'elements': {}}, 'accessType': "select", 'user': "hrt_1", 'userGroups': [], 'requestData': "request-" + str(resource_id)}, 'result': {'isAudited': 'true', 'isAllowed': 'true', 'policyId': 2}}
+
+ resource_keys = resource_elements.keys()
+ for resource_key in resource_keys:
+ resource_item = resource_elements[resource_key]
+ resource_value = resource_item['values'][0]
+ temp['request']['resource']['elements'][resource_key] = resource_value
+
+ final_list.append(temp)
+
+# Writing JSON file
+
+with open("test_requests_hive.json", "w") as outfile:
+ json.dump(final_list, outfile)
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestConfiguration.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestConfiguration.java
new file mode 100644
index 0000000..ec43374
--- /dev/null
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestConfiguration.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.policyengine;
+
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+
+import java.net.URL;
+
+public class PerfTestConfiguration extends RangerConfiguration {
+ PerfTestConfiguration(URL configURL) {
+ super();
+ this.addResourceIfReadable(configURL.getPath());
+ }
+}
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
index 37ac9ea..287b9d8 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
@@ -125,4 +125,11 @@ public class PerfTestEngine {
return ret;
}
+ public void cleanUp() {
+ if (policyEvaluationEngine != null) {
+ ((RangerPolicyEngineImpl)policyEvaluationEngine).releaseResources(true);
+ policyEvaluationEngine = null;
+ }
+ }
+
}
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
index 0379fe6..2af5856 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
@@ -20,6 +20,8 @@
package org.apache.ranger.policyengine;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+
import java.net.URL;
public class PerfTestOptions {
@@ -32,6 +34,8 @@ public class PerfTestOptions {
private final int concurrentClientCount;
private final int iterationsCount;
private final URL perfConfigurationFileURL;
+ private final boolean isPolicyTrieOptimizedForSpace;
+ private final boolean isTagTrieOptimizedForSpace;
PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isTrieLookupPrefixDisabled, boolean isOnDemandTriePostSetupDisabled, URL perfConfigurationFileURL) {
this.servicePoliciesFileURL = servicePoliciesFileURL;
@@ -42,6 +46,10 @@ public class PerfTestOptions {
this.isTrieLookupPrefixDisabled = isTrieLookupPrefixDisabled;
this.isOnDemandTriePostSetupDisabled = isOnDemandTriePostSetupDisabled;
this.perfConfigurationFileURL = perfConfigurationFileURL;
+
+ RangerConfiguration configuration = new PerfTestConfiguration(perfConfigurationFileURL);
+ this.isPolicyTrieOptimizedForSpace = configuration.getBoolean("ranger.policyengine.option.optimize.policy.trie.for.space", false);
+ this.isTagTrieOptimizedForSpace = configuration.getBoolean("ranger.policyengine.option.optimize.tag.trie.for.space", false);
}
public URL getServicePoliciesFileURL() {
@@ -72,4 +80,7 @@ public class PerfTestOptions {
return this.perfConfigurationFileURL;
}
+ public boolean getIsPolicyTrieOptimizedForSpace() { return isPolicyTrieOptimizedForSpace; }
+ public boolean getIsTagTrieOptimizedForSpace() { return isTagTrieOptimizedForSpace; }
+
}
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
index 760ebb9..f667628 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
@@ -21,6 +21,7 @@ package org.apache.ranger.policyengine;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import org.apache.ranger.plugin.util.PerfDataRecorder;
@@ -62,9 +63,15 @@ public class RangerPolicyenginePerfTester {
policyEngineOptions.cacheAuditResults = false;
policyEngineOptions.disableTrieLookupPrefilter = perfTestOptions.getIsTrieLookupPrefixDisabled();
policyEngineOptions.optimizeTrieForRetrieval = perfTestOptions.getIsOnDemandTriePostSetupDisabled();
+ policyEngineOptions.optimizeTagTrieForSpace = perfTestOptions.getIsTagTrieOptimizedForSpace();
+ policyEngineOptions.optimizeTrieForSpace = perfTestOptions.getIsPolicyTrieOptimizedForSpace();
URL configurationFileURL = perfTestOptions.getPerfConfigurationFileURL();
+ RangerConfiguration configuration = new PerfTestConfiguration(configurationFileURL);
+ policyEngineOptions.optimizeTrieForSpace = configuration.getBoolean("ranger.policyengine.option.optimize.policy.trie.for.space", false);
+ policyEngineOptions.optimizeTagTrieForSpace = configuration.getBoolean("ranger.policyengine.option.optimize.tag.trie.for.space", false);
+
PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions, configurationFileURL);
if (!perfTestEngine.init()) {
LOG.error("Error initializing test data. Existing...");
@@ -115,12 +122,12 @@ public class RangerPolicyenginePerfTester {
}
Runtime runtime = Runtime.getRuntime();
- runtime.gc();
+ runtime.gc();
long totalMemory = runtime.totalMemory();
long freeMemory = runtime.freeMemory();
- LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);
+ LOG.info("Before performance-run start: Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);
LOG.info("Starting " + perfTestClients.size() + " clients..");
for (PerfTestClient client : perfTestClients) {
@@ -138,14 +145,6 @@ public class RangerPolicyenginePerfTester {
while (client.isAlive()) {
try {
client.join(1000);
-
- runtime.gc();
-
- totalMemory = runtime.totalMemory();
- freeMemory = runtime.freeMemory();
-
- LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);
-
} catch (InterruptedException interruptedException) {
LOG.error("PerfTestClient.join() was interrupted");
}
@@ -158,6 +157,14 @@ public class RangerPolicyenginePerfTester {
LOG.info("Completed performance-run");
+ runtime.gc();
+ totalMemory = runtime.totalMemory();
+ freeMemory = runtime.freeMemory();
+
+ LOG.info("After performance-run end: Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);
+
+ perfTestEngine.cleanUp();
+
PerfDataRecorder.printStatistics();
}
diff --git a/ranger-tools/src/test/resources/testdata/ranger-config.xml b/ranger-tools/src/test/resources/testdata/ranger-config.xml
index 933d670..218db07 100644
--- a/ranger-tools/src/test/resources/testdata/ranger-config.xml
+++ b/ranger-tools/src/test/resources/testdata/ranger-config.xml
@@ -15,4 +15,13 @@
<name>ranger.policyengine.trie.builder.thread.count</name>
<value>1</value>
</property>
+ <property>
+ <name>ranger.policyengine.option.optimize.policy.trie.for.space</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>ranger.policyengine.option.optimize.tag.trie.for.space</name>
+ <value>true</value>
+ </property>
+
</configuration>
\ No newline at end of file
diff --git a/ranger-tools/src/test/resources/testdata/test_requests_hive.json b/ranger-tools/src/test/resources/testdata/test_requests_hive.json
index 0db7207..afba86a 100644
--- a/ranger-tools/src/test/resources/testdata/test_requests_hive.json
+++ b/ranger-tools/src/test/resources/testdata/test_requests_hive.json
@@ -1,7 +1,7 @@
[
- {"name":"'select default/tbl-0/col-2;' for hrt_1",
+ {"name":"'select finance_1/tax_2010_1/col-2;' for hrt_1",
"request":{
- "resource":{"elements":{"database":"default","table":"tbl-0","column":"col-2"}},
+ "resource":{"elements":{"database":"finance_1","table":"tax_2010_1","column":"col-2"}},
"accessType":"select","user":"hrt_1","userGroups":[],"requestData":"use default"
},
"result":{"isAudited":true,"isAllowed":true,"policyId":2}
diff --git a/ranger-tools/testdata/ranger-config.xml b/ranger-tools/testdata/ranger-config.xml
index 933d670..08a83b9 100644
--- a/ranger-tools/testdata/ranger-config.xml
+++ b/ranger-tools/testdata/ranger-config.xml
@@ -15,4 +15,13 @@
<name>ranger.policyengine.trie.builder.thread.count</name>
<value>1</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>ranger.policyengine.option.optimize.policy.trie.for.space</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>ranger.policyengine.option.optimize.tag.trie.for.space</name>
+ <value>true</value>
+ </property>
+
+</configuration>