You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by me...@apache.org on 2018/09/04 12:58:33 UTC
[23/28] ranger git commit: RANGER-2191: Update ranger-tool with new
options to control Trie
RANGER-2191: Update ranger-tool with new options to control Trie
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/383864c3
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/383864c3
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/383864c3
Branch: refs/heads/ranger-1.1
Commit: 383864c3ac5b0f6c013b39d04604d78895ead7bb
Parents: fbcaaaf
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Thu Aug 16 23:53:39 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:48:40 2018 +0530
----------------------------------------------------------------------
.../RangerDefaultPolicyEvaluator.java | 4 +--
.../ranger/plugin/util/RangerResourceTrie.java | 20 +++++++------
ranger-tools/scripts/README.txt | 9 ++++--
.../ranger/policyengine/CommandLineParser.java | 30 ++++++++++++++++++--
.../ranger/policyengine/PerfTestOptions.java | 14 ++++++++-
.../RangerPolicyenginePerfTester.java | 9 ++++++
ranger-tools/src/test/resources/commandline | 2 +-
.../test/resources/testdata/ranger-config.xml | 18 ++++++++++++
ranger-tools/testdata/ranger-config.xml | 18 ++++++++++++
9 files changed, 106 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
index 2a5ee54..289ec9b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
@@ -207,7 +207,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
@Override
public void evaluate(RangerAccessRequest request, RangerAccessResult result) {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")");
+ LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")");
}
RangerPerfTracer perf = null;
@@ -257,7 +257,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
RangerPerfTracer.log(perf);
if(LOG.isDebugEnabled()) {
- LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")");
+ LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")");
}
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index e702684..9c7c775 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -68,7 +68,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + ")");
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + ")");
}
int builderThreadCount = RangerConfiguration.getInstance().getInt(TRIE_BUILDER_THREAD_COUNT, 1);
@@ -154,6 +154,12 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
LOG.debug("==> buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + (builderThreadCount > 1) + ")");
}
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(resourceDef=" + resourceDef.getName() + ")");
+ }
+
TrieNode<T> ret = new TrieNode<>(null);
final boolean isMultiThreaded = builderThreadCount > 1;
final List<ResourceTrieBuilderThread> builderThreads;
@@ -243,7 +249,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
RangerPerfTracer postSetupPerf = null;
if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + "-postSetup)");
+ postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + "-postSetup)");
}
ret.postSetup(null, comparator);
@@ -259,6 +265,8 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
cleanUpThreads(builderThreads);
}
+ RangerPerfTracer.logAlways(perf);
+
if(LOG.isDebugEnabled()) {
LOG.debug("<== buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + isMultiThreaded + ") :" + ret);
}
@@ -324,11 +332,6 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
private void insert(TrieNode<T> currentRoot, String resource, boolean isRecursive, T evaluator) {
- RangerPerfTracer perf = null;
-
- if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.insert(resource=" + resource + ")");
- }
TrieNode<T> curr = currentRoot;
final String prefix = getNonWildcardPrefix(resource);
@@ -344,7 +347,6 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
curr.addEvaluator(evaluator);
}
- RangerPerfTracer.logAlways(perf);
}
private String getNonWildcardPrefix(String str) {
@@ -559,7 +561,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
RangerPerfTracer postSetupPerf = null;
if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(thread=" + this.getName() + "-postSetup)");
+ postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(thread=" + this.getName() + "-postSetup)");
}
thisRoot.postSetup(parentWildcardEvaluators, comparator);
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/scripts/README.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt
index 55170d5..81b5b66 100644
--- a/ranger-tools/scripts/README.txt
+++ b/ranger-tools/scripts/README.txt
@@ -51,6 +51,8 @@ This file describes how to build, setup, configure and run the performance testi
tag-definitions, and service-resources with their associated tags;
testdata/test_requests_hive.json - Contains access requests to be made to the policy-engine;
+
+ testdata/ranger-config.xml - Contains any required Ranger configuration variables
Please review the contents of these files and modify to suit your profiling needs.
@@ -58,10 +60,13 @@ This file describes how to build, setup, configure and run the performance testi
6. Run the tool with the following command
- % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run>
+ % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run> -t -d -f <ranger-configuration-file> -p <test-modules-file>
+
+ where, -t indicates enabling Trie,
+ -d indicates enabling lazy post-setup of Trie structure,
Example:
- % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1
+ % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1 -t -d -f testdata/ranger-config.xml -p testdata/test_modules.txt
7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file as shown below. This is for time spent in evaluating access by Ranger Policy Engine during the course of a test run. The time values shown are in milliseconds.
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
index e8edd9e..fe9ec17 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
@@ -55,6 +55,12 @@ public class CommandLineParser
private boolean isDynamicReorderingDisabled = true;
private boolean isTrieLookupPrefixDisabled = true;
+ private boolean isLazyTriePostSetupDisabled = true;
+
+ private String configurationFileName;
+ private URL configurationFileURL;
+
+
private Options options = new Options();
CommandLineParser() {}
@@ -63,7 +69,7 @@ public class CommandLineParser
PerfTestOptions ret = null;
if (parseArguments(args) && validateInputFiles()) {
// Instantiate a data-object and return
- ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingDisabled, isTrieLookupPrefixDisabled);
+ ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingDisabled, isTrieLookupPrefixDisabled, isLazyTriePostSetupDisabled, configurationFileURL);
} else {
showUsage();
}
@@ -78,7 +84,10 @@ public class CommandLineParser
-r request-file-name-list
-n number-of-iterations
-p modules-to-collect-stats
+ -f configuration-file-name
-o
+ -t
+ -d
If the concurrent-client-count is more than the number of files in the request-file-name-list,
then reuse the request-file-names in a round-robin way
@@ -98,8 +107,11 @@ public class CommandLineParser
options.addOption("p", "statistics", true, "Modules for stat collection File Name");
options.addOption("c", "clients", true, "Number of concurrent clients");
options.addOption("n", "cycles", true, "Number of iterations");
+ options.addOption("f", "configurations", true, "Configuration File Name");
options.addOption("o", "optimize", false, "Enable usage-based policy reordering");
options.addOption("t", "trie-prefilter", false, "Enable trie-prefilter");
+ options.addOption("d", "trie-lazy-setup", false, "Enable lazy trie-setup");
+
org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser();
@@ -133,11 +145,20 @@ public class CommandLineParser
isTrieLookupPrefixDisabled = false;
}
+ if (commandLine.hasOption("d")) {
+ isLazyTriePostSetupDisabled = false;
+ }
+
+ configurationFileName = commandLine.getOptionValue("f");
+
if (LOG.isDebugEnabled()) {
LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames));
LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount);
LOG.debug("isDynamicReorderingDisabled=" + isDynamicReorderingDisabled);
LOG.debug("isTrieLookupPrefixDisabled=" + isTrieLookupPrefixDisabled);
+ LOG.debug("isLazyTriePostSetupDisabled=" + isLazyTriePostSetupDisabled);
+ LOG.debug("configurationFileName=" + configurationFileName);
+
}
ret = true;
@@ -164,11 +185,14 @@ public class CommandLineParser
if (servicePoliciesFileURL != null) {
if (requestFileNames != null) {
if (validateRequestFiles()) {
+ ret = true;
if (statCollectionFileName != null) {
statCollectionFileURL = getInputFileURL(statCollectionFileName);
ret = statCollectionFileURL != null;
- } else {
- ret = true;
+ }
+ if (ret && configurationFileName != null) {
+ configurationFileURL = getInputFileURL(configurationFileName);
+ ret = configurationFileURL != null;
}
}
} else {
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
index d6e04ea..1486f52 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
@@ -28,12 +28,15 @@ public class PerfTestOptions {
private final URL statCollectionFileURL;
private final boolean isDynamicReorderingDisabled;
private final boolean isTrieLookupPrefixDisabled;
+ private final boolean isOnDemandTriePostSetupDisabled;
private final int concurrentClientCount;
private final int iterationsCount;
+ private final URL perfConfigurationFileURL;
- PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingDisabled, boolean isTrieLookupPrefixDisabled) {
+
+ PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingDisabled, boolean isTrieLookupPrefixDisabled, boolean isOnDemandTriePostSetupDisabled, URL perfConfigurationFileURL) {
this.servicePoliciesFileURL = servicePoliciesFileURL;
this.requestFileURLs = requestFileURLs;
this.statCollectionFileURL = statCollectionFileURL;
@@ -41,6 +44,8 @@ public class PerfTestOptions {
this.concurrentClientCount = concurrentClientCount;
this.isDynamicReorderingDisabled = isDynamicReorderingDisabled;
this.isTrieLookupPrefixDisabled = isTrieLookupPrefixDisabled;
+ this.isOnDemandTriePostSetupDisabled = isOnDemandTriePostSetupDisabled;
+ this.perfConfigurationFileURL = perfConfigurationFileURL;
}
public URL getServicePoliciesFileURL() {
@@ -66,4 +71,11 @@ public class PerfTestOptions {
public boolean getIsDynamicReorderingDisabled() { return isDynamicReorderingDisabled; }
public boolean getIsTrieLookupPrefixDisabled() { return isTrieLookupPrefixDisabled; }
+
+ public boolean getIsOnDemandTriePostSetupDisabled() { return isOnDemandTriePostSetupDisabled; }
+
+ public URL getPerfConfigurationFileURL() {
+ return this.perfConfigurationFileURL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
index 78cbe02..53e5cf0 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
@@ -21,6 +21,7 @@ package org.apache.ranger.policyengine;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import org.apache.ranger.plugin.util.PerfDataRecorder;
@@ -61,6 +62,14 @@ public class RangerPolicyenginePerfTester {
policyEngineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED;
policyEngineOptions.cacheAuditResults = false;
policyEngineOptions.disableTrieLookupPrefilter = perfTestOptions.getIsTrieLookupPrefixDisabled();
+ policyEngineOptions.optimizeTrieForRetrieval = perfTestOptions.getIsOnDemandTriePostSetupDisabled();
+
+ URL configurationFileURL = perfTestOptions.getPerfConfigurationFileURL();
+
+ if (configurationFileURL != null) {
+ RangerConfiguration config = RangerConfiguration.getInstance();
+ config.addResource(configurationFileURL);
+ }
PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions, perfTestOptions.getIsDynamicReorderingDisabled());
if (!perfTestEngine.init()) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/test/resources/commandline
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/commandline b/ranger-tools/src/test/resources/commandline
index 9ea690e..73338d5 100644
--- a/ranger-tools/src/test/resources/commandline
+++ b/ranger-tools/src/test/resources/commandline
@@ -17,4 +17,4 @@
# under the License.
#
--s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1
+-s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1 -t -d -f /testdata/ranger-config.xml
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/test/resources/testdata/ranger-config.xml
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/testdata/ranger-config.xml b/ranger-tools/src/test/resources/testdata/ranger-config.xml
new file mode 100644
index 0000000..933d670
--- /dev/null
+++ b/ranger-tools/src/test/resources/testdata/ranger-config.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<configuration>
+ <property>
+ <name>ranger.policyengine.trie.builder.thread.count</name>
+ <value>1</value>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/testdata/ranger-config.xml
----------------------------------------------------------------------
diff --git a/ranger-tools/testdata/ranger-config.xml b/ranger-tools/testdata/ranger-config.xml
new file mode 100644
index 0000000..933d670
--- /dev/null
+++ b/ranger-tools/testdata/ranger-config.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<configuration>
+ <property>
+ <name>ranger.policyengine.trie.builder.thread.count</name>
+ <value>1</value>
+ </property>
+</configuration>
\ No newline at end of file