You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/12/22 02:57:11 UTC
[5/5] incubator-ranger git commit: RANGER-794: Ranger policy engine
performance measurement
RANGER-794: Ranger policy engine performance measurement
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/fbf4f353
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/fbf4f353
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/fbf4f353
Branch: refs/heads/master
Commit: fbf4f3533d0c39d018d2ac92538f77761ca461d3
Parents: ea559d3
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Mon Dec 21 17:53:51 2015 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Dec 21 17:53:51 2015 -0800
----------------------------------------------------------------------
.../RangerFileBasedTagRetriever.java | 149 +
.../contextenricher/RangerTagEnricher.java | 9 +-
.../ranger/plugin/util/PerfDataRecorder.java | 113 +
.../plugin/util/RangerPerfCollectorTracer.java | 37 +
.../ranger/plugin/util/RangerPerfTracer.java | 34 +-
.../plugin/util/RangerPerfTracerFactory.java | 39 +
pom.xml | 3 +
ranger-tools/.gitignore | 2 +
ranger-tools/conf/log4j.properties | 50 +
ranger-tools/pom.xml | 69 +
ranger-tools/scripts/README.txt | 65 +
ranger-tools/scripts/ranger-perftester.sh | 32 +
ranger-tools/scripts/summary.awk | 17 +
.../ranger/policyengine/CommandLineParser.java | 275 +
.../ranger/policyengine/PerfTestClient.java | 161 +
.../ranger/policyengine/PerfTestEngine.java | 126 +
.../ranger/policyengine/PerfTestOptions.java | 60 +
.../RangerPolicyenginePerfTester.java | 140 +
.../ranger/policyengine/PerfTesterTest.java | 112 +
ranger-tools/src/test/resources/commandline | 20 +
.../src/test/resources/log4j.properties | 50 +
.../test/resources/testdata/test_modules.txt | 24 +
.../resources/testdata/test_requests_hive.json | 10 +
.../testdata/test_servicepolicies_hive.json | 488 +
.../testdata/test_servicetags_hive.json | 62 +
ranger-tools/testdata/test_modules.txt | 23 +
ranger-tools/testdata/test_requests_hive.json | 257 +
.../testdata/test_servicepolicies_hive.json | 252899 ++++++++++++++++
.../testdata/test_servicetags_hive.json | 23966 ++
src/main/assembly/ranger-tools.xml | 134 +
30 files changed, 279413 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
new file mode 100644
index 0000000..95291be
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.contextenricher;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.util.ServiceTags;
+
+import java.io.*;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Map;
+
+public class RangerFileBasedTagRetriever extends RangerTagRetriever {
+ private static final Log LOG = LogFactory.getLog(RangerFileBasedTagRetriever.class);
+
+
+ private URL serviceTagsFileURL;
+ private String serviceTagsFileName;
+
+ @Override
+ public void init(Map<String, String> options) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> init()" );
+ }
+ String serviceTagsFileNameProperty = "serviceTagsFileName";
+ String serviceTagsDefaultFileName = "/testdata/test_servicetags_hive.json";
+
+ if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) {
+ InputStream serviceTagsFileStream = null;
+
+
+ // Open specified file from options- it should contain service-tags
+
+ serviceTagsFileName = options != null? options.get(serviceTagsFileNameProperty) : null;
+
+ serviceTagsFileName = serviceTagsFileName == null ? serviceTagsDefaultFileName : serviceTagsFileName;
+
+ File f = new File(serviceTagsFileName);
+
+ if (f.exists() && f.isFile() && f.canRead()) {
+ try {
+ serviceTagsFileStream = new FileInputStream(f);
+ serviceTagsFileURL = f.toURI().toURL();
+ } catch (FileNotFoundException exception) {
+ LOG.error("Error processing input file:" + serviceTagsFileName + " or no privilege for reading file " + serviceTagsFileName, exception);
+ } catch (MalformedURLException malformedException) {
+ LOG.error("Error processing input file:" + serviceTagsFileName + " cannot be converted to URL " + serviceTagsFileName, malformedException);
+ }
+ } else {
+
+ URL fileURL = getClass().getResource(serviceTagsFileName);
+ if (fileURL == null) {
+ if (!serviceTagsFileName.startsWith("/")) {
+ fileURL = getClass().getResource("/" + serviceTagsFileName);
+ }
+ }
+
+ if (fileURL == null) {
+ fileURL = ClassLoader.getSystemClassLoader().getResource(serviceTagsFileName);
+ if (fileURL == null) {
+ if (!serviceTagsFileName.startsWith("/")) {
+ fileURL = ClassLoader.getSystemClassLoader().getResource("/" + serviceTagsFileName);
+ }
+ }
+ }
+
+ if (fileURL != null) {
+
+ try {
+ serviceTagsFileStream = fileURL.openStream();
+ serviceTagsFileURL = fileURL;
+ } catch (Exception exception) {
+ LOG.error(serviceTagsFileName + " is not a file", exception);
+ }
+ } else {
+ LOG.warn("Error processing input file: URL not found for " + serviceTagsFileName + " or no privilege for reading file " + serviceTagsFileName);
+ }
+ }
+
+ } else {
+ LOG.error("FATAL: Cannot find service/serviceDef/serviceTagsFile to use for retrieving tags. Will NOT be able to retrieve tags.");
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== init() : serviceTagsFileName=" + serviceTagsFileName);
+ }
+ }
+
+ @Override
+ public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> retrieveTags(lastKnownVersion=" + lastKnownVersion + ", serviceTagsFilePath=" + serviceTagsFileName);
+ }
+
+ ServiceTags serviceTags = null;
+
+ if (serviceTagsFileURL != null) {
+ try {
+ InputStream serviceTagsFileStream = serviceTagsFileURL.openStream();
+ Reader reader = new InputStreamReader(serviceTagsFileStream);
+
+ Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z")
+ .setPrettyPrinting()
+ .create();
+
+ serviceTags = gsonBuilder.fromJson(reader, ServiceTags.class);
+
+ if (serviceTags.getTagVersion() <= lastKnownVersion) {
+ // No change in serviceTags
+ serviceTags = null;
+ }
+ } catch (Exception e) {
+ LOG.warn("Error processing input file: or no privilege for reading file " + serviceTagsFileName);
+ }
+ } else {
+ LOG.error("Error reading file: " + serviceTagsFileName);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== retrieveTags(lastKnownVersion=" + lastKnownVersion);
+ }
+
+ return serviceTags;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index e9fc42c..c8fc056 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -304,15 +304,16 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
try {
- populateTags();
-
+ // Sleep first and then fetch tags
if (pollingIntervalMs > 0) {
Thread.sleep(pollingIntervalMs);
} else {
break;
}
+ populateTags();
+
} catch (InterruptedException excp) {
- LOG.info("RangerTagRefresher(pollingIntervalMs=" + pollingIntervalMs + ").run() : interrupted! Exiting thread", excp);
+ LOG.debug("RangerTagRefresher(pollingIntervalMs=" + pollingIntervalMs + ").run() : interrupted! Exiting thread", excp);
break;
}
}
@@ -329,7 +330,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_ENRICHER_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_ENRICHER_LOG, "RangerTagRefresher.populateTags(serviceName=" + tagRetriever.getServiceName() + ",lastKnownVersion" + lastKnownVersion + ")");
+ perf = RangerPerfTracer.getPerfTracer(PERF_ENRICHER_LOG, "RangerTagRefresher.populateTags(serviceName=" + tagRetriever.getServiceName() + ",lastKnownVersion=" + lastKnownVersion + ")");
}
serviceTags = tagRetriever.retrieveTags(lastKnownVersion);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
new file mode 100644
index 0000000..72da8e8
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.util;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class PerfDataRecorder {
+ private static final Log LOG = LogFactory.getLog(PerfDataRecorder.class);
+ private static final Log PERF = RangerPerfTracer.getPerfLogger(PerfDataRecorder.class);
+
+ static PerfDataRecorder instance = null;
+ private Map<String, PerfStatistic> perfStatistics = new HashMap<String, PerfStatistic>();
+
+ public static void initialize(List<String> names) {
+ if (getPerfDataRecorder() == null) {
+ instance = new PerfDataRecorder();
+ }
+ instance.init(names);
+ }
+
+ public static PerfDataRecorder getPerfDataRecorder() {
+ return instance;
+ }
+
+ public void dumpStatistics() {
+ for (Map.Entry<String, PerfStatistic> entry : perfStatistics.entrySet()) {
+
+ String tag = entry.getKey();
+ PerfStatistic perfStatistic = entry.getValue();
+
+ long averageTimeSpent = 0L;
+ long minTimeSpent = 0L;
+ long maxTimeSpent = 0L;
+ if (perfStatistic.numberOfInvocations.get() != 0L) {
+ averageTimeSpent = perfStatistic.millisecondsSpent.get()/perfStatistic.numberOfInvocations.get();
+ minTimeSpent = perfStatistic.minTimeSpent.get();
+ maxTimeSpent = perfStatistic.maxTimeSpent.get();
+ }
+
+ String logMsg = "[" + tag + "]" +
+ " execCount:" + perfStatistic.numberOfInvocations +
+ ", totalTimeTaken:" + perfStatistic.millisecondsSpent +
+ ", maxTimeTaken:" + maxTimeSpent +
+ ", minTimeTaken:" + minTimeSpent +
+ ", avgTimeTaken:" + averageTimeSpent;
+
+ LOG.info(logMsg);
+ PERF.debug(logMsg);
+ }
+ }
+
+ void record(String tag, long elapsedTime) {
+ PerfStatistic perfStatistic = perfStatistics.get(tag);
+ if (perfStatistic != null) {
+ perfStatistic.addPerfDataItem(elapsedTime);
+ }
+ }
+
+ private void init(List<String> names) {
+ if (CollectionUtils.isNotEmpty(names)) {
+ for (String name : names) {
+ // Create structure
+ perfStatistics.put(name, new PerfStatistic());
+ }
+ }
+ }
+
+ private class PerfStatistic {
+ private AtomicLong numberOfInvocations = new AtomicLong(0L);
+ private AtomicLong millisecondsSpent = new AtomicLong(0L);
+ private AtomicLong minTimeSpent = new AtomicLong(Long.MAX_VALUE);
+ private AtomicLong maxTimeSpent = new AtomicLong(Long.MIN_VALUE);
+
+ void addPerfDataItem(final long timeTaken) {
+ numberOfInvocations.getAndIncrement();
+ millisecondsSpent.getAndAdd(timeTaken);
+
+ long min = minTimeSpent.get();
+ if (timeTaken < min) {
+ minTimeSpent.compareAndSet(min, timeTaken);
+ }
+
+ long max = maxTimeSpent.get();
+ if (timeTaken > max) {
+ maxTimeSpent.compareAndSet(max, timeTaken);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
new file mode 100644
index 0000000..e7b3865
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.util;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+
+public class RangerPerfCollectorTracer extends RangerPerfTracer {
+ private final PerfDataRecorder recorder;
+
+ public RangerPerfCollectorTracer(Log logger, String tag, String data, PerfDataRecorder recorder) {
+ super(logger, tag, data);
+ this.recorder = recorder;
+ }
+
+ @Override
+ public void log() {
+ recorder.record(tag, getElapsedTime());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
index fc84bcd..175c4e4 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
@@ -24,10 +24,13 @@ import org.apache.commons.logging.LogFactory;
import org.apache.commons.lang.StringUtils;
public class RangerPerfTracer {
- private final Log logger;
- private final String tag;
+ protected final Log logger;
+ protected final String tag;
+ protected final String data;
private final long startTimeMs;
+ private final static String tagEndMarker = "(";
+
public static Log getPerfLogger(String name) {
return LogFactory.getLog("ranger.perf." + name);
}
@@ -37,15 +40,27 @@ public class RangerPerfTracer {
}
public static boolean isPerfTraceEnabled(Log logger) {
- return logger.isInfoEnabled();
+ return logger.isDebugEnabled();
}
public static RangerPerfTracer getPerfTracer(Log logger, String tag) {
- return logger.isInfoEnabled() ? new RangerPerfTracer(logger, tag) : null;
+ String data = "";
+ String realTag = "";
+
+ if (tag != null) {
+ int indexOfTagEndMarker = StringUtils.indexOf(tag, tagEndMarker);
+ if (indexOfTagEndMarker != -1) {
+ realTag = StringUtils.substring(tag, 0, indexOfTagEndMarker);
+ data = StringUtils.substring(tag, indexOfTagEndMarker);
+ } else {
+ realTag = tag;
+ }
+ }
+ return RangerPerfTracerFactory.getPerfTracer(logger, realTag, data);
}
- public static RangerPerfTracer getPerfTracer(Log logger, Object... tagParts) {
- return logger.isInfoEnabled() ? new RangerPerfTracer(logger, StringUtils.join(tagParts)) : null;
+ public static RangerPerfTracer getPerfTracer(Log logger, String tag, String data) {
+ return RangerPerfTracerFactory.getPerfTracer(logger, tag, data);
}
public static void log(RangerPerfTracer tracer) {
@@ -54,9 +69,10 @@ public class RangerPerfTracer {
}
}
- public RangerPerfTracer(Log logger, String tag) {
+ public RangerPerfTracer(Log logger, String tag, String data) {
this.logger = logger;
this.tag = tag;
+ this.data = data;
startTimeMs = System.currentTimeMillis();
}
@@ -73,8 +89,8 @@ public class RangerPerfTracer {
}
public void log() {
- if(logger.isInfoEnabled()) {
- logger.info("[PERF] " + tag + ": " + getElapsedTime());
+ if(logger.isDebugEnabled()) {
+ logger.debug("[PERF] " + tag + data + ": " + getElapsedTime());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java
new file mode 100644
index 0000000..8db2d45
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.util;
+
+import org.apache.commons.logging.Log;
+
+public class RangerPerfTracerFactory {
+
+ private static PerfDataRecorder perfDataRecorder = PerfDataRecorder.getPerfDataRecorder();
+
+ static RangerPerfTracer getPerfTracer(Log logger, String tag, String data) {
+
+ RangerPerfTracer ret = null;
+
+ if (perfDataRecorder != null) {
+ ret = new RangerPerfCollectorTracer(logger, tag, data, perfDataRecorder);
+ } else if (logger.isDebugEnabled()) {
+ ret = new RangerPerfTracer(logger, tag, data);
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 108bd17..23ef49e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -107,6 +107,7 @@
<module>ranger-storm-plugin-shim</module>
<module>ranger-kafka-plugin-shim</module>
<module>ranger-examples</module>
+ <module>ranger-tools</module>
</modules>
<properties>
<javac.source.version>1.7</javac.source.version>
@@ -409,6 +410,7 @@
<descriptor>src/main/assembly/tagsync.xml</descriptor>
<descriptor>src/main/assembly/migration-util.xml</descriptor>
<descriptor>src/main/assembly/kms.xml</descriptor>
+ <descriptor>src/main/assembly/ranger-tools.xml</descriptor>
<descriptor>src/main/assembly/ranger-src.xml</descriptor>
</descriptors>
</configuration>
@@ -514,6 +516,7 @@
<exclude>**/samples/**/*.json</exclude>
<exclude>**/.externalToolBuilders/*</exclude>
<exclude>*.patch</exclude>
+ <exclude>**/testdata/*.json</exclude>
<exclude>atlassian-ide-plugin.xml</exclude>
<exclude>**/.pydevproject</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/.gitignore
----------------------------------------------------------------------
diff --git a/ranger-tools/.gitignore b/ranger-tools/.gitignore
new file mode 100644
index 0000000..5ac84b1
--- /dev/null
+++ b/ranger-tools/.gitignore
@@ -0,0 +1,2 @@
+/target/
+ranger-perftester.iml
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/ranger-tools/conf/log4j.properties b/ranger-tools/conf/log4j.properties
new file mode 100644
index 0000000..ccb9db4
--- /dev/null
+++ b/ranger-tools/conf/log4j.properties
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##-- To prevent junits from cluttering the build run by default all test runs send output to null appender
+log4j.appender.devnull=org.apache.log4j.varia.NullAppender
+# ranger.root.logger=FATAL,devnull
+
+##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console
+ranger.root.logger=INFO,console
+
+log4j.rootLogger=${ranger.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %L %m%n
+
+#
+# ranger.perf log level
+#
+ranger.perf.logger=DEBUG,PERF
+ranger.perf.log.file=ranger-perf-test.log
+
+log4j.logger.ranger.perf=${ranger.perf.logger}
+log4j.additivity.ranger.perf=false
+
+log4j.appender.PERF=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.PERF.File=${ranger.perf.log.file}
+log4j.appender.PERF.layout=org.apache.log4j.PatternLayout
+log4j.appender.PERF.layout.ConversionPattern=%m%n
+log4j.appender.PERF.DatePattern=.yyyy-MM-dd
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/pom.xml
----------------------------------------------------------------------
diff --git a/ranger-tools/pom.xml b/ranger-tools/pom.xml
new file mode 100644
index 0000000..f7347f7
--- /dev/null
+++ b/ranger-tools/pom.xml
@@ -0,0 +1,69 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>ranger</artifactId>
+ <groupId>org.apache.ranger</groupId>
+ <version>0.5.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>ranger-tools</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Ranger Tools</name>
+ <url>http://maven.apache.org</url>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>${commons.logging.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons.lang.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>security_plugins.ranger-plugins-common</groupId>
+ <artifactId>ranger-plugins-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/scripts/README.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt
new file mode 100644
index 0000000..9eb71ba
--- /dev/null
+++ b/ranger-tools/scripts/README.txt
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+This file describes how to build, unpackage and run the performance testing tool.
+
+1. Build Apache Ranger using the following command.
+ % mvn clean compile package assembly:assembly
+
+ The following artifact will be created under target directory.
+
+ target/ranger-0.5.0-ranger-tools.tar.gz
+
+2. Copy this artifact to the directory where you want to run the tool.
+
+ % cp target/ranger-0.5.0-ranger-tools.tar.gz <perf-tool-run-dir>
+ % cd <perf-tool-run-dir>
+
+3. Unzip the artifact.
+
+ % tar xvfz ranger-0.5.0-ranger-tools.tar.gz
+
+ This will create the following directory structure under <perf-tool-run-dir>
+
+ ranger-0.5.0-ranger-tools
+ ranger-0.5.0-ranger-tools/conf
+ ranger-0.5.0-ranger-tools/dist
+ ranger-0.5.0-ranger-tools/lib
+ ranger-0.5.0-ranger-tools/scripts
+ ranger-0.5.0-ranger-tools/testdata
+
+4. % cd ranger-0.5.0-ranger-tools
+
+5. Setup configuration
+
+ Following sample data files are packaged with the perf-tool:
+ service-policies - testdata/test_servicepolicies_hive.json
+ requests - testdata/test_requests_hive.json
+ modules-to-profile - testdata/test_modules.txt
+ service-tags - testdata/test_servicetags_hive.json (used only for tag-based policies; referenced from service-policies file.
+
+ Please review the contents of these files and modify (or copy/modify) to suite your policy and request needs.
+
+ Update conf/log4j.properties to specify the filename where perf run results will be written to. Property to update is 'ranger.perf.logger'.
+
+5. Run the tool with the command,
+
+ % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -p <profiled-modules-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run>
+
+ Example:
+ % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -p testdata/test_modules.txt -c 2 -n 1
+
+6. At the end of the run, the performance-statistics are printed in the log file in conf/log4j.properties file.
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/scripts/ranger-perftester.sh
----------------------------------------------------------------------
diff --git a/ranger-tools/scripts/ranger-perftester.sh b/ranger-tools/scripts/ranger-perftester.sh
new file mode 100755
index 0000000..46c8e0e
--- /dev/null
+++ b/ranger-tools/scripts/ranger-perftester.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cdir=$(cd "$(dirname "$0")"; pwd)
+cp="${cdir}/dist/*:${cdir}/lib/*:${cdir}/conf:."
+
+if [ "${JAVA_HOME}" != "" ]
+then
+ export JAVA_HOME
+ PATH="${JAVA_HOME}/bin:${PATH}"
+ export PATH
+fi
+
+JAVA_CMD="java -cp ${cp} org.apache.ranger.policyengine.RangerPolicyenginePerfTester"
+
+cd ${cdir}
+
+echo "JAVA command = $JAVA_CMD " "$@"
+$JAVA_CMD "$@"
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/scripts/summary.awk
----------------------------------------------------------------------
diff --git a/ranger-tools/scripts/summary.awk b/ranger-tools/scripts/summary.awk
new file mode 100755
index 0000000..16e49f3
--- /dev/null
+++ b/ranger-tools/scripts/summary.awk
@@ -0,0 +1,17 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+awk '{ORS=" "; print $NF; for (i=6; i<NF; i++) print $i;print "\n" }' $1 | sort -n -r
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
new file mode 100644
index 0000000..a45d71a
--- /dev/null
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.policyengine;
+
+import org.apache.commons.cli.*;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Arrays;
+
+public class CommandLineParser
+{
+ static final Log LOG = LogFactory.getLog(CommandLineParser.class);
+
+ private String servicePoliciesFileName;
+ private String[] requestFileNames;
+ private String statCollectionFileName;
+
+ private URL servicePoliciesFileURL;
+ private URL[] requestFileURLs;
+ private URL statCollectionFileURL;
+
+
+ private int concurrentClientCount = 1;
+ private int iterationsCount = 1;
+
+ private Options options = new Options();
+
+ CommandLineParser() {}
+
+ final PerfTestOptions parse(final String[] args) {
+ PerfTestOptions ret = null;
+ if (parseArguments(args) && validateInputFiles()) {
+ // Instantiate a data-object and return
+ ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount);
+ } else {
+ showUsage(-1);
+ }
+ return ret;
+ }
+
+ // Parse the arguments
+
+ /* Arguments :
+ -s servicePolicies-file-name
+ -c concurrent-client-count
+ -r request-file-name-list
+ -n number-of-iterations
+ -p modules-to-collect-stats
+
+ If the concurrent-client-count is more than the number of files in the request-file-name-list,
+ then reuse the request-file-names in a round-robin way
+
+ */
+
+ final boolean parseArguments(final String[] args) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> parseArguments()");
+ }
+ boolean ret = false;
+
+ options.addOption("h", "help", false, "show help.");
+ options.addOption("s", "service-policies", true, "Policies File Name");
+ options.addOption("r", "requests", true, "Request Definition File Name");
+ options.addOption("p", "statistics", true, "Modules for stat collection File Name");
+ options.addOption("c", "clients", true, "Number of concurrent clients");
+ options.addOption("n", "cycles", true, "Number of iterations");
+
+ org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser();
+
+ try {
+ CommandLine commandLine = commandLineParser.parse(options, args);
+
+ if (commandLine.hasOption("h")) {
+ showUsage(0);
+ }
+
+ servicePoliciesFileName = commandLine.getOptionValue("s");
+ requestFileNames = commandLine.getOptionValues("r");
+ statCollectionFileName = commandLine.getOptionValue("p");
+
+ concurrentClientCount = 1;
+ String clientOptionValue = commandLine.getOptionValue("c");
+ if (clientOptionValue != null) {
+ concurrentClientCount = Integer.parseInt(clientOptionValue);
+ }
+
+ iterationsCount = 1;
+ String iterationsOptionValue = commandLine.getOptionValue("n");
+ if (iterationsOptionValue != null) {
+ iterationsCount = Integer.parseInt(iterationsOptionValue);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames));
+ LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount);
+ }
+
+ ret = true;
+ } catch (Exception exception) {
+ LOG.error("Error processing command-line arguments: ", exception);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== parseArguments() : " + ret);
+ }
+
+ return ret;
+ }
+
+ final boolean validateInputFiles() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> validateInputFiles()");
+ }
+
+ boolean ret = false;
+
+ if (servicePoliciesFileName != null) {
+ this.servicePoliciesFileURL = getInputFileURL(servicePoliciesFileName);
+ if (servicePoliciesFileURL != null) {
+ if (requestFileNames != null) {
+ if (validateRequestFiles()) {
+ if (statCollectionFileName != null) {
+ statCollectionFileURL = getInputFileURL(statCollectionFileName);
+ ret = statCollectionFileURL != null;
+ } else {
+ LOG.error("Error processing stat-collection-module file");
+ }
+ }
+ } else {
+ LOG.error("Error processing requests file: No requests files provided.");
+ }
+ } else {
+ LOG.error("Error processing service-policies file: unreadable service-policies file: " + servicePoliciesFileName);
+ }
+ } else {
+ LOG.error("Error processing service-policies file: null service-policies file");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== validateInputFiles(): " + ret);
+ }
+
+ return ret;
+ }
+
+ final boolean validateRequestFiles() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> validateRequestFiles()");
+ }
+ boolean ret = requestFileNames.length > 0;
+
+ if (ret) {
+ requestFileURLs = new URL[requestFileNames.length];
+
+ for (int i = 0; ret && i < requestFileNames.length; i++) {
+ if (requestFileNames[i] != null) {
+ if ((requestFileURLs[i] = getInputFileURL(requestFileNames[i])) == null) {
+ LOG.error("Cannot read file: " + requestFileNames[i]);
+ ret = false;
+ }
+ } else {
+ LOG.error("Error processing request-file: null input file-name for request-file");
+ ret = false;
+ }
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== validateRequestFiles(): " + ret);
+ }
+ return ret;
+ }
+
+ public static URL getInputFileURL(final String name) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> getResourceFileURL(" + name + ")");
+ }
+ URL ret = null;
+ InputStream in = null;
+
+
+ if (StringUtils.isNotBlank(name)) {
+
+ File f = new File(name);
+
+ if (f.exists() && f.isFile() && f.canRead()) {
+ try {
+
+ in = new FileInputStream(f);
+ ret = f.toURI().toURL();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("URL:" + ret);
+ }
+
+ } catch (FileNotFoundException exception) {
+ LOG.error("Error processing input file:" + name + " or no privilege for reading file " + name, exception);
+ } catch (MalformedURLException malformedException) {
+ LOG.error("Error processing input file:" + name + " cannot be converted to URL " + name, malformedException);
+ }
+ } else {
+
+ URL fileURL = CommandLineParser.class.getResource(name);
+ if (fileURL == null) {
+ if (!name.startsWith("/")) {
+ fileURL = CommandLineParser.class.getResource("/" + name);
+ }
+ }
+
+ if (fileURL == null) {
+ fileURL = ClassLoader.getSystemClassLoader().getResource(name);
+ if (fileURL == null) {
+ if (!name.startsWith("/")) {
+ fileURL = ClassLoader.getSystemClassLoader().getResource("/" + name);
+ }
+ }
+ }
+
+ if (fileURL != null) {
+ try {
+ in = fileURL.openStream();
+ ret = fileURL;
+ } catch (Exception exception) {
+ LOG.error(name + " cannot be opened:", exception);
+ }
+ } else {
+ LOG.warn("Error processing input file: URL not found for " + name + " or no privilege for reading file " + name);
+ }
+ }
+ }
+ if (in != null) {
+ try {
+ in.close();
+ } catch (Exception e) {
+ // Ignore
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== getResourceFileURL(" + name + ", URL=" + ret + ")");
+ }
+ return ret;
+ }
+
+ void showUsage(int exitCode) {
+ HelpFormatter formater = new HelpFormatter();
+ formater.printHelp("perfTester", options);
+
+ LOG.info("Exiting...");
+
+ System.exit(exitCode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java
new file mode 100644
index 0000000..b88d670
--- /dev/null
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestClient.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.policyengine;
+
+import com.google.gson.*;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.policyengine.*;
+
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.lang.reflect.Type;
+import java.net.URL;
+import java.util.List;
+
+public class PerfTestClient extends Thread {
+ static final Log LOG = LogFactory.getLog(PerfTestClient.class);
+
+ final PerfTestEngine perfTestEngine;
+ final int clientId;
+ final URL requestFileURL;
+ final int maxCycles;
+
+ List<RequestData> requests = null;
+ static Gson gsonBuilder = null;
+
+ static {
+
+ gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z")
+ .setPrettyPrinting()
+ .registerTypeAdapter(RangerAccessRequest.class, new RangerAccessRequestDeserializer())
+ .registerTypeAdapter(RangerAccessResource.class, new RangerResourceDeserializer())
+ .create();
+ }
+
+ public PerfTestClient(final PerfTestEngine perfTestEngine, final int clientId, final URL requestFileURL, final int maxCycles) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> PerfTestClient(clientId=" + clientId + ", maxCycles=" + maxCycles +")" );
+ }
+
+ this.perfTestEngine = perfTestEngine;
+ this.clientId = clientId;
+ this.requestFileURL = requestFileURL;
+ this.maxCycles = maxCycles;
+
+ setName("PerfTestClient-" + clientId);
+ setDaemon(true);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== PerfTestClient(clientId=" + clientId + ", maxCycles=" + maxCycles +")" );
+ }
+ }
+
+ public boolean init() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> init()" );
+ }
+
+ boolean ret = false;
+
+ Reader reader = null;
+
+ try {
+
+ InputStream in = requestFileURL.openStream();
+
+ reader = new InputStreamReader(in);
+
+ Type listType = new TypeToken<List<RequestData>>() {
+ }.getType();
+
+ requests = gsonBuilder.fromJson(reader, listType);
+
+ ret = true;
+ }
+ catch (Exception excp) {
+ LOG.error("Error opening request data stream or loading load request data from file, URL=" + requestFileURL, excp);
+ }
+ finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (Exception excp) {
+ LOG.error("Error closing file ", excp);
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== init() : " + ret );
+ }
+ return ret;
+ }
+
+ @Override
+ public void run() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> run()" );
+ }
+
+ try {
+ for (int i = 0; i < maxCycles; i++) {
+ for (RequestData data : requests) {
+ perfTestEngine.execute(data.request);
+ }
+ }
+ } catch (Exception excp) {
+ LOG.error("PerfTestClient.run() : interrupted! Exiting thread", excp);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== run()" );
+ }
+ }
+
+ private class RequestData {
+ public String name;
+ public RangerAccessRequest request;
+ public RangerAccessResult result;
+ }
+
+ static class RangerAccessRequestDeserializer implements JsonDeserializer<RangerAccessRequest> {
+ @Override
+ public RangerAccessRequest deserialize(JsonElement jsonObj, Type type,
+ JsonDeserializationContext context) throws JsonParseException {
+ RangerAccessRequestImpl ret = gsonBuilder.fromJson(jsonObj, RangerAccessRequestImpl.class);
+
+ ret.setAccessType(ret.getAccessType()); // to force computation of isAccessTypeAny and isAccessTypeDelegatedAdmin
+
+ return ret;
+ }
+ }
+
+ static class RangerResourceDeserializer implements JsonDeserializer<RangerAccessResource> {
+ @Override
+ public RangerAccessResource deserialize(JsonElement jsonObj, Type type,
+ JsonDeserializationContext context) throws JsonParseException {
+ return gsonBuilder.fromJson(jsonObj, RangerAccessResourceImpl.class);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
new file mode 100644
index 0000000..1b5f6fd
--- /dev/null
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.policyengine;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler;
+import org.apache.ranger.plugin.policyengine.*;
+import org.apache.ranger.plugin.util.ServicePolicies;
+
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URL;
+
+public class PerfTestEngine {
+ static final Log LOG = LogFactory.getLog(PerfTestEngine.class);
+
+ private final URL servicePoliciesFileURL;
+ private RangerPolicyEngine policyEvaluationEngine;
+
+ public PerfTestEngine(final URL servicePoliciesFileURL) {
+ this.servicePoliciesFileURL = servicePoliciesFileURL;
+ }
+
+ public boolean init() {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> init()");
+ }
+
+ boolean ret = false;
+
+ Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z")
+ .setPrettyPrinting()
+ .create();
+
+ Reader reader = null;
+ ServicePolicies servicePolicies;
+
+ try {
+ InputStream in = servicePoliciesFileURL.openStream();
+
+ reader = new InputStreamReader(in);
+
+ servicePolicies = gsonBuilder.fromJson(reader, ServicePolicies.class);
+
+ RangerPolicyEngineOptions engineOptions = new RangerPolicyEngineOptions();
+ engineOptions.disableTagPolicyEvaluation = false;
+
+ policyEvaluationEngine = new RangerPolicyEngineImpl("perf-test", servicePolicies, engineOptions);
+
+ ret = true;
+
+ } catch (Exception excp) {
+ LOG.error("Error opening service-policies file or loading service-policies from file, URL=" + servicePoliciesFileURL, excp);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (Exception excp) {
+ LOG.error("Error closing file", excp);
+ }
+ }
+ }
+
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== init() : " + ret);
+ }
+
+ return ret;
+
+ }
+ public boolean execute(final RangerAccessRequest request) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> execute(" + request + ")");
+ }
+
+ boolean ret = true;
+
+ if (policyEvaluationEngine != null) {
+
+ RangerAccessResultProcessor auditHandler = null;
+
+ policyEvaluationEngine.preProcess(request);
+
+ RangerAccessResult result = policyEvaluationEngine.isAccessAllowed(request, auditHandler);
+ } else {
+ LOG.error("Error executing request: PolicyEngine is null!");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== execute(" + request + ") : " + ret);
+ }
+
+ return ret;
+ }
+
+ public void cleanup() {
+ if (policyEvaluationEngine != null) {
+ policyEvaluationEngine.cleanup();
+ policyEvaluationEngine = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
new file mode 100644
index 0000000..f30cbd7
--- /dev/null
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.policyengine;
+
+
+import java.net.URL;
+
+public class PerfTestOptions {
+ private final URL servicePoliciesFileURL;
+ private final URL[] requestFileURLs;
+ private final URL statCollectionFileURL;
+
+
+ private final int concurrentClientCount;
+ private final int iterationsCount;
+
+ PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount) {
+ this.servicePoliciesFileURL = servicePoliciesFileURL;
+ this.requestFileURLs = requestFileURLs;
+ this.statCollectionFileURL = statCollectionFileURL;
+ this.iterationsCount = iterationsCount;
+ this.concurrentClientCount = concurrentClientCount;
+ }
+
+ public URL getServicePoliciesFileURL() {
+ return this.servicePoliciesFileURL;
+ }
+
+ public URL[] getRequestFileURLs() {
+ return this.requestFileURLs;
+ }
+
+ public URL getStatCollectionFileURL() {
+ return this.statCollectionFileURL;
+ }
+
+ public int getConcurrentClientCount() {
+ return concurrentClientCount;
+ }
+
+ public int getIterationsCount() {
+ return iterationsCount;
+ }}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
new file mode 100644
index 0000000..28cc558
--- /dev/null
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.policyengine;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.util.PerfDataRecorder;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+
+public class RangerPolicyenginePerfTester {
+ static final Log LOG = LogFactory.getLog(RangerPolicyenginePerfTester.class);
+
+ public static void main(String[] args) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerPolicyenginePerfTester.main()");
+ }
+
+ CommandLineParser commandLineParser = new CommandLineParser();
+
+ PerfTestOptions perfTestOptions = commandLineParser.parse(args);
+
+ URL statCollectionFileURL = perfTestOptions.getStatCollectionFileURL();
+
+ List<String> perfModuleNames = buildPerfModuleNames(statCollectionFileURL);
+
+ PerfDataRecorder.initialize(perfModuleNames);
+
+ URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL();
+
+ PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL);
+ if (!perfTestEngine.init()) {
+ LOG.error("Error initializing test data. Existing...");
+ System.exit(1);
+ }
+
+ URL[] requestFileURLs = perfTestOptions.getRequestFileURLs();
+ int requestFilesCount = requestFileURLs.length;
+
+ int clientsCount = perfTestOptions.getConcurrentClientCount();
+ List<PerfTestClient> perfTestClients = new ArrayList<PerfTestClient>(clientsCount);
+
+ for (int i = 0; i < clientsCount; i++) {
+
+ URL requestFileURL = requestFileURLs[i % requestFilesCount];
+
+ PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, i, requestFileURL, perfTestOptions.getIterationsCount());
+
+ if (!perfTestClient.init()) {
+ LOG.error("Error initializing PerfTestClient: (id=" + i + ")");
+ } else {
+ perfTestClients.add(perfTestClient);
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Number of perfTestClients=" + perfTestClients.size());
+ }
+
+ for (PerfTestClient client : perfTestClients) {
+ try {
+ client.start();
+ } catch (Throwable t) {
+ LOG.error("Error in starting client: " + client.getName(), t);
+ }
+ }
+
+ LOG.info("Waiting for " + perfTestClients.size() + " clients to finish up");
+
+ for (PerfTestClient client : perfTestClients) {
+ try {
+ if (client.isAlive()) {
+ LOG.info("Waiting for " + client.getName() + " to finish up.");
+ client.join();
+ }
+ } catch (InterruptedException interruptedException) {
+ LOG.error("PerfTestClient.join() was interrupted");
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerPolicyenginePerfTester.main()");
+ }
+
+ LOG.info("Completed performance-run");
+
+ perfTestEngine.cleanup();
+
+ PerfDataRecorder.getPerfDataRecorder().dumpStatistics();
+ }
+
+ private static List<String> buildPerfModuleNames(URL statCollectionFileURL) {
+ List<String> perfModuleNames = new ArrayList<String>();
+
+ try (
+ InputStream inStream = statCollectionFileURL.openStream();
+ InputStreamReader reader = new InputStreamReader(inStream, Charset.forName("UTF-8"));
+ BufferedReader br = new BufferedReader(reader);
+ ) {
+
+ String line;
+
+ while ((line = br.readLine()) != null) {
+ line = line.trim();
+ if (!line.isEmpty() && !line.startsWith("#")) {
+ String[] moduleNames = line.split(" ");
+ for (int i = 0; i < moduleNames.length; i++) {
+ perfModuleNames.add(moduleNames[i]);
+ }
+ }
+ }
+ } catch (Exception exception) {
+ System.out.println("Error reading arguments:" + exception);
+ }
+
+ return perfModuleNames;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/java/org/apache/ranger/policyengine/PerfTesterTest.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/java/org/apache/ranger/policyengine/PerfTesterTest.java b/ranger-tools/src/test/java/org/apache/ranger/policyengine/PerfTesterTest.java
new file mode 100644
index 0000000..2d7c52e
--- /dev/null
+++ b/ranger-tools/src/test/java/org/apache/ranger/policyengine/PerfTesterTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.policyengine;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.Charset;
+
+/**
+ * Unit test for simple App.
+ */
+public class PerfTesterTest
+ extends TestCase
+
+{
+ /**
+ * Create the test case
+ *
+ * @param testName name of the test case
+ */
+ public PerfTesterTest( String testName )
+ {
+ super( testName );
+ }
+
+ /**
+ * @return the suite of tests being tested
+ */
+ public static Test suite()
+ {
+ return new TestSuite( PerfTesterTest.class );
+ }
+
+ /**
+ * Rigourous Test :-)
+ */
+
+ public void testMain() {
+
+ String[] args = readCommandLine();
+
+ if (args != null) {
+ RangerPolicyenginePerfTester.main(args);
+ }
+ }
+
+
+ public void testArgParsing() {
+ String[] args = readCommandLine();
+
+ if (args != null) {
+ CommandLineParser commandLineParser = new CommandLineParser();
+ PerfTestOptions parseResult = commandLineParser.parse(args);
+ assertNotNull(parseResult);
+ }
+ }
+
+ String[] readCommandLine() {
+ // Read arguments from a file - with hardcoded name 'commandline'
+
+ String[] ret = null;
+
+ URL commandLineFileURL = CommandLineParser.getInputFileURL("/commandline");
+ if (commandLineFileURL != null) {
+ try (
+ InputStream inStream = commandLineFileURL.openStream();
+ InputStreamReader reader = new InputStreamReader(inStream, Charset.forName("UTF-8"));
+ BufferedReader br = new BufferedReader(reader);
+ ) {
+
+
+ String line;
+
+ while ((line = br.readLine()) != null) {
+ line = line.trim();
+ if (!line.isEmpty() && !line.startsWith("#")) {
+ ret = line.split(" ");
+ break;
+ }
+ }
+
+ } catch (Exception exception) {
+ System.out.println("Error reading arguments:" + exception);
+ }
+ }
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/commandline
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/commandline b/ranger-tools/src/test/resources/commandline
new file mode 100644
index 0000000..9ea690e
--- /dev/null
+++ b/ranger-tools/src/test/resources/commandline
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+-s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/log4j.properties b/ranger-tools/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a9a8881
--- /dev/null
+++ b/ranger-tools/src/test/resources/log4j.properties
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##-- To prevent junits from cluttering the build run by default all test runs send output to null appender
+log4j.appender.devnull=org.apache.log4j.varia.NullAppender
+# ranger.root.logger=FATAL,devnull
+
+##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console
+ranger.root.logger=INFO,console
+
+log4j.rootLogger=${ranger.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %L %m%n
+
+#
+# ranger.perf log level
+#
+ranger.perf.logger=DEBUG,PERF
+ranger.perf.log.file=/tmp/ranger-perf-test.log
+
+log4j.logger.ranger.perf=${ranger.perf.logger}
+log4j.additivity.ranger.perf=false
+
+log4j.appender.PERF=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.PERF.File=${ranger.perf.log.file}
+log4j.appender.PERF.layout=org.apache.log4j.PatternLayout
+log4j.appender.PERF.layout.ConversionPattern=%m%n
+log4j.appender.PERF.DatePattern=.yyyy-MM-dd
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/testdata/test_modules.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/testdata/test_modules.txt b/ranger-tools/src/test/resources/testdata/test_modules.txt
new file mode 100644
index 0000000..8eb5746
--- /dev/null
+++ b/ranger-tools/src/test/resources/testdata/test_modules.txt
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+RangerTagEnricher.setServiceTags
+RangerPolicyEngine.init
+RangerPolicyEngine.preProcess
+RangerPolicyEngine.isAccessAllowedNoAudit
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/testdata/test_requests_hive.json
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/testdata/test_requests_hive.json b/ranger-tools/src/test/resources/testdata/test_requests_hive.json
new file mode 100644
index 0000000..0db7207
--- /dev/null
+++ b/ranger-tools/src/test/resources/testdata/test_requests_hive.json
@@ -0,0 +1,10 @@
+ [
+ {"name":"'select default/tbl-0/col-2;' for hrt_1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"tbl-0","column":"col-2"}},
+ "accessType":"select","user":"hrt_1","userGroups":[],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ]
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/testdata/test_servicepolicies_hive.json
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/testdata/test_servicepolicies_hive.json b/ranger-tools/src/test/resources/testdata/test_servicepolicies_hive.json
new file mode 100644
index 0000000..4d3a166
--- /dev/null
+++ b/ranger-tools/src/test/resources/testdata/test_servicepolicies_hive.json
@@ -0,0 +1,488 @@
+{
+ "serviceName": "cl1_hive",
+ "serviceId": 2,
+ "policies": [
+ {
+ "service": "cl1_hive",
+ "name": "cl1_hive-1-20151212014502",
+ "isAuditEnabled": false,
+ "resources": {
+ "database": {
+ "values": [
+ "*"
+ ],
+ "isExcludes": false,
+ "isRecursive": false
+ },
+ "column": {
+ "values": [
+ "*"
+ ],
+ "isExcludes": false,
+ "isRecursive": false
+ },
+ "table": {
+ "values": [
+ "*"
+ ],
+ "isExcludes": false,
+ "isRecursive": false
+ }
+ },
+ "policyItems": [
+ {
+ "accesses": [
+ {
+ "type": "select",
+ "isAllowed": true
+ },
+ {
+ "type": "update",
+ "isAllowed": true
+ },
+ {
+ "type": "create",
+ "isAllowed": true
+ },
+ {
+ "type": "drop",
+ "isAllowed": true
+ },
+ {
+ "type": "alter",
+ "isAllowed": true
+ },
+ {
+ "type": "index",
+ "isAllowed": true
+ },
+ {
+ "type": "lock",
+ "isAllowed": true
+ },
+ {
+ "type": "all",
+ "isAllowed": true
+ }
+ ],
+ "users": [
+ "ambari-qa"
+ ],
+ "groups": [],
+ "conditions": [],
+ "delegateAdmin": true,
+ "isEnabled": true
+ }
+ ],
+ "denyPolicyItems": [],
+ "allowExceptions": [],
+ "denyExceptions": [],
+ "id": 2,
+ "isEnabled": true
+ }
+ ],
+ "serviceDef": {
+ "name": "hive",
+ "implClass": "org.apache.ranger.services.hive.RangerServiceHive",
+ "label": "Hive Server2",
+ "options": {},
+ "configs": [
+ {
+ "itemId": 1,
+ "name": "username",
+ "type": "string",
+ "mandatory": true,
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": "",
+ "label": "Username"
+ },
+ {
+ "itemId": 2,
+ "name": "password",
+ "type": "password",
+ "mandatory": true,
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": "",
+ "label": "Password"
+ },
+ {
+ "itemId": 3,
+ "name": "jdbc.driverClassName",
+ "type": "string",
+ "mandatory": true,
+ "defaultValue": "org.apache.hive.jdbc.HiveDriver",
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": ""
+ },
+ {
+ "itemId": 4,
+ "name": "jdbc.url",
+ "type": "string",
+ "mandatory": true,
+ "defaultValue": "",
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": ""
+ },
+ {
+ "itemId": 5,
+ "name": "commonNameForCertificate",
+ "type": "string",
+ "mandatory": false,
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": "",
+ "label": "Common Name for Certificate"
+ }
+ ],
+ "resources": [
+ {
+ "itemId": 1,
+ "name": "database",
+ "type": "string",
+ "level": 10,
+ "mandatory": true,
+ "lookupSupported": true,
+ "recursiveSupported": false,
+ "excludesSupported": true,
+ "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher",
+ "matcherOptions": {
+ "wildCard": "true",
+ "ignoreCase": "true"
+ },
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": "",
+ "label": "Hive Database"
+ },
+ {
+ "itemId": 2,
+ "name": "table",
+ "type": "string",
+ "level": 20,
+ "parent": "database",
+ "mandatory": true,
+ "lookupSupported": true,
+ "recursiveSupported": false,
+ "excludesSupported": true,
+ "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher",
+ "matcherOptions": {
+ "wildCard": "true",
+ "ignoreCase": "true"
+ },
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": "",
+ "label": "Hive Table"
+ },
+ {
+ "itemId": 3,
+ "name": "udf",
+ "type": "string",
+ "level": 20,
+ "parent": "database",
+ "mandatory": true,
+ "lookupSupported": true,
+ "recursiveSupported": false,
+ "excludesSupported": true,
+ "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher",
+ "matcherOptions": {
+ "wildCard": "true",
+ "ignoreCase": "true"
+ },
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": "",
+ "label": "Hive UDF"
+ },
+ {
+ "itemId": 4,
+ "name": "column",
+ "type": "string",
+ "level": 30,
+ "parent": "table",
+ "mandatory": true,
+ "lookupSupported": true,
+ "recursiveSupported": false,
+ "excludesSupported": true,
+ "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher",
+ "matcherOptions": {
+ "wildCard": "true",
+ "ignoreCase": "true"
+ },
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": "",
+ "label": "Hive Column"
+ }
+ ],
+ "accessTypes": [
+ {
+ "itemId": 1,
+ "name": "select",
+ "label": "select",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 2,
+ "name": "update",
+ "label": "update",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 3,
+ "name": "create",
+ "label": "Create",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 4,
+ "name": "drop",
+ "label": "Drop",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 5,
+ "name": "alter",
+ "label": "Alter",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 6,
+ "name": "index",
+ "label": "Index",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 7,
+ "name": "lock",
+ "label": "Lock",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 8,
+ "name": "all",
+ "label": "All",
+ "impliedGrants": [
+ "select",
+ "update",
+ "create",
+ "drop",
+ "alter",
+ "index",
+ "lock"
+ ]
+ }
+ ],
+ "policyConditions": [
+ {
+ "itemId": 1,
+ "name": "resources-accessed-together",
+ "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerHiveResourcesAccessedTogetherCondition",
+ "evaluatorOptions": {},
+ "label": "Hive Resources Accessed Together?"
+ }
+ ],
+ "contextEnrichers": [],
+ "enums": [],
+ "id": 3,
+ "isEnabled": true
+ },
+ "tagPolicies": {
+ "serviceName": "tagdev",
+ "serviceId": 3,
+ "policyVersion": 1,
+ "policyUpdateTime": "20151212-01:48:43.000-+0000",
+ "policies": [
+ {
+ "service": "tagdev",
+ "name": "tagdev-EXPIRES_ON",
+ "isAuditEnabled": true,
+ "resources": {
+ "tag": {
+ "values": [
+ "EXPIRES_ON"
+ ],
+ "isExcludes": false,
+ "isRecursive": false
+ }
+ },
+ "policyItems": [],
+ "denyPolicyItems": [
+ {
+ "accesses": [
+ {
+ "type": "hive:select",
+ "isAllowed": true
+ },
+ {
+ "type": "hive:update",
+ "isAllowed": true
+ },
+ {
+ "type": "hive:create",
+ "isAllowed": true
+ },
+ {
+ "type": "hive:drop",
+ "isAllowed": true
+ },
+ {
+ "type": "hive:alter",
+ "isAllowed": true
+ },
+ {
+ "type": "hive:index",
+ "isAllowed": true
+ },
+ {
+ "type": "hive:lock",
+ "isAllowed": true
+ },
+ {
+ "type": "hive:all",
+ "isAllowed": true
+ }
+ ],
+ "users": [],
+ "groups": [
+ "public"
+ ],
+ "conditions": [
+ {
+ "type": "accessed-after-expiry",
+ "values": [
+ "yes"
+ ]
+ }
+ ],
+ "delegateAdmin": true,
+ "isEnabled": true
+ }
+ ],
+ "allowExceptions": [],
+ "denyExceptions": [],
+ "id": 4,
+ "isEnabled": true
+ }
+ ],
+ "serviceDef": {
+ "name": "tag",
+ "implClass": "org.apache.ranger.services.tag.RangerServiceTag",
+ "label": "TAG",
+ "options": {
+ "ui.pages": "tag-based-policies"
+ },
+ "configs": [],
+ "resources": [
+ {
+ "itemId": 1,
+ "name": "tag",
+ "type": "string",
+ "level": 1,
+ "mandatory": true,
+ "lookupSupported": true,
+ "recursiveSupported": false,
+ "excludesSupported": false,
+ "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher",
+ "matcherOptions": {
+ "wildCard": "false",
+ "ignoreCase": "false"
+ },
+ "validationRegEx": "",
+ "validationMessage": "",
+ "uiHint": "{ \"singleValue\":true }",
+ "label": "TAG"
+ }
+ ],
+ "accessTypes": [
+ {
+ "itemId": 3004,
+ "name": "hive:select",
+ "label": "select",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 3005,
+ "name": "hive:update",
+ "label": "update",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 3006,
+ "name": "hive:create",
+ "label": "Create",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 3007,
+ "name": "hive:drop",
+ "label": "Drop",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 3008,
+ "name": "hive:alter",
+ "label": "Alter",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 3009,
+ "name": "hive:index",
+ "label": "Index",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 3010,
+ "name": "hive:lock",
+ "label": "Lock",
+ "impliedGrants": []
+ },
+ {
+ "itemId": 3011,
+ "name": "hive:all",
+ "label": "All",
+ "impliedGrants": [
+ "hive:select",
+ "hive:update",
+ "hive:create",
+ "hive:drop",
+ "hive:alter",
+ "hive:index",
+ "hive:lock"
+ ]
+ }
+ ],
+ "policyConditions": [
+ {
+ "itemId": 1,
+ "name": "accessed-after-expiry",
+ "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptTemplateConditionEvaluator",
+ "evaluatorOptions": {
+ "scriptTemplate": "ctx.isAccessedAfter(\u0027expiry_date\u0027);"
+ },
+ "uiHint": "{ \"singleValue\":true }",
+ "label": "Accessed after expiry_date (yes/no)?"
+ }
+ ],
+ "contextEnrichers": [
+ {
+ "itemId": 1,
+ "name": "TagEnricher",
+ "enricher": "org.apache.ranger.plugin.contextenricher.RangerTagEnricher",
+ "enricherOptions": {
+ "tagRetrieverClassName": "org.apache.ranger.plugin.contextenricher.RangerFileBasedTagRetriever",
+ "tagRefresherPollingInterval": "60000",
+ "serviceTagsFileName":"/testdata/test_servicetags_hive.json"
+ }
+ }
+ ],
+ "enums": [],
+ "id": 100,
+ "isEnabled": true
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/src/test/resources/testdata/test_servicetags_hive.json
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/testdata/test_servicetags_hive.json b/ranger-tools/src/test/resources/testdata/test_servicetags_hive.json
new file mode 100644
index 0000000..73c9264
--- /dev/null
+++ b/ranger-tools/src/test/resources/testdata/test_servicetags_hive.json
@@ -0,0 +1,62 @@
+{
+ "op": "add_or_update",
+ "tagModel": "shared",
+ "serviceName": "cl1_hive",
+ "tagVersion": 500,
+ "tagDefinitions": {
+ "1": {
+ "name": "EXPIRES_ON_1",
+ "source": "Internal",
+ "attributeDefs": [
+ {
+ "name": "activates_on",
+ "type": "datetime"
+ },
+ {
+ "name": "expiry_date",
+ "type": "datetime"
+ }
+ ],
+ "id": 1,
+ "isEnabled": true
+ }
+ },
+ "tags": {
+ "1": {
+ "type": "EXPIRES_ON_1",
+ "attributes": {
+ "expiry_date": "2017/12/31",
+ "activates_on": "2017/01/01"
+ },
+ "id": 1,
+ "isEnabled": true
+ }
+ },
+ "serviceResources": [
+ {
+ "resourceElements": {
+ "database": {
+ "values": [
+ "finance_1"
+ ],
+ "isExcludes": false,
+ "isRecursive": false
+ },
+ "table": {
+ "values": [
+ "tax_2010_1"
+ ],
+ "isExcludes": false,
+ "isRecursive": false
+ }
+ },
+ "id": 1,
+ "isEnabled": true
+ }
+ ],
+ "resourceToTagIds": {
+ "1": [
+ 1
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/fbf4f353/ranger-tools/testdata/test_modules.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/testdata/test_modules.txt b/ranger-tools/testdata/test_modules.txt
new file mode 100644
index 0000000..03771fa
--- /dev/null
+++ b/ranger-tools/testdata/test_modules.txt
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+RangerTagEnricher.setServiceTags
+RangerPolicyEngine.init
+RangerPolicyEngine.preProcess
+RangerPolicyEngine.isAccessAllowedNoAudit