You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2016/03/20 19:30:09 UTC
[6/8] incubator-ranger git commit: RANGER-794: commit id
fc5314e8b79e9d754c12a63bb67a0f0190ddfe9f
RANGER-794: commit id fc5314e8b79e9d754c12a63bb67a0f0190ddfe9f
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/36fbb78f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/36fbb78f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/36fbb78f
Branch: refs/heads/ranger-0.5
Commit: 36fbb78f0bb41e4a1f3e25c050b079f09ed4f668
Parents: bec2fef
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Mon Jan 18 15:31:34 2016 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sun Mar 20 10:34:46 2016 -0700
----------------------------------------------------------------------
.../ranger/plugin/util/PerfDataRecorder.java | 58 ++++++++++++++++----
.../plugin/util/RangerPerfCollectorTracer.java | 6 +-
.../plugin/util/RangerPerfTracerFactory.java | 6 +-
ranger-tools/conf/log4j.properties | 11 +---
ranger-tools/scripts/README.txt | 21 +++----
.../ranger/policyengine/CommandLineParser.java | 4 +-
.../RangerPolicyenginePerfTester.java | 29 +++++++---
ranger-tools/testdata/test_modules.txt | 25 +++++++++
8 files changed, 111 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
index 72da8e8..9b29075 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
@@ -23,6 +23,8 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -32,25 +34,43 @@ public class PerfDataRecorder {
private static final Log LOG = LogFactory.getLog(PerfDataRecorder.class);
private static final Log PERF = RangerPerfTracer.getPerfLogger(PerfDataRecorder.class);
- static PerfDataRecorder instance = null;
+ private static volatile PerfDataRecorder instance = null;
private Map<String, PerfStatistic> perfStatistics = new HashMap<String, PerfStatistic>();
+ private boolean initPerfStatisticsOnce = true;
public static void initialize(List<String> names) {
- if (getPerfDataRecorder() == null) {
- instance = new PerfDataRecorder();
+ if (instance == null) {
+ synchronized (PerfDataRecorder.class) {
+ if (instance == null) {
+ instance = new PerfDataRecorder(names);
+ }
+ }
}
- instance.init(names);
}
- public static PerfDataRecorder getPerfDataRecorder() {
- return instance;
+ public static boolean collectStatistics() {
+ return instance != null;
+ }
+
+ public static void printStatistics() {
+ if (instance != null) {
+ instance.dumpStatistics();
+ }
}
+ public static void recordStatistic(String tag, long elapsedTime) {
+ if (instance != null) {
+ instance.record(tag, elapsedTime);
+ }
+ }
+
+ private void dumpStatistics() {
+ List<String> tags = new ArrayList<String>(perfStatistics.keySet());
- public void dumpStatistics() {
- for (Map.Entry<String, PerfStatistic> entry : perfStatistics.entrySet()) {
+ Collections.sort(tags);
- String tag = entry.getKey();
- PerfStatistic perfStatistic = entry.getValue();
+ for (String tag : tags) {
+
+ PerfStatistic perfStatistic = perfStatistics.get(tag);
long averageTimeSpent = 0L;
long minTimeSpent = 0L;
@@ -73,19 +93,33 @@ public class PerfDataRecorder {
}
}
- void record(String tag, long elapsedTime) {
+ private void record(String tag, long elapsedTime) {
PerfStatistic perfStatistic = perfStatistics.get(tag);
+
+ if (perfStatistic == null && !initPerfStatisticsOnce) {
+ synchronized (PerfDataRecorder.class) {
+ perfStatistic = perfStatistics.get(tag);
+ if (perfStatistic == null) {
+ perfStatistic = new PerfStatistic();
+ perfStatistics.put(tag, perfStatistic);
+ }
+ }
+ }
+
if (perfStatistic != null) {
perfStatistic.addPerfDataItem(elapsedTime);
}
+
}
- private void init(List<String> names) {
+ private PerfDataRecorder(List<String> names) {
if (CollectionUtils.isNotEmpty(names)) {
for (String name : names) {
// Create structure
perfStatistics.put(name, new PerfStatistic());
}
+ } else {
+ initPerfStatisticsOnce = false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
index d092859..d899c6f 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
@@ -22,15 +22,13 @@ package org.apache.ranger.plugin.util;
import org.apache.commons.logging.Log;
public class RangerPerfCollectorTracer extends RangerPerfTracer {
- private final PerfDataRecorder recorder;
- public RangerPerfCollectorTracer(Log logger, String tag, String data, PerfDataRecorder recorder) {
+ public RangerPerfCollectorTracer(Log logger, String tag, String data) {
super(logger, tag, data);
- this.recorder = recorder;
}
@Override
public void log() {
- recorder.record(tag, getElapsedTime());
+ PerfDataRecorder.recordStatistic(tag, getElapsedTime());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java
index 8db2d45..1153091 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracerFactory.java
@@ -23,14 +23,12 @@ import org.apache.commons.logging.Log;
public class RangerPerfTracerFactory {
- private static PerfDataRecorder perfDataRecorder = PerfDataRecorder.getPerfDataRecorder();
-
static RangerPerfTracer getPerfTracer(Log logger, String tag, String data) {
RangerPerfTracer ret = null;
- if (perfDataRecorder != null) {
- ret = new RangerPerfCollectorTracer(logger, tag, data, perfDataRecorder);
+ if (PerfDataRecorder.collectStatistics()) {
+ ret = new RangerPerfCollectorTracer(logger, tag, data);
} else if (logger.isDebugEnabled()) {
ret = new RangerPerfTracer(logger, tag, data);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/ranger-tools/conf/log4j.properties b/ranger-tools/conf/log4j.properties
index 86f5c18..21f7fad 100644
--- a/ranger-tools/conf/log4j.properties
+++ b/ranger-tools/conf/log4j.properties
@@ -13,11 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-##-- To prevent junits from cluttering the build run by default all test runs send output to null appender
-log4j.appender.devnull=org.apache.log4j.varia.NullAppender
-# ranger.root.logger=FATAL,devnull
-
-##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console
ranger.root.logger=INFO,console
log4j.rootLogger=${ranger.root.logger}
@@ -25,14 +20,10 @@ log4j.rootLogger=${ranger.root.logger}
# Logging Threshold
log4j.threshold=ALL
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %L %m%n
+log4j.appender.console.layout.ConversionPattern=%m%n
#
# ranger.perf log level
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/scripts/README.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt
index dda6fd1..53a3a8b 100644
--- a/ranger-tools/scripts/README.txt
+++ b/ranger-tools/scripts/README.txt
@@ -37,28 +37,29 @@ This file describes how to build, setup, configure and run the performance testi
ranger-0.5.0-ranger-tools/conf
ranger-0.5.0-ranger-tools/dist
ranger-0.5.0-ranger-tools/lib
- ranger-0.5.0-ranger-tools/scripts
- ranger-0.5.0-ranger-tools/testdata
4. % cd ranger-0.5.0-ranger-tools
5. Configure the policies and requests to use in the test run
Following sample data files are packaged with the perf-tool:
- service-policies - testdata/test_servicepolicies_hive.json
- requests - testdata/test_requests_hive.json
- modules-to-profile - testdata/test_modules.txt
- Please review the contents of these files and modify (or copy/modify) to suite your policy and request needs.
+ testdata/test_servicepolicies_hive.json - Contains service-policies used to initialize the policy-engine;
- Update conf/log4j.properties to specify the filename where perf run results will be written to. Property to update is 'ranger.perf.logger'.
+ testdata/test_requests_hive.json - Contains access requests to be made to the policy-engine;
+
+ Please review the contents of these files and modify to suit your profiling needs.
+
+ Update conf/log4j.properties to specify the filename where perf run results will be written to. Property to update is 'log4j.appender.PERF.File'.
6. Run the tool with the following command
- % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -p <profiled-modules-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run>
+ % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run>
Example:
- % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -p testdata/test_modules.txt -c 2 -n 1
+ % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1
+
+7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file as shown below. This is for time spent in evaluating access by Ranger Policy Engine during the course of a test run. The time values shown are in milliseconds.
-7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file.
+[RangerPolicyEngine.isAccessAllowed] execCount:64, totalTimeTaken:1873, maxTimeTaken:276, minTimeTaken:4, avgTimeTaken:29
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
index a45d71a..0dc79a0 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
@@ -147,8 +147,8 @@ public class CommandLineParser
if (statCollectionFileName != null) {
statCollectionFileURL = getInputFileURL(statCollectionFileName);
ret = statCollectionFileURL != null;
- } else {
- LOG.error("Error processing stat-collection-module file");
+ } else {
+ ret = true;
}
}
} else {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
index 28cc558..bcd1c68 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
@@ -45,7 +45,7 @@ public class RangerPolicyenginePerfTester {
URL statCollectionFileURL = perfTestOptions.getStatCollectionFileURL();
- List<String> perfModuleNames = buildPerfModuleNames(statCollectionFileURL);
+ List<String> perfModuleNames = statCollectionFileURL != null ? buildPerfModuleNames(statCollectionFileURL) : new ArrayList<String>();
PerfDataRecorder.initialize(perfModuleNames);
@@ -108,17 +108,19 @@ public class RangerPolicyenginePerfTester {
perfTestEngine.cleanup();
- PerfDataRecorder.getPerfDataRecorder().dumpStatistics();
+ PerfDataRecorder.printStatistics();
}
private static List<String> buildPerfModuleNames(URL statCollectionFileURL) {
List<String> perfModuleNames = new ArrayList<String>();
- try (
- InputStream inStream = statCollectionFileURL.openStream();
- InputStreamReader reader = new InputStreamReader(inStream, Charset.forName("UTF-8"));
- BufferedReader br = new BufferedReader(reader);
- ) {
+ InputStream inStream = null;
+ InputStreamReader reader = null;
+ BufferedReader br = null;
+ try {
+ inStream = statCollectionFileURL.openStream();
+ reader = new InputStreamReader(inStream, Charset.forName("UTF-8"));
+ br = new BufferedReader(reader);
String line;
@@ -132,6 +134,19 @@ public class RangerPolicyenginePerfTester {
}
}
} catch (Exception exception) {
+ try {
+ if (br != null) {
+ br.close();
+ }
+ if (reader != null) {
+ reader.close();
+ }
+ if (inStream != null) {
+ inStream.close();
+ }
+ } catch (Exception e) {
+ // Ignore
+ }
System.out.println("Error reading arguments:" + exception);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/36fbb78f/ranger-tools/testdata/test_modules.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/testdata/test_modules.txt b/ranger-tools/testdata/test_modules.txt
index 9ffcbfc..f317aaf 100644
--- a/ranger-tools/testdata/test_modules.txt
+++ b/ranger-tools/testdata/test_modules.txt
@@ -19,6 +19,7 @@
PolicyRefresher.loadPolicy
RangerPolicyEngine.init
+RangerPolicyEngine.cleanUp
RangerContextEnricher.init
RangerPolicyEvaluator.init
RangerPolicyItemEvaluator.init
@@ -34,3 +35,27 @@ RangerTagRefresher.populateTags
RangerPolicyEvaluator.isAccessAllowed
RangerPolicyRetriever.getServicePolicies
RangerTagDBReceiver.getTags
+ServiceREST.createServiceDef
+ServiceREST.updateServiceDef
+ServiceREST.deleteServiceDef
+ServiceREST.getServiceDef
+ServiceREST.getServiceDefByName
+ServiceREST.getServiceDefs
+ServiceREST.createService
+ServiceREST.updateService
+ServiceREST.deleteService
+ServiceREST.getService
+ServiceREST.getServices
+ServiceREST.countService
+ServiceREST.validateConfig
+ServiceREST.lookupResource
+ServiceREST.grantAccess
+ServiceREST.revokeAccess
+ServiceREST.createPolicy
+ServiceREST.updatePolicy
+ServiceREST.deletePolicy
+ServiceREST.getPolicy
+ServiceREST.getPolicies
+ServiceREST.countPolicies
+ServiceREST.getServicePolicies
+ServiceREST.getServicePoliciesIfUpdated
\ No newline at end of file