You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2016/08/16 01:45:24 UTC
incubator-ranger git commit: RANGER-1146: Policy engine optimization:
dynamic reordering of policy evaluation order
Repository: incubator-ranger
Updated Branches:
refs/heads/master b935426d1 -> 3a36875e9
RANGER-1146: Policy engine optimization: dynamic reordering of policy evaluation order
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/3a36875e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/3a36875e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/3a36875e
Branch: refs/heads/master
Commit: 3a36875e96aa2f689edd71ab77a8c74a6355493a
Parents: b935426
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Sat Aug 6 13:08:06 2016 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Aug 15 18:34:14 2016 -0700
----------------------------------------------------------------------
.../plugin/policyengine/RangerAccessResult.java | 26 +++
.../plugin/policyengine/RangerPolicyEngine.java | 2 +
.../policyengine/RangerPolicyEngineImpl.java | 170 +++++++++++++++++--
.../policyengine/RangerPolicyRepository.java | 104 +++++++++---
.../RangerAbstractPolicyEvaluator.java | 36 +++-
.../RangerDefaultPolicyEvaluator.java | 7 +-
.../RangerOptimizedPolicyEvaluator.java | 5 -
.../policyevaluator/RangerPolicyEvaluator.java | 9 +
.../ranger/plugin/service/RangerBasePlugin.java | 64 ++++++-
.../ranger/plugin/util/PerfDataRecorder.java | 41 ++---
.../ranger/plugin/util/RangerPerfTracer.java | 9 +
.../plugin/policyengine/TestPolicyEngine.java | 4 +
.../ranger/policyengine/CommandLineParser.java | 11 +-
.../ranger/policyengine/PerfTestEngine.java | 17 +-
.../ranger/policyengine/PerfTestOptions.java | 9 +-
.../RangerPolicyenginePerfTester.java | 32 ++--
.../test/resources/testdata/test_modules.txt | 3 +-
ranger-tools/testdata/test_modules.txt | 4 +-
18 files changed, 465 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
index 49d32b2..5543848 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
@@ -31,7 +31,9 @@ public class RangerAccessResult {
private boolean isAllowed = false;
private boolean isAuditedDetermined = false;
private boolean isAudited = false;
+ private long auditPolicyId = -1;
private long policyId = -1;
+ private long evaluatedPoliciesCount = 0;
private String reason = null;
public RangerAccessResult(final String serviceName, final RangerServiceDef serviceDef, final RangerAccessRequest request) {
@@ -42,7 +44,9 @@ public class RangerAccessResult {
this.isAllowed = false;
this.isAuditedDetermined = false;
this.isAudited = false;
+ this.auditPolicyId = -1;
this.policyId = -1;
+ this.evaluatedPoliciesCount = 0;
this.reason = null;
}
@@ -50,12 +54,15 @@ public class RangerAccessResult {
this.isAccessDetermined = other.getIsAccessDetermined();
this.isAllowed = other.getIsAllowed();
this.policyId = other.getPolicyId();
+ this.evaluatedPoliciesCount = other.evaluatedPoliciesCount;
this.reason = other.getReason();
}
public void setAuditResultFrom(final RangerAccessResult other) {
this.isAuditedDetermined = other.getIsAuditedDetermined();
this.isAudited = other.getIsAudited();
+ this.auditPolicyId = other.getAuditPolicyId();
+ this.evaluatedPoliciesCount += other.getEvaluatedPoliciesCount();
}
/**
@@ -144,12 +151,29 @@ public class RangerAccessResult {
}
/**
+ * @return the auditPolicyId
+ */
+ public long getAuditPolicyId() {
+ return auditPolicyId;
+ }
+
+ public long getEvaluatedPoliciesCount() { return this.evaluatedPoliciesCount; }
+ /**
* @param policyId the policyId to set
*/
public void setPolicyId(long policyId) {
this.policyId = policyId;
}
+ /**
+ * @param policyId the auditPolicyId to set
+ */
+ public void setAuditPolicyId(long policyId) {
+ this.auditPolicyId = policyId;
+ }
+
+ public void incrementEvaluatedPoliciesCount() { this.evaluatedPoliciesCount++; }
+
public int getServiceType() {
int ret = -1;
@@ -177,6 +201,8 @@ public class RangerAccessResult {
sb.append("isAuditedDetermined={").append(isAuditedDetermined).append("} ");
sb.append("isAudited={").append(isAudited).append("} ");
sb.append("policyId={").append(policyId).append("} ");
+ sb.append("auditPolicyId={").append(auditPolicyId).append("} ");
+ sb.append("evaluatedPoliciesCount={").append(evaluatedPoliciesCount).append("} ");
sb.append("reason={").append(reason).append("} ");
sb.append("}");
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
index 3e69d03..a5e92da 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
@@ -69,6 +69,8 @@ public interface RangerPolicyEngine {
RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request);
+ void reorderPolicyEvaluators();
+
boolean preCleanup();
void setUseForwardedIPAddress(boolean useForwardedIPAddress);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
index 1b8433b..90da6af 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
@@ -20,6 +20,7 @@
package org.apache.ranger.plugin.policyengine;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -34,7 +35,13 @@ import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
import org.apache.ranger.plugin.util.RangerPerfTracer;
import org.apache.ranger.plugin.util.ServicePolicies;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
public class RangerPolicyEngineImpl implements RangerPolicyEngine {
private static final Log LOG = LogFactory.getLog(RangerPolicyEngineImpl.class);
@@ -43,6 +50,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
private static final Log PERF_POLICYENGINE_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policyengine.request");
private static final Log PERF_POLICYENGINE_AUDIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.audit");
private static final Log PERF_CONTEXTENRICHER_REQUEST_LOG = RangerPerfTracer.getPerfLogger("contextenricher.request");
+ private static final Log PERF_POLICYENGINE_REBALANCE_LOG = RangerPerfTracer.getPerfLogger("policyengine.rebalance");
+ private static final Log PERF_POLICYENGINE_USAGE_LOG = RangerPerfTracer.getPerfLogger("policyengine.usage");
private static final int MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR = 100;
@@ -51,6 +60,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
private List<RangerContextEnricher> allContextEnrichers;
+ private final Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
+
private boolean useForwardedIPAddress = false;
private String[] trustedProxyAddresses = null;
@@ -132,6 +143,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
this.allContextEnrichers = tmpList;
+ policyEvaluatorsMap = createPolicyEvaluatorsMap();
+
RangerPerfTracer.log(perf);
if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) {
@@ -160,6 +173,15 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
return policyRepository.getPolicyVersion();
}
+ public RangerPolicyEvaluator getPolicyEvaluator(Long id) {
+ return policyEvaluatorsMap.get(id);
+ }
+
+ public RangerPolicy getPolicy(Long id) {
+ RangerPolicyEvaluator evaluator = getPolicyEvaluator(id);
+ return evaluator != null ? evaluator.getPolicy() : null;
+ }
+
@Override
public RangerAccessResult createAccessResult(RangerAccessRequest request) {
return new RangerAccessResult(this.getServiceName(), policyRepository.getServiceDef(), request);
@@ -231,6 +253,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
RangerAccessResult ret = isAccessAllowedNoAudit(request);
+ updatePolicyUsageCounts(request, ret);
+
if (resultProcessor != null) {
RangerPerfTracer perfAuditTracer = null;
@@ -304,6 +328,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
ret.setIsAudited(false);
}
+ updatePolicyUsageCounts(request, ret);
+
if (resultProcessor != null) {
resultProcessor.processResult(ret);
}
@@ -339,6 +365,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
ret.setIsAudited(false);
}
+ updatePolicyUsageCounts(request, ret);
+
if (resultProcessor != null) {
resultProcessor.processResult(ret);
}
@@ -561,6 +589,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
List<RangerPolicyEvaluator> evaluators = policyRepository.getPolicyEvaluators();
for (RangerPolicyEvaluator evaluator : evaluators) {
+ ret.incrementEvaluatedPoliciesCount();
evaluator.evaluate(request, ret);
if(ret.getIsAllowed() && !evaluator.hasDeny()) { // all policies having deny have been evaluated
@@ -617,6 +646,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
tagEvalResult.setAuditResultFrom(result);
for (RangerPolicyEvaluator evaluator : evaluators) {
+ tagEvalResult.incrementEvaluatedPoliciesCount();
if(! evaluator.isMatch(tagEvalRequest.getResource()))
continue;
@@ -641,6 +671,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
if (tagEvalResult.getIsAudited()) {
result.setIsAudited(true);
+ result.setAuditPolicyId(tagEvalResult.getAuditPolicyId());
}
if(!result.getIsAccessDetermined() && tagEvalResult.getIsAccessDetermined()) {
@@ -667,25 +698,40 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
}
}
- private void setResourceServiceDef(RangerAccessRequest request) {
- RangerAccessResource resource = request.getResource();
+ @Override
+ public void reorderPolicyEvaluators() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> reorderPolicyEvaluators()");
+ }
+ RangerPerfTracer perf = null;
- if (resource.getServiceDef() == null) {
- if (resource instanceof RangerMutableResource) {
- RangerMutableResource mutable = (RangerMutableResource) resource;
- mutable.setServiceDef(getServiceDef());
- } else {
- LOG.debug("RangerPolicyEngineImpl.setResourceServiceDef(): Cannot set ServiceDef in RangerTagResourceMap.");
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REBALANCE_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderPolicyEvaluators()");
+ }
+ if (MapUtils.isNotEmpty(policyEvaluatorsMap)) {
+ for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) {
+ entry.getValue().setUsageCountImmutable();
}
}
- }
- private boolean hasTagPolicies() {
- return tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies());
- }
+ if (tagPolicyRepository != null) {
+ tagPolicyRepository.reorderPolicyEvaluators();
+ }
+ if (policyRepository != null) {
+ policyRepository.reorderPolicyEvaluators();
+ }
- private boolean hasResourcePolicies() {
- return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies());
+ if (MapUtils.isNotEmpty(policyEvaluatorsMap)) {
+ for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) {
+ entry.getValue().resetUsageCount();
+ }
+ }
+
+ RangerPerfTracer.log(perf);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== reorderPolicyEvaluators()");
+ }
}
@Override
@@ -783,6 +829,100 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
return sb;
}
+
+
+ private void setResourceServiceDef(RangerAccessRequest request) {
+ RangerAccessResource resource = request.getResource();
+
+ if (resource.getServiceDef() == null) {
+ if (resource instanceof RangerMutableResource) {
+ RangerMutableResource mutable = (RangerMutableResource) resource;
+ mutable.setServiceDef(getServiceDef());
+ } else {
+ LOG.debug("RangerPolicyEngineImpl.setResourceServiceDef(): Cannot set ServiceDef in RangerTagResourceMap.");
+ }
+ }
+ }
+
+ private boolean hasTagPolicies() {
+ return tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies());
+ }
+
+ private boolean hasResourcePolicies() {
+ return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies());
+ }
+
+ private Map<Long, RangerPolicyEvaluator> createPolicyEvaluatorsMap() {
+ Map<Long, RangerPolicyEvaluator> tmpPolicyEvaluatorMap = new HashMap<Long, RangerPolicyEvaluator>();
+
+ if (tagPolicyRepository != null) {
+ for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getPolicyEvaluators()) {
+ tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+ }
+ for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getDataMaskPolicyEvaluators()) {
+ tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+ }
+ for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getRowFilterPolicyEvaluators()) {
+ tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+ }
+ }
+ for (RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) {
+ tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+ }
+ for (RangerPolicyEvaluator evaluator : policyRepository.getDataMaskPolicyEvaluators()) {
+ tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+ }
+ for (RangerPolicyEvaluator evaluator : policyRepository.getRowFilterPolicyEvaluators()) {
+ tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+ }
+
+ return Collections.unmodifiableMap(tmpPolicyEvaluatorMap);
+ }
+
+ private void updatePolicyUsageCounts(RangerAccessRequest accessRequest, RangerAccessResult accessResult) {
+
+ boolean auditCountUpdated = false;
+
+ if (accessResult.getIsAccessDetermined()) {
+ RangerPolicyEvaluator accessPolicy = getPolicyEvaluator(accessResult.getPolicyId());
+
+ if (accessPolicy != null) {
+
+ if (accessPolicy.getPolicy().getIsAuditEnabled()) {
+ updateUsageCount(accessPolicy, 2);
+ accessResult.setAuditPolicyId(accessResult.getPolicyId());
+
+ auditCountUpdated = true;
+ } else {
+ updateUsageCount(accessPolicy, 1);
+ }
+
+ }
+ }
+
+ if (!auditCountUpdated && accessResult.getIsAuditedDetermined()) {
+ long auditPolicyId = accessResult.getAuditPolicyId();
+ RangerPolicyEvaluator auditPolicy = auditPolicyId == -1 ? null : getPolicyEvaluator(auditPolicyId);
+
+ updateUsageCount(auditPolicy, 1);
+ }
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_USAGE_LOG)) {
+ RangerAccessRequestImpl rangerAccessRequest = (RangerAccessRequestImpl) accessRequest;
+ RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_USAGE_LOG,
+ "RangerPolicyEngine.usage(accessingUser=" + rangerAccessRequest.getUser()
+ + ",accessedResource=" + rangerAccessRequest.getResource().getAsString()
+ + ",accessType=" + rangerAccessRequest.getAccessType()
+ + ",evaluatedPoliciesCount=" + accessResult.getEvaluatedPoliciesCount() + ")");
+ RangerPerfTracer.logAlways(perf);
+ }
+ }
+
+ private void updateUsageCount(RangerPolicyEvaluator evaluator, int number) {
+ if (evaluator != null) {
+ evaluator.incrementUsageCount(number);
+ }
+ }
}
class RangerTagResource extends RangerAccessResourceImpl {
private static final String KEY_TAG = "tag";
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 01a547c..0012329 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -42,7 +42,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-public class RangerPolicyRepository {
+class RangerPolicyRepository {
private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class);
private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init");
@@ -51,6 +51,25 @@ public class RangerPolicyRepository {
AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT
}
+ class AuditInfo {
+ final boolean isAudited;
+ final long auditPolicyId;
+
+ AuditInfo() {
+ this(false, -1);
+ }
+ AuditInfo(boolean isAudited, long auditPolicyId) {
+ this.isAudited = isAudited;
+ this.auditPolicyId = auditPolicyId;
+ }
+ long getAuditPolicyId() {
+ return this.auditPolicyId;
+ }
+ boolean getIsAudited() {
+ return isAudited;
+ }
+ }
+
private final String serviceName;
private final String appId;
private final RangerServiceDef serviceDef;
@@ -61,7 +80,7 @@ public class RangerPolicyRepository {
private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators;
private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators;
private final AuditModeEnum auditModeEnum;
- private final Map<String, Boolean> accessAuditCache;
+ private final Map<String, AuditInfo> accessAuditCache;
private final String componentServiceName;
private final RangerServiceDef componentServiceDef;
@@ -98,7 +117,7 @@ public class RangerPolicyRepository {
final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64 * 1024;
int auditResultCacheSize = RangerConfiguration.getInstance().getInt(propertyName, RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE);
- accessAuditCache = Collections.synchronizedMap(new CacheMap<String, Boolean>(auditResultCacheSize));
+ accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize));
} else {
accessAuditCache = null;
}
@@ -166,15 +185,15 @@ public class RangerPolicyRepository {
public List<RangerContextEnricher> getContextEnrichers() { return contextEnrichers; }
- public List<RangerPolicyEvaluator> getPolicyEvaluators() {
+ List<RangerPolicyEvaluator> getPolicyEvaluators() {
return policyEvaluators;
}
- public List<RangerPolicyEvaluator> getDataMaskPolicyEvaluators() {
+ List<RangerPolicyEvaluator> getDataMaskPolicyEvaluators() {
return dataMaskPolicyEvaluators;
}
- public List<RangerPolicyEvaluator> getRowFilterPolicyEvaluators() {
+ List<RangerPolicyEvaluator> getRowFilterPolicyEvaluators() {
return rowFilterPolicyEvaluators;
}
@@ -323,10 +342,8 @@ public class RangerPolicyRepository {
return policyItems;
}
- public static boolean isDelegateAdminPolicy(RangerPolicy policy) {
- boolean ret = false;
-
- ret = hasDelegateAdminItems(policy.getPolicyItems())
+ private static boolean isDelegateAdminPolicy(RangerPolicy policy) {
+ boolean ret = hasDelegateAdminItems(policy.getPolicyItems())
|| hasDelegateAdminItems(policy.getDenyPolicyItems())
|| hasDelegateAdminItems(policy.getAllowExceptions())
|| hasDelegateAdminItems(policy.getDenyExceptions());
@@ -510,42 +527,49 @@ public class RangerPolicyRepository {
LOG.debug("==> RangerPolicyRepository.setAuditEnabledFromCache()");
}
- Boolean value = null;
+ final boolean auditResult;
+ final boolean foundInCache;
switch (auditModeEnum) {
case AUDIT_ALL:
- value = Boolean.TRUE;
+ auditResult = true;
+ foundInCache = true;
break;
case AUDIT_NONE:
- value = Boolean.FALSE;
+ auditResult = false;
+ foundInCache = true;
break;
default:
- if (accessAuditCache != null) {
- value = accessAuditCache.get(request.getResource().getAsString());
+ AuditInfo auditInfo = accessAuditCache != null ? accessAuditCache.get(request.getResource().getAsString()) : null;
+ if (auditInfo != null) {
+ auditResult = auditInfo.getIsAudited();
+ result.setAuditPolicyId(auditInfo.getAuditPolicyId());
+ foundInCache = true;
+ } else {
+ auditResult = false;
+ foundInCache = false;
}
break;
}
- if ((value != null)) {
- result.setIsAudited(value);
+ if (foundInCache) {
+ result.setIsAudited(auditResult);
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== RangerPolicyRepository.setAuditEnabledFromCache()");
}
- return value != null;
+ return foundInCache;
}
- void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult ret) {
+ void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult result) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyRepository.storeAuditEnabledInCache()");
}
- if (accessAuditCache != null && ret.getIsAuditedDetermined()) {
- String strResource = request.getResource().getAsString();
- Boolean value = ret.getIsAudited() ? Boolean.TRUE : Boolean.FALSE;
- accessAuditCache.put(strResource, value);
+ if (accessAuditCache != null && result.getIsAuditedDetermined()) {
+ accessAuditCache.put(request.getResource().getAsString(), new AuditInfo(result.getIsAudited(), result.getAuditPolicyId()));
}
if (LOG.isDebugEnabled()) {
@@ -553,11 +577,7 @@ public class RangerPolicyRepository {
}
}
- /**
- * Remove nulls from policy resource values
- * @param policy
- */
- boolean scrubPolicy(RangerPolicy policy) {
+ private boolean scrubPolicy(RangerPolicy policy) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyRepository.scrubPolicy(" + policy + ")");
}
@@ -584,6 +604,34 @@ public class RangerPolicyRepository {
return altered;
}
+ void reorderPolicyEvaluators() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> reorderPolicyEvaluators()");
+ }
+
+ this.policyEvaluators = getReorderedPolicyEvaluators(this.policyEvaluators);
+ this.dataMaskPolicyEvaluators = getReorderedPolicyEvaluators(this.dataMaskPolicyEvaluators);
+ this.rowFilterPolicyEvaluators = getReorderedPolicyEvaluators(this.rowFilterPolicyEvaluators);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== reorderPolicyEvaluators()");
+ }
+ }
+
+ private List<RangerPolicyEvaluator> getReorderedPolicyEvaluators(List<RangerPolicyEvaluator> evaluators) {
+ List<RangerPolicyEvaluator> ret = evaluators;
+
+ if (CollectionUtils.isNotEmpty(evaluators)) {
+
+ ret = new ArrayList<RangerPolicyEvaluator>(evaluators);
+ Collections.sort(ret);
+
+ ret = Collections.unmodifiableList(ret);
+ }
+
+ return ret;
+ }
+
@Override
public String toString( ) {
StringBuilder sb = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
index adc7d8c..8f45661 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
@@ -35,6 +35,8 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
private RangerPolicy policy = null;
private RangerServiceDef serviceDef = null;
private int evalOrder = 0;
+ protected long usageCount = 0;
+ protected boolean usageCountMutable = true;
@Override
@@ -74,10 +76,14 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
}
@Override
+ public long getUsageCount() {
+ return usageCount;
+ }
+
+ @Override
public int getEvalOrder() {
return evalOrder;
}
-
@Override
public boolean isAuditEnabled() {
return policy != null && policy.getIsAuditEnabled();
@@ -89,7 +95,17 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
LOG.debug("==> RangerAbstractPolicyEvaluator.compareTo()");
}
- int result = Integer.compare(this.getEvalOrder(), other.getEvalOrder());
+ int result;
+ if (hasDeny() && !other.hasDeny()) {
+ result = -1;
+ } else if (!hasDeny() && other.hasDeny()) {
+ result = 1;
+ } else {
+ result = Long.compare(other.getUsageCount(), this.usageCount);
+ if (result == 0) {
+ result = Integer.compare(this.evalOrder, other.getEvalOrder());
+ }
+ }
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerAbstractPolicyEvaluator.compareTo(), result:" + result);
@@ -103,6 +119,22 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
}
@Override
+ public void incrementUsageCount(int number) {
+ if (usageCountMutable) usageCount += number;
+ }
+
+ @Override
+ public void setUsageCountImmutable() {
+ this.usageCountMutable = false;
+ }
+
+ @Override
+ public void resetUsageCount() {
+ this.usageCount = 0;
+ this.usageCountMutable = true;
+ }
+
+ @Override
public String toString( ) {
StringBuilder sb = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
index 368a695..f954ccf 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
@@ -177,6 +177,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
// We are done for determining if audit is needed for this policy
if (isAuditEnabled()) {
result.setIsAudited(true);
+ result.setAuditPolicyId(getPolicy().getId());
}
}
}
@@ -249,6 +250,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
if (isResourceMatch || isResourceHeadMatch) {
if (isAuditEnabled()) {
result.setIsAudited(true);
+ result.setAuditPolicyId(getPolicy().getId());
}
}
}
@@ -270,6 +272,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
evaluatePolicyItems(request, result);
}
}
+
}
RangerPerfTracer.log(perf);
@@ -314,6 +317,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
if (isResourceMatch || isResourceHeadMatch) {
if (isAuditEnabled()) {
result.setIsAudited(true);
+ result.setAuditPolicyId(getPolicy().getId());
}
}
}
@@ -990,5 +994,6 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
}
return ret;
- }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java
index 2e777ae..00b24d1 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java
@@ -47,7 +47,6 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator
private static final String RANGER_POLICY_EVAL_MATCH_ONE_CHARACTER_STRING = "?";
private static final int RANGER_POLICY_EVAL_SCORE_DEFAULT = 10000;
- private static final int RANGER_POLICY_EVAL_SCORE_DISCOUNT_POLICY_HAS_DENY = 4000;
private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_RESOURCE = 100;
private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_USERSGROUPS = 25;
@@ -202,10 +201,6 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator
evalOrder -= customConditionsDiscount;
}
- if (hasDeny()) {
- evalOrder -= RANGER_POLICY_EVAL_SCORE_DISCOUNT_POLICY_HAS_DENY;
- }
-
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerOptimizedPolicyEvaluator.computeEvalOrder(), policyName:" + policy.getName() + ", priority:" + evalOrder);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
index be97830..a7dc833 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
@@ -52,6 +52,14 @@ public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator>
int getEvalOrder();
+ long getUsageCount();
+
+ void incrementUsageCount(int number);
+
+ void setUsageCountImmutable();
+
+ void resetUsageCount();
+
int getCustomConditionsCount();
boolean isAuditEnabled();
@@ -73,4 +81,5 @@ public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator>
boolean isAccessAllowed(Map<String, RangerPolicyResource> resources, String user, Set<String> userGroups, String accessType);
void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAccessInfo result);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index a000125..760c4a3 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -22,6 +22,8 @@ package org.apache.ranger.plugin.service;
import java.util.Collection;
import java.util.Hashtable;
import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -30,7 +32,17 @@ import org.apache.ranger.admin.client.RangerAdminClient;
import org.apache.ranger.admin.client.RangerAdminRESTClient;
import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.model.RangerServiceDef;
-import org.apache.ranger.plugin.policyengine.*;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
+import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
+import org.apache.ranger.plugin.policyengine.RangerAccessResult;
+import org.apache.ranger.plugin.policyengine.RangerAccessResultProcessor;
+import org.apache.ranger.plugin.policyengine.RangerDataMaskResult;
+import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
+import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl;
+import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
+import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo;
+import org.apache.ranger.plugin.policyengine.RangerRowFilterResult;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import org.apache.ranger.plugin.util.GrantRevokeRequest;
import org.apache.ranger.plugin.util.PolicyRefresher;
@@ -51,6 +63,7 @@ public class RangerBasePlugin {
private RangerAccessResultProcessor resultProcessor = null;
private boolean useForwardedIPAddress = false;
private String[] trustedProxyAddresses = null;
+ private Timer policyEngineRefreshTimer;
Map<String, LogHistory> logHistoryList = new Hashtable<String, RangerBasePlugin.LogHistory>();
int logInterval = 30000; // 30 seconds
@@ -126,6 +139,32 @@ public class RangerBasePlugin {
refresher = new PolicyRefresher(this, serviceType, appId, serviceName, admin, pollingIntervalMs, cacheDir);
refresher.setDaemon(true);
refresher.startRefresher();
+
+ long policyReorderIntervalMs = RangerConfiguration.getInstance().getLong(propertyPrefix + ".policy.policyReorderInterval", 60 * 1000);
+ if (policyReorderIntervalMs >= 0 && policyReorderIntervalMs < 15 * 1000) {
+ policyReorderIntervalMs = 15 * 1000;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(propertyPrefix + ".policy.policyReorderInterval:" + policyReorderIntervalMs);
+ }
+
+ if (policyReorderIntervalMs > 0) {
+ policyEngineRefreshTimer = new Timer("PolicyEngineRefreshTimer", true);
+ try {
+ policyEngineRefreshTimer.schedule(new PolicyEngineRefresher(this), policyReorderIntervalMs, policyReorderIntervalMs);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Scheduled PolicyEngineRefresher to reorder policies nbased on number of evaluations in and every " + policyReorderIntervalMs + " milliseconds");
+ }
+ } catch (IllegalStateException exception) {
+ LOG.error("Error scheduling policyEngineRefresher:", exception);
+ LOG.error("*** PolicyEngine will NOT be reorderd based on number of evaluations every " + policyReorderIntervalMs + " milliseconds ***");
+ policyEngineRefreshTimer = null;
+ }
+ } else {
+ LOG.info("Policies will NOT be reordered based on number of evaluations because "
+ + propertyPrefix + ".policy.policyReorderInterval is set to a negative number[" + policyReorderIntervalMs +"]");
+ }
}
public void setPolicies(ServicePolicies policies) {
@@ -154,17 +193,25 @@ public class RangerBasePlugin {
RangerPolicyEngine policyEngine = this.policyEngine;
+ Timer policyEngineRefreshTimer = this.policyEngineRefreshTimer;
+
this.serviceName = null;
this.policyEngine = null;
this.refresher = null;
+ this.policyEngineRefreshTimer = null;
if (refresher != null) {
refresher.stopRefresher();
}
+ if (policyEngineRefreshTimer != null) {
+ policyEngineRefreshTimer.cancel();
+ }
+
if (policyEngine != null) {
policyEngine.cleanup();
}
+
}
public void setResultProcessor(RangerAccessResultProcessor resultProcessor) {
@@ -400,4 +447,19 @@ public class RangerBasePlugin {
int counter=0;
}
+ private class PolicyEngineRefresher extends TimerTask {
+ private final RangerBasePlugin plugin;
+
+ PolicyEngineRefresher(RangerBasePlugin plugin) {
+ this.plugin = plugin;
+ }
+
+ @Override
+ public void run() {
+ RangerPolicyEngine policyEngine = plugin.policyEngine;
+ if (policyEngine != null) {
+ policyEngine.reorderPolicyEvaluators();
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
index ddc42c0..3fe96ac 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
@@ -36,7 +36,6 @@ public class PerfDataRecorder {
private static volatile PerfDataRecorder instance = null;
private Map<String, PerfStatistic> perfStatistics = new HashMap<String, PerfStatistic>();
- private boolean initPerfStatisticsOnce = true;
public static void initialize(List<String> names) {
if (instance == null) {
@@ -57,6 +56,13 @@ public class PerfDataRecorder {
instance.dumpStatistics();
}
}
+
+ public static void clearStatistics() {
+ if (instance != null) {
+ instance.clear();
+ }
+ }
+
public static void recordStatistic(String tag, long elapsedTime) {
if (instance != null) {
instance.record(tag, elapsedTime);
@@ -73,19 +79,16 @@ public class PerfDataRecorder {
PerfStatistic perfStatistic = perfStatistics.get(tag);
long averageTimeSpent = 0L;
- long minTimeSpent = 0L;
- long maxTimeSpent = 0L;
+
if (perfStatistic.numberOfInvocations.get() != 0L) {
averageTimeSpent = perfStatistic.millisecondsSpent.get()/perfStatistic.numberOfInvocations.get();
- minTimeSpent = perfStatistic.minTimeSpent.get();
- maxTimeSpent = perfStatistic.maxTimeSpent.get();
}
String logMsg = "[" + tag + "]" +
- " execCount:" + perfStatistic.numberOfInvocations +
- ", totalTimeTaken:" + perfStatistic.millisecondsSpent +
- ", maxTimeTaken:" + maxTimeSpent +
- ", minTimeTaken:" + minTimeSpent +
+ " execCount:" + perfStatistic.numberOfInvocations.get() +
+ ", totalTimeTaken:" + perfStatistic.millisecondsSpent.get() +
+ ", maxTimeTaken:" + perfStatistic.maxTimeSpent.get() +
+ ", minTimeTaken:" + perfStatistic.minTimeSpent.get() +
", avgTimeTaken:" + averageTimeSpent;
LOG.info(logMsg);
@@ -93,23 +96,25 @@ public class PerfDataRecorder {
}
}
+ private void clear() {
+ perfStatistics.clear();
+ }
+
private void record(String tag, long elapsedTime) {
PerfStatistic perfStatistic = perfStatistics.get(tag);
- if (perfStatistic == null && !initPerfStatisticsOnce) {
+ if (perfStatistic == null) {
synchronized (PerfDataRecorder.class) {
perfStatistic = perfStatistics.get(tag);
- if (perfStatistic == null) {
+
+ if(perfStatistic == null) {
perfStatistic = new PerfStatistic();
perfStatistics.put(tag, perfStatistic);
}
}
}
- if (perfStatistic != null) {
- perfStatistic.addPerfDataItem(elapsedTime);
- }
-
+ perfStatistic.addPerfDataItem(elapsedTime);
}
private PerfDataRecorder(List<String> names) {
@@ -118,8 +123,6 @@ public class PerfDataRecorder {
// Create structure
perfStatistics.put(name, new PerfStatistic());
}
- } else {
- initPerfStatisticsOnce = false;
}
}
@@ -134,12 +137,12 @@ public class PerfDataRecorder {
millisecondsSpent.getAndAdd(timeTaken);
long min = minTimeSpent.get();
- if (timeTaken < min) {
+ if(timeTaken < min) {
minTimeSpent.compareAndSet(min, timeTaken);
}
long max = maxTimeSpent.get();
- if (timeTaken > max) {
+ if(timeTaken > max) {
maxTimeSpent.compareAndSet(max, timeTaken);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
index e130cc7..4b17110 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
@@ -71,6 +71,11 @@ public class RangerPerfTracer {
}
}
+ public static void logAlways(RangerPerfTracer tracer) {
+ if(tracer != null) {
+ tracer.logAlways();
+ }
+ }
public RangerPerfTracer(Log logger, String tag, String data) {
this.logger = logger;
this.tag = tag;
@@ -96,4 +101,8 @@ public class RangerPerfTracer {
logger.debug("[PERF] " + tag + data + ": " + elapsedTime);
}
}
+ public void logAlways() {
+ long elapsedTime = getElapsedTime();
+ logger.debug("[PERF] " + tag + data + ": " + elapsedTime);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
----------------------------------------------------------------------
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
index 1d03ef8..ab5626e 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
@@ -300,11 +300,15 @@ public class TestPolicyEngine {
policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions);
policyEngine.setUseForwardedIPAddress(useForwardedIPAddress);
policyEngine.setTrustedProxyAddresses(trustedProxyAddresses);
+ long requestCount = 0L;
RangerAccessRequest request = null;
for(TestData test : testCase.tests) {
request = test.request;
+ if ((requestCount++ % 10) == 1) {
+ policyEngine.reorderPolicyEvaluators();
+ }
if (request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_TAGS) ||
request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) {
// Create a new AccessRequest
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
index ab7d532..d6c028f 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
@@ -52,6 +52,8 @@ public class CommandLineParser
private int concurrentClientCount = 1;
private int iterationsCount = 1;
+ private boolean isDynamicReorderingEnabled = false;
+
private Options options = new Options();
CommandLineParser() {}
@@ -60,7 +62,7 @@ public class CommandLineParser
PerfTestOptions ret = null;
if (parseArguments(args) && validateInputFiles()) {
// Instantiate a data-object and return
- ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount);
+ ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingEnabled);
} else {
showUsage();
}
@@ -75,6 +77,7 @@ public class CommandLineParser
-r request-file-name-list
-n number-of-iterations
-p modules-to-collect-stats
+ -o
If the concurrent-client-count is more than the number of files in the request-file-name-list,
then reuse the request-file-names in a round-robin way
@@ -94,6 +97,7 @@ public class CommandLineParser
options.addOption("p", "statistics", true, "Modules for stat collection File Name");
options.addOption("c", "clients", true, "Number of concurrent clients");
options.addOption("n", "cycles", true, "Number of iterations");
+ options.addOption("o", "optimize", false, "Enable usage-based policy reordering");
org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser();
@@ -120,9 +124,14 @@ public class CommandLineParser
if (iterationsOptionValue != null) {
iterationsCount = Integer.parseInt(iterationsOptionValue);
}
+ if (commandLine.hasOption("o")) {
+ isDynamicReorderingEnabled = true;
+ }
+
if (LOG.isDebugEnabled()) {
LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames));
LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount);
+ LOG.debug("isDynamicReorderingEnabled=" + isDynamicReorderingEnabled);
}
ret = true;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
index 38c6e2f..cf83260 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
@@ -24,6 +24,7 @@ import com.google.gson.GsonBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.plugin.policyengine.*;
+import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import org.apache.ranger.plugin.util.ServicePolicies;
import java.io.InputStream;
@@ -31,15 +32,20 @@ import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.nio.charset.Charset;
+import java.util.concurrent.atomic.AtomicLong;
public class PerfTestEngine {
static final Log LOG = LogFactory.getLog(PerfTestEngine.class);
+ static private final long POLICY_ENGINE_REORDER_AFTER_PROCESSING_REQUESTS_COUNT = 100;
private final URL servicePoliciesFileURL;
+ private final boolean isDynamicReorderingEnabled;
private RangerPolicyEngine policyEvaluationEngine;
+ private AtomicLong requestCount = new AtomicLong();
- public PerfTestEngine(final URL servicePoliciesFileURL) {
+ public PerfTestEngine(final URL servicePoliciesFileURL, boolean isDynamicReorderingEnabled) {
this.servicePoliciesFileURL = servicePoliciesFileURL;
+ this.isDynamicReorderingEnabled = isDynamicReorderingEnabled;
}
public boolean init() {
@@ -66,9 +72,12 @@ public class PerfTestEngine {
RangerPolicyEngineOptions engineOptions = new RangerPolicyEngineOptions();
engineOptions.disableTagPolicyEvaluation = false;
+ engineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED;
policyEvaluationEngine = new RangerPolicyEngineImpl("perf-test", servicePolicies, engineOptions);
+ requestCount.set(0L);
+
ret = true;
} catch (Exception excp) {
@@ -101,6 +110,12 @@ public class PerfTestEngine {
if (policyEvaluationEngine != null) {
+ long processedRequestCount = requestCount.getAndIncrement();
+
+ if (isDynamicReorderingEnabled && (processedRequestCount % POLICY_ENGINE_REORDER_AFTER_PROCESSING_REQUESTS_COUNT) == 0) {
+ policyEvaluationEngine.reorderPolicyEvaluators();
+ }
+
policyEvaluationEngine.preProcess(request);
ret = policyEvaluationEngine.isAccessAllowed(request, null);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
index f30cbd7..321ee69 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
@@ -26,17 +26,19 @@ public class PerfTestOptions {
private final URL servicePoliciesFileURL;
private final URL[] requestFileURLs;
private final URL statCollectionFileURL;
+ private final boolean isDynamicReorderingEnabled;
private final int concurrentClientCount;
private final int iterationsCount;
- PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount) {
+ PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingEnabled) {
this.servicePoliciesFileURL = servicePoliciesFileURL;
this.requestFileURLs = requestFileURLs;
this.statCollectionFileURL = statCollectionFileURL;
this.iterationsCount = iterationsCount;
this.concurrentClientCount = concurrentClientCount;
+ this.isDynamicReorderingEnabled = isDynamicReorderingEnabled;
}
public URL getServicePoliciesFileURL() {
@@ -57,4 +59,7 @@ public class PerfTestOptions {
public int getIterationsCount() {
return iterationsCount;
- }}
+ }
+
+ public boolean getIsDynamicReorderingEnabled() { return isDynamicReorderingEnabled; }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
index e752c62..50432f7 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
@@ -54,7 +54,7 @@ public class RangerPolicyenginePerfTester {
URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL();
- PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL);
+ PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, perfTestOptions.getIsDynamicReorderingEnabled());
if (!perfTestEngine.init()) {
LOG.error("Error initializing test data. Existing...");
System.exit(1);
@@ -63,6 +63,22 @@ public class RangerPolicyenginePerfTester {
URL[] requestFileURLs = perfTestOptions.getRequestFileURLs();
int requestFilesCount = requestFileURLs.length;
+ // warm-up policy engine
+ LOG.error("Warming up..");
+ try {
+ for(URL requestFileURL : requestFileURLs) {
+ PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, 0, requestFileURL, 1);
+
+ perfTestClient.init();
+ perfTestClient.run();
+ }
+ } catch(Throwable t) {
+ LOG.error("Error during warmup", t);
+ }
+ LOG.error("Warmed up!");
+
+ PerfDataRecorder.clearStatistics();
+
int clientsCount = perfTestOptions.getConcurrentClientCount();
List<PerfTestClient> perfTestClients = new ArrayList<PerfTestClient>(clientsCount);
@@ -89,12 +105,9 @@ public class RangerPolicyenginePerfTester {
long totalMemory = runtime.totalMemory();
long freeMemory = runtime.freeMemory();
- LOG.info("Initial Memory Statistics:");
- LOG.info("\t\tMaximum memory available for the process:\t" + runtime.maxMemory());
- LOG.info("\t\tInitial In-Use memory:\t\t\t\t" + (totalMemory-freeMemory));
- LOG.info("\t\tInitial Free memory:\t\t\t\t" + freeMemory);
- LOG.info("\n\n");
+ LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);
+ LOG.info("Starting " + perfTestClients.size() + " clients..");
for (PerfTestClient client : perfTestClients) {
try {
client.start();
@@ -102,13 +115,13 @@ public class RangerPolicyenginePerfTester {
LOG.error("Error in starting client: " + client.getName(), t);
}
}
+ LOG.info("Started " + perfTestClients.size() + " clients");
LOG.info("Waiting for " + perfTestClients.size() + " clients to finish up");
for (PerfTestClient client : perfTestClients) {
while (client.isAlive()) {
try {
- LOG.info("Waiting for " + client.getName() + " to finish up.");
client.join(1000);
runtime.gc();
@@ -116,10 +129,7 @@ public class RangerPolicyenginePerfTester {
totalMemory = runtime.totalMemory();
freeMemory = runtime.freeMemory();
- LOG.info("Memory Statistics:");
- LOG.info("\t\tCurrent In-Use memory:\t\t" + (totalMemory-freeMemory));
- LOG.info("\t\tCurrent Free memory:\t\t" + freeMemory);
- LOG.info("\n\n");
+ LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);
} catch (InterruptedException interruptedException) {
LOG.error("PerfTestClient.join() was interrupted");
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/test/resources/testdata/test_modules.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/testdata/test_modules.txt b/ranger-tools/src/test/resources/testdata/test_modules.txt
index 8eb5746..a355ec8 100644
--- a/ranger-tools/src/test/resources/testdata/test_modules.txt
+++ b/ranger-tools/src/test/resources/testdata/test_modules.txt
@@ -21,4 +21,5 @@ RangerTagEnricher.setServiceTags
RangerPolicyEngine.init
RangerPolicyEngine.preProcess
RangerPolicyEngine.isAccessAllowedNoAudit
-
+RangerPolicyEngine.reorderPolicyEvaluators
+RangerPolicyEngine.usage
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/testdata/test_modules.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/testdata/test_modules.txt b/ranger-tools/testdata/test_modules.txt
index f317aaf..f3a2f82 100644
--- a/ranger-tools/testdata/test_modules.txt
+++ b/ranger-tools/testdata/test_modules.txt
@@ -20,6 +20,8 @@
PolicyRefresher.loadPolicy
RangerPolicyEngine.init
RangerPolicyEngine.cleanUp
+RangerPolicyEngine.reorderPolicyEvaluators
+RangerPolicyEngine.usage
RangerContextEnricher.init
RangerPolicyEvaluator.init
RangerPolicyItemEvaluator.init
@@ -58,4 +60,4 @@ ServiceREST.getPolicy
ServiceREST.getPolicies
ServiceREST.countPolicies
ServiceREST.getServicePolicies
-ServiceREST.getServicePoliciesIfUpdated
\ No newline at end of file
+ServiceREST.getServicePoliciesIfUpdated