You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2016/08/30 00:08:13 UTC
incubator-ranger git commit: RANGER-1162: optimize retrieval of tags
for resource, using trie lookup
Repository: incubator-ranger
Updated Branches:
refs/heads/master 682424276 -> ff4e2e7b6
RANGER-1162: optimize retrieval of tags for resource, using trie lookup
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/ff4e2e7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/ff4e2e7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/ff4e2e7b
Branch: refs/heads/master
Commit: ff4e2e7b6847679388a13ddd8396e191e0106f18
Parents: 6824242
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Fri Aug 26 01:31:05 2016 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Aug 29 16:32:56 2016 -0700
----------------------------------------------------------------------
.../RangerServiceResourceMatcher.java | 47 +++-
.../contextenricher/RangerTagEnricher.java | 103 ++++++-
.../policyengine/RangerPolicyEngineImpl.java | 6 +-
.../policyengine/RangerPolicyRepository.java | 8 +-
.../RangerAbstractPolicyEvaluator.java | 60 ++--
.../policyevaluator/RangerPolicyEvaluator.java | 9 +-
.../RangerDefaultPolicyResourceMatcher.java | 5 +
.../RangerPolicyResourceEvaluator.java | 38 +++
.../RangerPolicyResourceMatcher.java | 2 +
.../ranger/plugin/util/RangerResourceTrie.java | 273 ++++++++++++-------
.../ranger/plugin/util/ServiceDefUtil.java | 37 +++
.../plugin/contextenricher/TestTagEnricher.java | 156 +++++++++++
.../plugin/policyengine/TestPolicyEngine.java | 3 +-
.../contextenricher/test_tagenricher_hive.json | 77 ++++++
14 files changed, 678 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
index 5c1ae64..cf7b8e7 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
@@ -19,24 +19,63 @@
package org.apache.ranger.plugin.contextenricher;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.policyengine.RangerAccessResource;
import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
+import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator;
+import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
+import org.apache.ranger.plugin.util.ServiceDefUtil;
-public class RangerServiceResourceMatcher {
- private final RangerServiceResource serviceResource;
+import java.util.Map;
+
+public class RangerServiceResourceMatcher implements RangerPolicyResourceEvaluator {
+ private final RangerServiceResource serviceResource;
private final RangerPolicyResourceMatcher policyResourceMatcher;
+ private final Integer leafResourceLevel;
public RangerServiceResourceMatcher(final RangerServiceResource serviceResource, RangerPolicyResourceMatcher policyResourceMatcher) {
- this.serviceResource = serviceResource;
+ this.serviceResource = serviceResource;
this.policyResourceMatcher = policyResourceMatcher;
+ this.leafResourceLevel = ServiceDefUtil.getLeafResourceLevel(getServiceDef(), getPolicyResource());
}
public RangerServiceResource getServiceResource() { return serviceResource; }
+ @Override
+ public long getId() {
+ return serviceResource != null ? serviceResource.getId() :-1;
+ }
+
+ @Override
public RangerPolicyResourceMatcher getPolicyResourceMatcher() { return policyResourceMatcher; }
+ @Override
+ public Map<String, RangerPolicy.RangerPolicyResource> getPolicyResource() {
+ return serviceResource != null ? serviceResource.getResourceElements() : null;
+ }
+
+ @Override
+ public RangerResourceMatcher getResourceMatcher(String resourceName) {
+ return policyResourceMatcher != null ? policyResourceMatcher.getResourceMatcher(resourceName) : null;
+ }
+
+ @Override
+ public Integer getLeafResourceLevel() {
+ return leafResourceLevel;
+ }
+
+ @Override
+ public int compareTo(RangerPolicyResourceEvaluator other) {
+ return Long.compare(getId(), other.getId());
+ }
+
public boolean isMatch(RangerAccessResource requestedResource) {
- return this.policyResourceMatcher.isExactHeadMatch(requestedResource);
+ return policyResourceMatcher != null ? policyResourceMatcher.isExactHeadMatch(requestedResource) : false;
+ }
+
+ RangerServiceDef getServiceDef() {
+ return policyResourceMatcher != null ? policyResourceMatcher.getServiceDef() : null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index c397ca1..5b60a53 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -27,6 +27,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
@@ -34,12 +35,16 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResource;
import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.apache.ranger.plugin.util.RangerResourceTrie;
import org.apache.ranger.plugin.util.ServiceTags;
import java.io.*;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
public class RangerTagEnricher extends RangerAbstractContextEnricher {
private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class);
@@ -47,16 +52,15 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init");
public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval";
+ public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName";
+ public static final String TAG_DISABLE_TRIE_PREFILTER_OPTION = "disableTrieLookupPrefilter";
- public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName";
-
- private RangerTagRefresher tagRefresher = null;
-
- private RangerTagRetriever tagRetriever = null;
-
- ServiceTags serviceTags = null;
-
- List<RangerServiceResourceMatcher> serviceResourceMatchers;
+ private RangerTagRefresher tagRefresher = null;
+ private RangerTagRetriever tagRetriever = null;
+ private ServiceTags serviceTags = null;
+ private List<RangerServiceResourceMatcher> serviceResourceMatchers = null;
+ private Map<String, RangerResourceTrie> serviceResourceTrie = null;
+ private boolean disableTrieLookupPrefilter = false;
@Override
public void init() {
@@ -70,6 +74,8 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
long pollingIntervalMs = getLongOption(TAG_REFRESHER_POLLINGINTERVAL_OPTION, 60 * 1000);
+ disableTrieLookupPrefilter = getBooleanOption(TAG_DISABLE_TRIE_PREFILTER_OPTION, false);
+
if (StringUtils.isNotBlank(tagRetrieverClassName)) {
try {
@@ -162,8 +168,19 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
}
+ Map<String, RangerResourceTrie> serviceResourceTrie = null;
+
+ if(!disableTrieLookupPrefilter) {
+ serviceResourceTrie = new HashMap<String, RangerResourceTrie>();
+
+ for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
+ serviceResourceTrie.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, resourceMatchers));
+ }
+ }
+
this.serviceResourceMatchers = resourceMatchers;
- this.serviceTags = serviceTags;
+ this.serviceResourceTrie = serviceResourceTrie;
+ this.serviceTags = serviceTags;
}
@Override
@@ -191,7 +208,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
}
List<RangerTag> ret = null;
- final List<RangerServiceResourceMatcher> serviceResourceMatchers = this.serviceResourceMatchers;
+ final List<RangerServiceResourceMatcher> serviceResourceMatchers = getEvaluators(resource);
if (CollectionUtils.isNotEmpty(serviceResourceMatchers)) {
@@ -226,6 +243,70 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
return ret;
}
+ private List<RangerServiceResourceMatcher> getEvaluators(RangerAccessResource resource) {
+ List<RangerServiceResourceMatcher> ret = null;
+
+ if(serviceResourceTrie == null) {
+ ret = serviceResourceMatchers;
+ } else {
+ Set<String> resourceKeys = resource == null ? null : resource.getKeys();
+
+ if (CollectionUtils.isNotEmpty(resourceKeys)) {
+ boolean isRetModifiable = false;
+
+ for (String resourceName : resourceKeys) {
+ RangerResourceTrie trie = serviceResourceTrie.get(resourceName);
+
+ if (trie == null) { // if no trie exists for this resource level, ignore and continue to next level
+ continue;
+ }
+
+ List<RangerServiceResourceMatcher> resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName));
+
+ if (CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out
+ ret = null;
+ } else if (ret == null) { // initialize ret with policies found for this resource
+ ret = resourceEvaluators;
+ } else { // remove policies from ret that are not in resourceEvaluators
+ if (isRetModifiable) {
+ ret.retainAll(resourceEvaluators);
+ } else {
+ final List<RangerServiceResourceMatcher> shorterList;
+ final List<RangerServiceResourceMatcher> longerList;
+
+ if (ret.size() < resourceEvaluators.size()) {
+ shorterList = ret;
+ longerList = resourceEvaluators;
+ } else {
+ shorterList = resourceEvaluators;
+ longerList = ret;
+ }
+
+ ret = new ArrayList<>(shorterList);
+ ret.retainAll(longerList);
+ isRetModifiable = true;
+ }
+ }
+
+ if (CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list
+ ret = null;
+ break;
+ }
+ }
+ }
+ }
+
+ if(ret == null) {
+ ret = Collections.emptyList();
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerTagEnricher.getEvaluators(" + resource.getAsString() + "): evaluatorCount=" + ret.size());
+ }
+
+ return ret;
+ }
+
static private List<RangerTag> getTagsForServiceResource(final ServiceTags serviceTags, final RangerServiceResource serviceResource) {
List<RangerTag> ret = new ArrayList<RangerTag>();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
index d0e8d62..6d3645f 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
@@ -706,12 +706,12 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
@Override
public void reorderPolicyEvaluators() {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> reorderPolicyEvaluators()");
+ LOG.debug("==> reorderEvaluators()");
}
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REBALANCE_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderPolicyEvaluators()");
+ perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderEvaluators()");
}
if (MapUtils.isNotEmpty(policyEvaluatorsMap)) {
for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) {
@@ -735,7 +735,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
RangerPerfTracer.log(perf);
if (LOG.isDebugEnabled()) {
- LOG.debug("<== reorderPolicyEvaluators()");
+ LOG.debug("<== reorderEvaluators()");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index d06aecd..d0b3f09 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -253,7 +253,7 @@ class RangerPolicyRepository {
continue;
}
- List<RangerPolicyEvaluator> resourceEvaluators = trie.getPoliciesForResource(resource.getValue(resourceName));
+ List<RangerPolicyEvaluator> resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName));
if(CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out
ret = null;
@@ -729,7 +729,7 @@ class RangerPolicyRepository {
void reorderPolicyEvaluators() {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> reorderPolicyEvaluators()");
+ LOG.debug("==> reorderEvaluators()");
}
if(disableTrieLookupPrefilter) {
@@ -743,7 +743,7 @@ class RangerPolicyRepository {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== reorderPolicyEvaluators()");
+ LOG.debug("<== reorderEvaluators()");
}
}
@@ -753,7 +753,7 @@ class RangerPolicyRepository {
RangerResourceTrie trie = entry.getValue();
if(trie != null) {
- trie.reorderPolicyEvaluators();
+ trie.reorderEvaluators();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
index 8f45661..9b48dfe 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
@@ -28,14 +28,19 @@ import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
+import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator;
+import org.apache.ranger.plugin.util.ServiceDefUtil;
+
+import java.util.Map;
public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvaluator {
private static final Log LOG = LogFactory.getLog(RangerAbstractPolicyEvaluator.class);
- private RangerPolicy policy = null;
- private RangerServiceDef serviceDef = null;
- private int evalOrder = 0;
- protected long usageCount = 0;
+ private RangerPolicy policy = null;
+ private RangerServiceDef serviceDef = null;
+ private Integer leafResourceLevel = null;
+ private int evalOrder = 0;
+ protected long usageCount = 0;
protected boolean usageCountMutable = true;
@@ -45,8 +50,9 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
LOG.debug("==> RangerAbstractPolicyEvaluator.init(" + policy + ", " + serviceDef + ")");
}
- this.policy = policy;
- this.serviceDef = serviceDef;
+ this.policy = policy;
+ this.serviceDef = serviceDef;
+ this.leafResourceLevel = ServiceDefUtil.getLeafResourceLevel(serviceDef, getPolicyResource());
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerAbstractPolicyEvaluator.init(" + policy + ", " + serviceDef + ")");
@@ -54,6 +60,16 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
}
@Override
+ public long getId() {
+ return policy != null ? policy.getId() :-1;
+ }
+
+ @Override
+ public Map<String, RangerPolicy.RangerPolicyResource> getPolicyResource() {
+ return policy !=null ? policy.getResources() : null;
+ }
+
+ @Override
public RangerPolicy getPolicy() {
return policy;
}
@@ -63,6 +79,11 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
return serviceDef;
}
+ @Override
+ public Integer getLeafResourceLevel() {
+ return leafResourceLevel;
+ }
+
public boolean hasAllow() {
return policy != null && CollectionUtils.isNotEmpty(policy.getPolicyItems());
}
@@ -90,21 +111,28 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
}
@Override
- public int compareTo(RangerPolicyEvaluator other) {
+ public int compareTo(RangerPolicyResourceEvaluator obj) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> RangerAbstractPolicyEvaluator.compareTo()");
+ LOG.debug("==> RangerAbstractPolicyEvaluator.compareTo()");
}
int result;
- if (hasDeny() && !other.hasDeny()) {
- result = -1;
- } else if (!hasDeny() && other.hasDeny()) {
- result = 1;
- } else {
- result = Long.compare(other.getUsageCount(), this.usageCount);
- if (result == 0) {
- result = Integer.compare(this.evalOrder, other.getEvalOrder());
+
+ if(obj instanceof RangerPolicyEvaluator) {
+ RangerPolicyEvaluator other = (RangerPolicyEvaluator)obj;
+
+ if (hasDeny() && !other.hasDeny()) {
+ result = -1;
+ } else if (!hasDeny() && other.hasDeny()) {
+ result = 1;
+ } else {
+ result = Long.compare(other.getUsageCount(), this.usageCount);
+ if (result == 0) {
+ result = Integer.compare(this.evalOrder, other.getEvalOrder());
+ }
}
+ } else {
+ result = Long.compare(getId(), obj.getId());
}
if(LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
index 23069cf..b60e06e 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
@@ -33,11 +33,10 @@ import org.apache.ranger.plugin.policyengine.RangerDataMaskResult;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo;
import org.apache.ranger.plugin.policyengine.RangerRowFilterResult;
-import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
-import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
+import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator;
-public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> {
+public interface RangerPolicyEvaluator extends RangerPolicyResourceEvaluator {
String EVALUATOR_TYPE_AUTO = "auto";
String EVALUATOR_TYPE_OPTIMIZED = "optimized";
String EVALUATOR_TYPE_CACHED = "cached";
@@ -66,10 +65,6 @@ public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator>
boolean isAuditEnabled();
- RangerPolicyResourceMatcher getPolicyResourceMatcher();
-
- RangerResourceMatcher getResourceMatcher(String resourceName);
-
void evaluate(RangerAccessRequest request, RangerAccessResult result);
void evaluate(RangerAccessRequest request, RangerDataMaskResult result);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
index 78589cd..8bde807 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
@@ -180,6 +180,11 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM
}
@Override
+ public RangerServiceDef getServiceDef() {
+ return serviceDef;
+ }
+
+ @Override
public RangerResourceMatcher getResourceMatcher(String resourceName) {
return matchers != null ? matchers.get(resourceName) : null;
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
new file mode 100644
index 0000000..eed58e1
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.policyresourcematcher;
+
+
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
+
+import java.util.Map;
+
+public interface RangerPolicyResourceEvaluator extends Comparable<RangerPolicyResourceEvaluator> {
+ long getId();
+
+ RangerPolicyResourceMatcher getPolicyResourceMatcher();
+
+ Map<String, RangerPolicy.RangerPolicyResource> getPolicyResource();
+
+ RangerResourceMatcher getResourceMatcher(String resourceName);
+
+ Integer getLeafResourceLevel();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java
index 80da868..a8810e5 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java
@@ -36,6 +36,8 @@ public interface RangerPolicyResourceMatcher {
void init();
+ RangerServiceDef getServiceDef();
+
RangerResourceMatcher getResourceMatcher(String resourceName);
boolean isMatch(RangerAccessResource resource);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index a5ffd1a..2079487 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -23,10 +23,9 @@ package org.apache.ranger.plugin.util;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerServiceDef;
-import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
+import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator;
import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher;
import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
@@ -37,7 +36,7 @@ import java.util.List;
import java.util.Map;
-public class RangerResourceTrie {
+public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class);
private static final String DEFAULT_WILDCARD_CHARS = "*?";
@@ -48,7 +47,7 @@ public class RangerResourceTrie {
private final String wildcardChars;
private final TrieNode root;
- public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<RangerPolicyEvaluator> evaluators) {
+ public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<T> evaluators) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ")");
}
@@ -63,31 +62,38 @@ public class RangerResourceTrie {
this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS : "";
this.root = new TrieNode(Character.valueOf((char)0));
- for(RangerPolicyEvaluator evaluator : evaluators) {
- RangerPolicy policy = evaluator.getPolicy();
- Map<String, RangerPolicyResource> policyResources = policy != null ? policy.getResources() : null;
+ for(T evaluator : evaluators) {
+ Map<String, RangerPolicyResource> policyResources = evaluator.getPolicyResource();
RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null;
if(policyResource == null) {
+ if(evaluator.getLeafResourceLevel() != null && resourceDef.getLevel() != null && evaluator.getLeafResourceLevel() < resourceDef.getLevel()) {
+ root.addWildcardEvaluator(evaluator);
+ }
+
continue;
}
if(policyResource.getIsExcludes()) {
- root.addWildcardPolicy(evaluator);
+ root.addWildcardEvaluator(evaluator);
} else {
RangerResourceMatcher resourceMatcher = evaluator.getResourceMatcher(resourceName);
if(resourceMatcher != null && resourceMatcher.isMatchAny()) {
- root.addWildcardPolicy(evaluator);
+ root.addWildcardEvaluator(evaluator);
} else {
- for (String resource : policyResource.getValues()) {
- insert(resource, policyResource.getIsRecursive(), evaluator);
+ if(CollectionUtils.isNotEmpty(policyResource.getValues())) {
+ for (String resource : policyResource.getValues()) {
+ insert(resource, policyResource.getIsRecursive(), evaluator);
+ }
}
}
}
}
- root.postSetup();
+ root.postSetup(null);
+
+ LOG.info(toString());
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + "): " + toString());
@@ -98,12 +104,12 @@ public class RangerResourceTrie {
return resourceName;
}
- public List<RangerPolicyEvaluator> getPoliciesForResource(String resource) {
+ public List<T> getEvaluatorsForResource(String resource) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> RangerResourceTrie.getPoliciesForResource(" + resource + ")");
+ LOG.debug("==> RangerResourceTrie.getEvaluatorsForResource(" + resource + ")");
}
- List<RangerPolicyEvaluator> ret = null;
+ List<T> ret = null;
TrieNode curr = root;
@@ -113,8 +119,8 @@ public class RangerResourceTrie {
TrieNode child = curr.getChild(ch);
if(child == null) {
- ret = curr.getWildcardPolicies();
- curr = null; // so that curr.getPolicies() will not be called below
+ ret = curr.getWildcardEvaluators();
+ curr = null; // so that curr.getEvaluators() will not be called below
break;
}
@@ -123,41 +129,46 @@ public class RangerResourceTrie {
if(ret == null) {
if(curr != null) {
- ret = curr.getPolicies();
+ ret = curr.getEvaluators();
}
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== RangerResourceTrie.getPoliciesForResource(" + resource + "): evaluatorCount=" + (ret == null ? 0 : ret.size()));
+ LOG.debug("<== RangerResourceTrie.getEvaluatorsForResource(" + resource + "): evaluatorCount=" + (ret == null ? 0 : ret.size()));
}
return ret;
}
- public int getNodeCount() {
- return root.getNodeCount();
+ public TrieData getTrieData() {
+ TrieData ret = new TrieData();
+
+ root.populateTrieData(ret);
+ ret.maxDepth = getMaxDepth();
+
+ return ret;
}
public int getMaxDepth() {
return root.getMaxDepth();
}
- public void reorderPolicyEvaluators() {
- root.reorderPolicyEvaluators();
+ public void reorderEvaluators() {
+ root.reorderEvaluators(null);
}
- private Character getLookupChar(char ch) {
- return optIgnoreCase ? Character.valueOf(Character.toLowerCase(ch)) : Character.valueOf(ch);
+ private final Character getLookupChar(char ch) {
+ if(optIgnoreCase) {
+ ch = Character.toLowerCase(ch);
+ }
+
+ return Character.valueOf(ch);
}
- private void insert(String resource, boolean isRecursive, RangerPolicyEvaluator evaluator) {
+ private void insert(String resource, boolean isRecursive, T evaluator) {
TrieNode curr = root;
boolean isWildcard = false;
- if(optIgnoreCase) {
- resource = resource.toLowerCase();
- }
-
final int len = resource.length();
for(int i = 0; i < len; i++) {
Character ch = getLookupChar(resource.charAt(i));
@@ -173,9 +184,9 @@ public class RangerResourceTrie {
}
if(isWildcard || isRecursive) {
- curr.addWildcardPolicy(evaluator);
+ curr.addWildcardEvaluator(evaluator);
} else {
- curr.addPolicy(evaluator);
+ curr.addEvaluator(evaluator);
}
}
@@ -183,20 +194,42 @@ public class RangerResourceTrie {
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("nodeCount=").append(getNodeCount());
- sb.append("; maxDepth=").append(getMaxDepth());
- sb.append(Character.LINE_SEPARATOR);
- root.toString("", sb);
+ TrieData trieData = getTrieData();
+
+ sb.append("resourceName=").append(resourceName);
+ sb.append("; optIgnoreCase=").append(optIgnoreCase);
+ sb.append("; optWildcard=").append(optWildcard);
+ sb.append("; wildcardChars=").append(wildcardChars);
+ sb.append("; nodeCount=").append(trieData.nodeCount);
+ sb.append("; leafNodeCount=").append(trieData.leafNodeCount);
+ sb.append("; singleChildNodeCount=").append(trieData.singleChildNodeCount);
+ sb.append("; maxDepth=").append(trieData.maxDepth);
+ sb.append("; evaluatorListCount=").append(trieData.evaluatorListCount);
+ sb.append("; wildcardEvaluatorListCount=").append(trieData.wildcardEvaluatorListCount);
+ sb.append("; evaluatorListRefCount=").append(trieData.evaluatorListRefCount);
+ sb.append("; wildcardEvaluatorListRefCount=").append(trieData.wildcardEvaluatorListRefCount);
return sb.toString();
}
+
+ public class TrieData {
+ int nodeCount = 0;
+ int leafNodeCount = 0;
+ int singleChildNodeCount = 0;
+ int maxDepth = 0;
+ int evaluatorListCount = 0;
+ int wildcardEvaluatorListCount = 0;
+ int evaluatorListRefCount = 0;
+ int wildcardEvaluatorListRefCount = 0;
+ }
}
-class TrieNode {
- private final Character c;
- private Map<Character, TrieNode> children = null;
- private List<RangerPolicyEvaluator> policies = null;
- private List<RangerPolicyEvaluator> wildcardPolicies = null;
+class TrieNode<T extends RangerPolicyResourceEvaluator> {
+ private final Character c;
+ private Map<Character, TrieNode> children = null;
+ private List<T> evaluators = null;
+ private List<T> wildcardEvaluators = null;
+ private boolean isSharingParentWildcardEvaluators = false;
TrieNode(Character c) {
this.c = c;
@@ -210,12 +243,12 @@ class TrieNode {
return children;
}
- List<RangerPolicyEvaluator> getPolicies() {
- return policies;
+ List<T> getEvaluators() {
+ return evaluators;
}
- List<RangerPolicyEvaluator> getWildcardPolicies() {
- return wildcardPolicies;
+ List<T> getWildcardEvaluators() {
+ return wildcardEvaluators;
}
TrieNode getChild(Character c) {
@@ -224,18 +257,38 @@ class TrieNode {
return ret;
}
- int getNodeCount() {
- int ret = 1;
+ void populateTrieData(RangerResourceTrie.TrieData trieData) {
+ trieData.nodeCount++;
+
+ if(wildcardEvaluators != null) {
+ if(isSharingParentWildcardEvaluators) {
+ trieData.wildcardEvaluatorListRefCount++;
+ } else {
+ trieData.wildcardEvaluatorListCount++;
+ }
+ }
+
+ if(evaluators != null) {
+ if(evaluators == wildcardEvaluators) {
+ trieData.evaluatorListRefCount++;
+ } else {
+ trieData.evaluatorListCount++;
+ }
+ }
+
+ if(children != null && children.size() > 0) {
+ if(children.size() == 1) {
+ trieData.singleChildNodeCount++;
+ }
- if(children != null) {
for(Map.Entry<Character, TrieNode> entry : children.entrySet()) {
TrieNode child = entry.getValue();
- ret += child.getNodeCount();
+ child.populateTrieData(trieData);
}
+ } else {
+ trieData.leafNodeCount++;
}
-
- return ret;
}
int getMaxDepth() {
@@ -271,69 +324,91 @@ class TrieNode {
return child;
}
- void addPolicy(RangerPolicyEvaluator evaluator) {
- if(policies == null) {
- policies = new ArrayList<RangerPolicyEvaluator>();
+ void addEvaluator(T evaluator) {
+ if(evaluators == null) {
+ evaluators = new ArrayList<T>();
}
- if(!policies.contains(evaluator)) {
- policies.add(evaluator);
+ if(!evaluators.contains(evaluator)) {
+ evaluators.add(evaluator);
}
}
- void addPolicies(List<RangerPolicyEvaluator> evaluators) {
- if(CollectionUtils.isNotEmpty(evaluators)) {
- for(RangerPolicyEvaluator evaluator : evaluators) {
- addPolicy(evaluator);
- }
+ void addWildcardEvaluator(T evaluator) {
+ if(wildcardEvaluators == null) {
+ wildcardEvaluators = new ArrayList<T>();
+ }
+
+ if(!wildcardEvaluators.contains(evaluator)) {
+ wildcardEvaluators.add(evaluator);
}
}
- void addWildcardPolicy(RangerPolicyEvaluator evaluator) {
- if(wildcardPolicies == null) {
- wildcardPolicies = new ArrayList<RangerPolicyEvaluator>();
+ void postSetup(List<T> parentWildcardEvaluators) {
+ // finalize wildcard-evaluators list by including parent's wildcard evaluators
+ if(parentWildcardEvaluators != null) {
+ if(CollectionUtils.isEmpty(this.wildcardEvaluators)) {
+ this.wildcardEvaluators = parentWildcardEvaluators;
+ } else {
+ for (T evaluator : parentWildcardEvaluators) {
+ addWildcardEvaluator(evaluator);
+ }
+ }
}
+ this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators;
- if(!wildcardPolicies.contains(evaluator)) {
- wildcardPolicies.add(evaluator);
+ // finalize evaluators list by including wildcard evaluators
+ if(wildcardEvaluators != null) {
+ if(CollectionUtils.isEmpty(this.evaluators)) {
+ this.evaluators = wildcardEvaluators;
+ } else {
+ for (T evaluator : wildcardEvaluators) {
+ addEvaluator(evaluator);
+ }
+ }
}
- }
- void addWildcardPolicies(List<RangerPolicyEvaluator> evaluators) {
- if(CollectionUtils.isNotEmpty(evaluators)) {
- for(RangerPolicyEvaluator evaluator : evaluators) {
- addWildcardPolicy(evaluator);
+ if(!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) {
+ Collections.sort(wildcardEvaluators);
+ }
+
+ if(evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) {
+ Collections.sort(evaluators);
+ }
+
+ if(children != null) {
+ for(Map.Entry<Character, TrieNode> entry : children.entrySet()) {
+ TrieNode child = entry.getValue();
+
+ child.postSetup(wildcardEvaluators);
}
}
}
- void postSetup() {
- addPolicies(wildcardPolicies);
+ void reorderEvaluators(List<T> parentWildcardEvaluators) {
+ boolean isEvaluatorsSameAsWildcardEvaluators = evaluators == wildcardEvaluators;
- if(wildcardPolicies != null) {
- Collections.sort(wildcardPolicies);
+ if(isSharingParentWildcardEvaluators) {
+ wildcardEvaluators = parentWildcardEvaluators;
+ } else {
+ wildcardEvaluators = getSortedCopy(wildcardEvaluators);
}
- if(policies != null) {
- Collections.sort(policies);
+ if(isEvaluatorsSameAsWildcardEvaluators) {
+ evaluators = wildcardEvaluators;
+ } else {
+ evaluators = getSortedCopy(evaluators);
}
if(children != null) {
for(Map.Entry<Character, TrieNode> entry : children.entrySet()) {
TrieNode child = entry.getValue();
- child.addWildcardPolicies(wildcardPolicies);
-
- child.postSetup();
+ child.reorderEvaluators(wildcardEvaluators);
}
}
}
- void reorderPolicyEvaluators() {
- wildcardPolicies = getSortedCopy(wildcardPolicies);
- policies = getSortedCopy(policies);
- }
-
public void toString(String prefix, StringBuilder sb) {
String nodeValue = prefix;
@@ -343,18 +418,18 @@ class TrieNode {
sb.append("nodeValue=").append(nodeValue);
sb.append("; childCount=").append(children == null ? 0 : children.size());
- sb.append("; policies=[ ");
- if(policies != null) {
- for(RangerPolicyEvaluator evaluator : policies) {
- sb.append(evaluator.getPolicy().getId()).append(" ");
+ sb.append("; evaluators=[ ");
+ if(evaluators != null) {
+ for(T evaluator : evaluators) {
+ sb.append(evaluator.getId()).append(" ");
}
}
sb.append("]");
- sb.append("; wildcardPolicies=[ ");
- if(wildcardPolicies != null) {
- for(RangerPolicyEvaluator evaluator : wildcardPolicies) {
- sb.append(evaluator.getPolicy().getId()).append(" ");
+ sb.append("; wildcardEvaluators=[ ");
+ if(wildcardEvaluators != null) {
+ for(T evaluator : wildcardEvaluators) {
+ sb.append(evaluator.getId()).append(" ");
}
}
sb.append("]");
@@ -370,16 +445,16 @@ class TrieNode {
}
public void clear() {
- children = null;
- policies = null;
- wildcardPolicies = null;
+ children = null;
+ evaluators = null;
+ wildcardEvaluators = null;
}
- private List<RangerPolicyEvaluator> getSortedCopy(List<RangerPolicyEvaluator> evaluators) {
- final List<RangerPolicyEvaluator> ret;
+ private List<T> getSortedCopy(List<T> evaluators) {
+ final List<T> ret;
if(CollectionUtils.isNotEmpty(evaluators)) {
- ret = new ArrayList<RangerPolicyEvaluator>(wildcardPolicies);
+ ret = new ArrayList<T>(evaluators);
Collections.sort(ret);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
index eaf60b7..0a2b451 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
@@ -22,6 +22,7 @@ package org.apache.ranger.plugin.util;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef;
@@ -73,6 +74,42 @@ public class ServiceDefUtil {
return serviceDef;
}
+ public static RangerResourceDef getResourceDef(RangerServiceDef serviceDef, String resource) {
+ RangerResourceDef ret = null;
+
+ if(serviceDef != null && resource != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) {
+ for(RangerResourceDef resourceDef : serviceDef.getResources()) {
+ if(StringUtils.equalsIgnoreCase(resourceDef.getName(), resource)) {
+ ret = resourceDef;
+ break;
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ public static Integer getLeafResourceLevel(RangerServiceDef serviceDef, Map<String, RangerPolicy.RangerPolicyResource> policyResource) {
+ Integer ret = null;
+
+ if(serviceDef != null && policyResource != null) {
+ for(Map.Entry<String, RangerPolicy.RangerPolicyResource> entry : policyResource.entrySet()) {
+ String resource = entry.getKey();
+ RangerResourceDef resourceDef = ServiceDefUtil.getResourceDef(serviceDef, resource);
+
+ if(resourceDef != null && resourceDef.getLevel() != null) {
+ if(ret == null) {
+ ret = resourceDef.getLevel();
+ } else if(ret < resourceDef.getLevel()) {
+ ret = resourceDef.getLevel();
+ }
+ }
+ }
+ }
+
+ return ret;
+ }
+
private static void normalizeDataMaskDef(RangerServiceDef serviceDef) {
if(serviceDef != null && serviceDef.getDataMaskDef() != null) {
List<RangerResourceDef> dataMaskResources = serviceDef.getDataMaskDef().getResources();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
new file mode 100644
index 0000000..30190ab
--- /dev/null
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.contextenricher;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
+import org.apache.ranger.plugin.contextenricher.TestTagEnricher.TagEnricherTestCase.TestData;
+import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.model.RangerServiceResource;
+import org.apache.ranger.plugin.model.RangerTag;
+import org.apache.ranger.plugin.model.RangerTagDef;
+import org.apache.ranger.plugin.policyengine.*;
+import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
+import org.apache.ranger.plugin.util.ServiceTags;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestTagEnricher {
+ static Gson gsonBuilder = null;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z")
+ .setPrettyPrinting()
+ .registerTypeAdapter(RangerAccessResource.class, new RangerResourceDeserializer())
+ .create();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ }
+
+ @Test
+ public void testTagEnricher_hive() {
+ String[] hiveTestResourceFiles = { "/contextenricher/test_tagenricher_hive.json" };
+
+ runTestsFromResourceFiles(hiveTestResourceFiles);
+ }
+
+ private void runTestsFromResourceFiles(String[] resourceNames) {
+ for(String resourceName : resourceNames) {
+ InputStream inStream = this.getClass().getResourceAsStream(resourceName);
+ InputStreamReader reader = new InputStreamReader(inStream);
+
+ runTests(reader, resourceName);
+ }
+ }
+
+ private void runTests(InputStreamReader reader, String testName) {
+ TagEnricherTestCase testCase = gsonBuilder.fromJson(reader, TagEnricherTestCase.class);
+
+ assertTrue("invalid input: " + testName, testCase != null && testCase.serviceDef != null && testCase.serviceResources != null && testCase.tests != null);
+
+ ServiceTags serviceTags = new ServiceTags();
+ serviceTags.setServiceName(testCase.serviceName);
+ serviceTags.setTagDefinitions(testCase.tagDefinitions);
+ serviceTags.setTags(testCase.tags);
+ serviceTags.setServiceResources(testCase.serviceResources);
+ serviceTags.setResourceToTagIds(testCase.resourceToTagIds);
+
+ RangerTagEnricher tagEnricher = new RangerTagEnricher();
+
+ tagEnricher.setServiceName(testCase.serviceName);
+ tagEnricher.setServiceDef(testCase.serviceDef);
+ tagEnricher.setServiceTags(serviceTags);
+
+ List<String> expectedTags = new ArrayList<String>();
+ List<String> resultTags = new ArrayList<String>();
+
+
+ for (TestData test : testCase.tests) {
+ RangerAccessRequestImpl request = new RangerAccessRequestImpl(test.resource, "", "testUser", null);
+
+ tagEnricher.enrich(request);
+
+ List<RangerTag> expected = test.result;
+ List<RangerTag> result = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext());
+
+ expectedTags.clear();
+ if(expected != null) {
+ for (RangerTag tag : expected) {
+ expectedTags.add(tag.getType());
+ }
+ Collections.sort(expectedTags);
+ }
+
+ resultTags.clear();
+ if(result != null) {
+ for(RangerTag tag : result) {
+ resultTags.add(tag.getType());
+ }
+ Collections.sort(resultTags);
+ }
+
+ assertEquals(test.name, expectedTags, resultTags);
+ }
+ }
+
+ static class TagEnricherTestCase {
+ public String serviceName;
+ public RangerServiceDef serviceDef;
+ public Map<Long, RangerTagDef> tagDefinitions;
+ public Map<Long, RangerTag> tags;
+ public List<RangerServiceResource> serviceResources;
+ public Map<Long, List<Long>> resourceToTagIds;
+ public List<TestData> tests;
+
+
+ class TestData {
+ public String name;
+ public RangerAccessResource resource;
+ public List<RangerTag> result;
+ }
+ }
+
+ static class RangerResourceDeserializer implements JsonDeserializer<RangerAccessResource> {
+ @Override
+ public RangerAccessResource deserialize(JsonElement jsonObj, Type type,
+ JsonDeserializationContext context) throws JsonParseException {
+ return gsonBuilder.fromJson(jsonObj, RangerAccessResourceImpl.class);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
----------------------------------------------------------------------
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
index f825f40..9937757 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
@@ -58,7 +58,6 @@ import static org.junit.Assert.*;
public class TestPolicyEngine {
- static RangerPolicyEngine policyEngine = null;
static Gson gsonBuilder = null;
@BeforeClass
@@ -304,7 +303,7 @@ public class TestPolicyEngine {
trustedProxyAddresses[i] = trustedProxyAddresses[i].trim();
}
}
- policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions);
+ RangerPolicyEngine policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions);
policyEngine.setUseForwardedIPAddress(useForwardedIPAddress);
policyEngine.setTrustedProxyAddresses(trustedProxyAddresses);
long requestCount = 0L;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json
----------------------------------------------------------------------
diff --git a/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json b/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json
new file mode 100644
index 0000000..317c651
--- /dev/null
+++ b/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json
@@ -0,0 +1,77 @@
+{
+ "serviceName":"hivedev",
+
+ "serviceDef":{
+ "name":"hive",
+ "id":3,
+ "resources":[
+ {"name":"database","level":1,"mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Database","description":"Hive Database"},
+ {"name":"table","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Table","description":"Hive Table"},
+ {"name":"udf","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive UDF","description":"Hive UDF"},
+ {"name":"column","level":3,"parent":"table","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Column","description":"Hive Column"}
+ ],
+ "accessTypes":[
+ {"name":"select","label":"Select"},
+ {"name":"update","label":"Update"},
+ {"name":"create","label":"Create"},
+ {"name":"drop","label":"Drop"},
+ {"name":"alter","label":"Alter"},
+ {"name":"index","label":"Index"},
+ {"name":"lock","label":"Lock"},
+ {"name":"all","label":"All"}
+ ]
+ },
+
+ "tagDefinitions": {
+ "1":{"name":"PII"},
+ "2":{"name":"EXPIRES_ON","attributeDefs":[{"name":"expiry_date","type":"date"}]},
+ "3":{"name":"FINANCE"}
+ },
+
+ "tags": {
+ "1":{"type":"PII"},
+ "2":{"type":"EXPIRES_ON","attributes":{"expiry_date":"2015/12/31"}},
+ "3":{"type":"FINANCE"}
+ },
+
+ "serviceResources": [
+ {"id":1,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["hr"]},"table":{"values":["employee"]},"column":{"values":["ssn"]}}},
+ {"id":2,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["finance"]}}},
+ {"id":3,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["finance"]},"table":{"values":["tax_2010"]}}},
+ {"id":4,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["finance"]},"table":{"values":["tax_2010"]},"column":{"values":["ssn"]}}}
+ ],
+
+ "resourceToTagIds": {
+ "1":[1],
+ "2":[3],
+ "3":[2],
+ "4":[1]
+ },
+
+ "tests":[
+ {"name":"hr.employee.ssn",
+ "resource":{"elements":{"database":"hr","table":"employee","column":"ssn"}},
+ "result":[{"type":"PII"}]
+ }
+ ,
+ {"name":"hr.employee.id",
+ "resource":{"elements":{"database":"hr","table":"employee","column":"id"}},
+ "result":[]
+ }
+ ,
+ {"name":"finance.tax_2010",
+ "resource":{"elements":{"database":"finance","table":"tax_2010"}},
+ "result":[{"type":"EXPIRES_ON"},{"type":"FINANCE"}]
+ }
+ ,
+ {"name":"finance.tax_2010.ssn",
+ "resource":{"elements":{"database":"finance","table":"tax_2010","column":"ssn"}},
+ "result":[{"type":"EXPIRES_ON"},{"type":"PII"},{"type":"FINANCE"}]
+ }
+ ,
+ {"name":"finance.tax_2010.id",
+ "resource":{"elements":{"database":"finance","table":"tax_2010","column":"id"}},
+ "result":[{"type":"EXPIRES_ON"},{"type":"FINANCE"}]
+ }
+ ]
+}