You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by me...@apache.org on 2018/09/04 12:58:11 UTC
[01/28] ranger git commit: RANGER-2147: Updated docs as part of
Ranger 1.1.0 release
Repository: ranger
Updated Branches:
refs/heads/ranger-1.1 747065d0b -> 8e19e67e2
RANGER-2147: Updated docs as part of Ranger 1.1.0 release
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/7eb78399
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/7eb78399
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/7eb78399
Branch: refs/heads/ranger-1.1
Commit: 7eb78399b3296dc4b1ebd80d936ca20ba5e443f4
Parents: 747065d
Author: Velmurugan Periasamy <ve...@apache.org>
Authored: Mon Jul 9 22:17:47 2018 -0400
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:15:19 2018 +0530
----------------------------------------------------------------------
docs/pom.xml | 2 +-
docs/src/site/site.xml | 1 +
docs/src/site/xdoc/download.xml | 12 +++++++++++-
3 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/7eb78399/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index 4b992e1..4cbd26e 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -24,7 +24,7 @@
<url>http://ranger.apache.org/</url>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.ranger</groupId>
- <version>1.1.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<artifactId>ranger</artifactId>
<packaging>pom</packaging>
<licenses>
http://git-wip-us.apache.org/repos/asf/ranger/blob/7eb78399/docs/src/site/site.xml
----------------------------------------------------------------------
diff --git a/docs/src/site/site.xml b/docs/src/site/site.xml
index 9cfb442..3da00cb 100644
--- a/docs/src/site/site.xml
+++ b/docs/src/site/site.xml
@@ -62,6 +62,7 @@ under the License.
<item name="Security Advisories" href="https://cwiki.apache.org/confluence/display/RANGER/Vulnerabilities+found+in+Ranger" />
</menu>
<menu name="Releases">
+ <item name="1.1.0" href="https://cwiki.apache.org/confluence/display/RANGER/Apache+Ranger+1.1.0+-+Release+Notes" />
<item name="1.0.0" href="https://cwiki.apache.org/confluence/display/RANGER/1.0.0+Release+-+Apache+Ranger" />
<item name="0.7.1" href="https://cwiki.apache.org/confluence/display/RANGER/0.7.1+Release+-+Apache+Ranger" />
<item name="0.7.0" href="https://cwiki.apache.org/confluence/display/RANGER/0.7.0+Release+-+Apache+Ranger" />
http://git-wip-us.apache.org/repos/asf/ranger/blob/7eb78399/docs/src/site/xdoc/download.xml
----------------------------------------------------------------------
diff --git a/docs/src/site/xdoc/download.xml b/docs/src/site/xdoc/download.xml
index 6a157fc..91f7cf1 100644
--- a/docs/src/site/xdoc/download.xml
+++ b/docs/src/site/xdoc/download.xml
@@ -31,7 +31,17 @@ LICENSE.txt and NOTICE.txt files contained in each release artifact.
<ul>
<li>
<p>
-Current Stable release is Apache Ranger 1.0.0:
+Current Stable release is Apache Ranger 1.1.0:
+</p>
+<p>
+<a href="https://www.apache.org/dyn/closer.lua/ranger/1.1.0/apache-ranger-1.1.0.tar.gz">apache-ranger-1.1.0.tar.gz</a>
+(<a href="https://www.apache.org/dist/ranger/1.1.0/apache-ranger-1.1.0.tar.gz.asc">PGP</a>)
+(<a href="https://www.apache.org/dist/ranger/1.1.0/apache-ranger-1.1.0.tar.gz.mds">Digests</a>)
+</p>
+</li>
+<li>
+<p>
+An older branch release is Apache Ranger 1.0.0:
</p>
<p>
<a href="https://www.apache.org/dyn/closer.lua/ranger/1.0.0/apache-ranger-1.0.0.tar.gz">apache-ranger-1.0.0.tar.gz</a>
[08/28] ranger git commit: RANGER-2173: Optimize Trie constuction and
Policy lookup
Posted by me...@apache.org.
RANGER-2173: Optimize Trie constuction and Policy lookup
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/35982827
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/35982827
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/35982827
Branch: refs/heads/ranger-1.1
Commit: 3598282745908ea1687693fb2359e71445972bf3
Parents: 1a35857
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Jul 31 16:30:47 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:33:43 2018 +0530
----------------------------------------------------------------------
.../ranger/plugin/util/RangerResourceTrie.java | 450 +++++++++++--------
agents-common/src/test/resources/log4j.xml | 4 +
2 files changed, 267 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/35982827/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index e7e8cf5..1723d14 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.util;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
@@ -31,7 +32,6 @@ import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
@@ -39,14 +39,16 @@ import java.util.Map;
public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class);
+ private static final Log PERF_TRIE_INIT_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.init");
+ private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.op");
private static final String DEFAULT_WILDCARD_CHARS = "*?";
- private final String resourceName;
- private final boolean optIgnoreCase;
- private final boolean optWildcard;
- private final String wildcardChars;
- private final TrieNode root;
+ private final String resourceName;
+ private final boolean optIgnoreCase;
+ private final boolean optWildcard;
+ private final String wildcardChars;
+ private final TrieNode<T> root;
private final Comparator<T> comparator;
public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<T> evaluators) {
@@ -58,6 +60,12 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ")");
}
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + ")");
+ }
+
Map<String, String> matcherOptions = resourceDef.getMatcherOptions();
boolean optReplaceTokens = RangerAbstractResourceMatcher.getOptionReplaceTokens(matcherOptions);
@@ -78,7 +86,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
this.optIgnoreCase = RangerAbstractResourceMatcher.getOptionIgnoreCase(matcherOptions);
this.optWildcard = RangerAbstractResourceMatcher.getOptionWildCard(matcherOptions);
this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS + tokenReplaceSpecialChars : "" + tokenReplaceSpecialChars;
- this.root = new TrieNode(Character.valueOf((char)0));
+ this.root = new TrieNode<>(null);
this.comparator = comparator;
for(T evaluator : evaluators) {
@@ -112,7 +120,15 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
root.postSetup(null, comparator);
- LOG.info(toString());
+ RangerPerfTracer.logAlways(perf);
+
+ if (PERF_TRIE_INIT_LOG.isTraceEnabled()) {
+ PERF_TRIE_INIT_LOG.trace(toString());
+
+ StringBuilder sb = new StringBuilder();
+ root.toString("", sb);
+ PERF_TRIE_INIT_LOG.trace("Trie Dump:\n{" + sb.toString() + "}");
+ }
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + "): " + toString());
@@ -140,7 +156,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
return null;
}
- public TrieData getTrieData() {
+ private TrieData getTrieData() {
TrieData ret = new TrieData();
root.populateTrieData(ret);
@@ -149,34 +165,33 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
return ret;
}
- public int getMaxDepth() {
+ private int getMaxDepth() {
return root.getMaxDepth();
}
- private final Character getLookupChar(char ch) {
- if(optIgnoreCase) {
- ch = Character.toLowerCase(ch);
- }
+ private Character getLookupChar(char ch) {
+ return optIgnoreCase ? Character.toLowerCase(ch) : ch;
+ }
- return Character.valueOf(ch);
+ private Character getLookupChar(String str, int index) {
+ return getLookupChar(str.charAt(index));
}
private void insert(String resource, boolean isRecursive, T evaluator) {
- TrieNode curr = root;
- boolean isWildcard = false;
- final int len = resource.length();
- for(int i = 0; i < len; i++) {
- Character ch = getLookupChar(resource.charAt(i));
+ RangerPerfTracer perf = null;
- if(optWildcard) {
- if (wildcardChars.indexOf(ch) != -1) {
- isWildcard = true;
- break;
- }
- }
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.insert(resource=" + resource + ")");
+ }
+
+ TrieNode<T> curr = root;
+
+ final String prefix = getNonWildcardPrefix(resource);
+ final boolean isWildcard = prefix.length() != resource.length();
- curr = curr.getOrCreateChild(ch);
+ if (StringUtils.isNotEmpty(prefix)) {
+ curr = curr.getOrCreateChild(prefix);
}
if(isWildcard || isRecursive) {
@@ -184,6 +199,20 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
} else {
curr.addEvaluator(evaluator);
}
+
+ RangerPerfTracer.logAlways(perf);
+ }
+
+ private String getNonWildcardPrefix(String str) {
+ if (!optWildcard) return str;
+ int minIndex = str.length();
+ for (int i = 0; i < wildcardChars.length(); i++) {
+ int index = str.indexOf(wildcardChars.charAt(i));
+ if (index != -1 && index < minIndex) {
+ minIndex = index;
+ }
+ }
+ return str.substring(0, minIndex);
}
private List<T> getEvaluatorsForResource(String resource) {
@@ -191,29 +220,38 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
LOG.debug("==> RangerResourceTrie.getEvaluatorsForResource(" + resource + ")");
}
- List<T> ret = null;
- TrieNode curr = root;
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerResourceTrie.getEvaluatorsForResource(resource=" + resource + ")");
+ }
- final int len = resource.length();
- for(int i = 0; i < len; i++) {
- Character ch = getLookupChar(resource.charAt(i));
- TrieNode child = curr.getChild(ch);
+ TrieNode<T> curr = root;
- if(child == null) {
- ret = curr.getWildcardEvaluators();
- curr = null; // so that curr.getEvaluators() will not be called below
+ final int len = resource.length();
+ int i = 0;
+
+ while (i < len) {
+ final TrieNode<T> child = curr.getChild(getLookupChar(resource, i));
+
+ if (child == null) {
break;
}
- curr = child;
- }
+ final String childStr = child.getStr();
- if(ret == null) {
- if(curr != null) {
- ret = curr.getEvaluators();
+ if (!resource.regionMatches(optIgnoreCase, i, childStr, 0, childStr.length())) {
+ break;
}
+
+ curr = child;
+ i += childStr.length();
}
+ List<T> ret = i == len ? curr.getEvaluators() : curr.getWildcardEvaluators();
+
+ RangerPerfTracer.logAlways(perf);
+
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerResourceTrie.getEvaluatorsForResource(" + resource + "): evaluatorCount=" + (ret == null ? 0 : ret.size()));
}
@@ -240,7 +278,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
if (ret == null) { // first resource: don't create map yet
ret = resourceEvaluators;
} else if (ret != resourceEvaluators) { // if evaluator list is same as earlier resources, retain the list, else create a map
- evaluatorsMap = new HashMap();
+ evaluatorsMap = new HashMap<>();
for (T evaluator : ret) {
evaluatorsMap.put(evaluator.getId(), evaluator);
@@ -261,7 +299,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
ret = new ArrayList<>(evaluatorsMap.values());
if (comparator != null) {
- Collections.sort(ret, comparator);
+ ret.sort(comparator);
}
}
@@ -294,7 +332,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
return sb.toString();
}
- public class TrieData {
+ class TrieData {
int nodeCount;
int leafNodeCount;
int singleChildNodeCount;
@@ -304,209 +342,247 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
int evaluatorListRefCount;
int wildcardEvaluatorListRefCount;
}
-}
-class TrieNode<T extends RangerPolicyResourceEvaluator> {
- private final Character c;
- private Map<Character, TrieNode> children;
- private List<T> evaluators;
- private List<T> wildcardEvaluators;
- private boolean isSharingParentWildcardEvaluators;
+ class TrieNode<U extends RangerPolicyResourceEvaluator> {
+ private String str;
+ private Map<Character, TrieNode<U>> children = new HashMap<>();
+ private List<U> evaluators;
+ private List<U> wildcardEvaluators;
+ private boolean isSharingParentWildcardEvaluators;
- TrieNode(Character c) {
- this.c = c;
- }
+ TrieNode(String str) {
+ this.str = str;
+ }
- Character getChar() {
- return c;
- }
+ String getStr() {
+ return str;
+ }
- Map<Character, TrieNode> getChildren() {
- return children;
- }
+ void setStr(String str) {
+ this.str = str;
+ }
- List<T> getEvaluators() {
- return evaluators;
- }
+ Map<Character, TrieNode<U>> getChildren() {
+ return children;
+ }
- List<T> getWildcardEvaluators() {
- return wildcardEvaluators;
- }
+ List<U> getEvaluators() {
+ return evaluators;
+ }
- TrieNode getChild(Character c) {
- TrieNode ret = children == null ? null : children.get(c);
+ List<U> getWildcardEvaluators() {
+ return wildcardEvaluators;
+ }
- return ret;
- }
+ TrieNode<U> getChild(Character ch) {
+ return children == null ? null : children.get(ch);
+ }
- void populateTrieData(RangerResourceTrie.TrieData trieData) {
- trieData.nodeCount++;
+ void populateTrieData(RangerResourceTrie.TrieData trieData) {
+ trieData.nodeCount++;
- if(wildcardEvaluators != null) {
- if(isSharingParentWildcardEvaluators) {
- trieData.wildcardEvaluatorListRefCount++;
- } else {
- trieData.wildcardEvaluatorListCount++;
+ if (wildcardEvaluators != null) {
+ if (isSharingParentWildcardEvaluators) {
+ trieData.wildcardEvaluatorListRefCount++;
+ } else {
+ trieData.wildcardEvaluatorListCount++;
+ }
}
- }
- if(evaluators != null) {
- if(evaluators == wildcardEvaluators) {
- trieData.evaluatorListRefCount++;
- } else {
- trieData.evaluatorListCount++;
+ if (evaluators != null) {
+ if (evaluators == wildcardEvaluators) {
+ trieData.evaluatorListRefCount++;
+ } else {
+ trieData.evaluatorListCount++;
+ }
}
- }
- if(children != null && !children.isEmpty()) {
- if(children.size() == 1) {
- trieData.singleChildNodeCount++;
- }
+ if (children != null && !children.isEmpty()) {
+ if (children.size() == 1) {
+ trieData.singleChildNodeCount++;
+ }
- for(Map.Entry<Character, TrieNode> entry : children.entrySet()) {
- TrieNode child = entry.getValue();
+ for (Map.Entry<Character, TrieNode<U>> entry : children.entrySet()) {
+ TrieNode child = entry.getValue();
- child.populateTrieData(trieData);
+ child.populateTrieData(trieData);
+ }
+ } else {
+ trieData.leafNodeCount++;
}
- } else {
- trieData.leafNodeCount++;
}
- }
- int getMaxDepth() {
- int ret = 0;
+ int getMaxDepth() {
+ int ret = 0;
- if(children != null) {
- for(Map.Entry<Character, TrieNode> entry : children.entrySet()) {
- TrieNode child = entry.getValue();
+ if (children != null) {
+ for (Map.Entry<Character, TrieNode<U>> entry : children.entrySet()) {
+ TrieNode<U> child = entry.getValue();
- int maxChildDepth = child.getMaxDepth();
+ int maxChildDepth = child.getMaxDepth();
- if(maxChildDepth > ret) {
- ret = maxChildDepth;
+ if (maxChildDepth > ret) {
+ ret = maxChildDepth;
+ }
}
}
- }
-
- return ret + 1;
- }
- TrieNode getOrCreateChild(Character c) {
- if(children == null) {
- children = new HashMap<>();
+ return ret + 1;
}
- TrieNode child = children.get(c);
+ TrieNode<U> getOrCreateChild(String str) {
+ int len = str.length();
- if(child == null) {
- child = new TrieNode(c);
- children.put(c, child);
- }
+ TrieNode<U> child = children.get(getLookupChar(str, 0));
- return child;
- }
+ if (child == null) {
+ child = new TrieNode<>(str);
+ addChild(child);
+ } else {
+ final String childStr = child.getStr();
+ final int childStrLen = childStr.length();
+
+ if (!StringUtils.equals(childStr, str)) {
+ final int numOfCharactersToMatch = childStrLen < len ? childStrLen : len;
+ int index = 1;
+ for (; index < numOfCharactersToMatch; index++) {
+ if (getLookupChar(childStr, index) != getLookupChar(str, index)) {
+ break;
+ }
+ }
+ if (index == numOfCharactersToMatch) {
+ // Matched all
+ if (childStrLen > len) {
+ // Existing node has longer string, need to break up this node
+ TrieNode<U> newChild = new TrieNode<>(str);
+ this.addChild(newChild);
+ child.setStr(childStr.substring(index));
+ newChild.addChild(child);
+ child = newChild;
+ } else {
+ // This is a longer string, build a child with leftover string
+ child = child.getOrCreateChild(str.substring(index));
+ }
+ } else {
+ // Partial match for both; both have leftovers
+ String matchedPart = str.substring(0, index);
+ TrieNode<U> newChild = new TrieNode<>(matchedPart);
+ this.addChild(newChild);
+ child.setStr(childStr.substring(index));
+ newChild.addChild(child);
+ child = newChild.getOrCreateChild(str.substring(index));
+ }
+ }
+ }
- void addEvaluator(T evaluator) {
- if(evaluators == null) {
- evaluators = new ArrayList<>();
+ return child;
}
- if(!evaluators.contains(evaluator)) {
- evaluators.add(evaluator);
+ private void addChild(TrieNode<U> child) {
+ children.put(getLookupChar(child.getStr(), 0), child);
}
- }
- void addWildcardEvaluator(T evaluator) {
- if(wildcardEvaluators == null) {
- wildcardEvaluators = new ArrayList<>();
- }
+ void addEvaluator(U evaluator) {
+ if (evaluators == null) {
+ evaluators = new ArrayList<>();
+ }
- if(!wildcardEvaluators.contains(evaluator)) {
- wildcardEvaluators.add(evaluator);
+ if (!evaluators.contains(evaluator)) {
+ evaluators.add(evaluator);
+ }
}
- }
- void postSetup(List<T> parentWildcardEvaluators, Comparator<T> comparator) {
- // finalize wildcard-evaluators list by including parent's wildcard evaluators
- if(parentWildcardEvaluators != null) {
- if(CollectionUtils.isEmpty(this.wildcardEvaluators)) {
- this.wildcardEvaluators = parentWildcardEvaluators;
- } else {
- for (T evaluator : parentWildcardEvaluators) {
- addWildcardEvaluator(evaluator);
- }
+ void addWildcardEvaluator(U evaluator) {
+ if (wildcardEvaluators == null) {
+ wildcardEvaluators = new ArrayList<>();
+ }
+
+ if (!wildcardEvaluators.contains(evaluator)) {
+ wildcardEvaluators.add(evaluator);
}
}
- this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators;
- // finalize evaluators list by including wildcard evaluators
- if(wildcardEvaluators != null) {
- if(CollectionUtils.isEmpty(this.evaluators)) {
- this.evaluators = wildcardEvaluators;
- } else {
- for (T evaluator : wildcardEvaluators) {
- addEvaluator(evaluator);
+ void postSetup(List<U> parentWildcardEvaluators, Comparator<U> comparator) {
+ // finalize wildcard-evaluators list by including parent's wildcard evaluators
+ if (parentWildcardEvaluators != null) {
+ if (CollectionUtils.isEmpty(this.wildcardEvaluators)) {
+ this.wildcardEvaluators = parentWildcardEvaluators;
+ } else {
+ for (U evaluator : parentWildcardEvaluators) {
+ addWildcardEvaluator(evaluator);
+ }
}
}
- }
+ this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators;
- if (comparator != null) {
- if (!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) {
- Collections.sort(wildcardEvaluators, comparator);
+ // finalize evaluators list by including wildcard evaluators
+ if (wildcardEvaluators != null) {
+ if (CollectionUtils.isEmpty(this.evaluators)) {
+ this.evaluators = wildcardEvaluators;
+ } else {
+ for (U evaluator : wildcardEvaluators) {
+ addEvaluator(evaluator);
+ }
+ }
}
- if (evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) {
- Collections.sort(evaluators, comparator);
+ if (comparator != null) {
+ if (!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) {
+ wildcardEvaluators.sort(comparator);
+ }
+
+ if (evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) {
+ evaluators.sort(comparator);
+ }
}
- }
- if(children != null) {
- for(Map.Entry<Character, TrieNode> entry : children.entrySet()) {
- TrieNode child = entry.getValue();
+ if (children != null) {
+ for (Map.Entry<Character, TrieNode<U>> entry : children.entrySet()) {
+ TrieNode<U> child = entry.getValue();
- child.postSetup(wildcardEvaluators, comparator);
+ child.postSetup(wildcardEvaluators, comparator);
+ }
}
}
- }
- public void toString(String prefix, StringBuilder sb) {
- String nodeValue = prefix;
+ public void toString(String prefix, StringBuilder sb) {
+ String nodeValue = prefix;
- if(c != 0) {
- nodeValue += c;
- }
+ if (str != null) {
+ nodeValue += str;
+ }
- sb.append("nodeValue=").append(nodeValue);
- sb.append("; childCount=").append(children == null ? 0 : children.size());
- sb.append("; evaluators=[ ");
- if(evaluators != null) {
- for(T evaluator : evaluators) {
- sb.append(evaluator.getId()).append(" ");
+ sb.append("nodeValue=").append(nodeValue);
+ sb.append("; childCount=").append(children == null ? 0 : children.size());
+ sb.append("; evaluators=[ ");
+ if (evaluators != null) {
+ for (U evaluator : evaluators) {
+ sb.append(evaluator.getId()).append(" ");
+ }
}
- }
- sb.append("]");
+ sb.append("]");
- sb.append("; wildcardEvaluators=[ ");
- if(wildcardEvaluators != null) {
- for(T evaluator : wildcardEvaluators) {
- sb.append(evaluator.getId()).append(" ");
+ sb.append("; wildcardEvaluators=[ ");
+ if (wildcardEvaluators != null) {
+ for (U evaluator : wildcardEvaluators) {
+ sb.append(evaluator.getId()).append(" ");
+ }
}
- }
- sb.append("]");
- sb.append(Character.LINE_SEPARATOR);
+ sb.append("]\n");
- if(children != null) {
- for(Map.Entry<Character, TrieNode> entry : children.entrySet()) {
- TrieNode child = entry.getValue();
+ if (children != null) {
+ for (Map.Entry<Character, TrieNode<U>> entry : children.entrySet()) {
+ TrieNode<U> child = entry.getValue();
- child.toString(nodeValue, sb);
+ child.toString(nodeValue, sb);
+ }
}
}
- }
- public void clear() {
- children = null;
- evaluators = null;
- wildcardEvaluators = null;
+ public void clear() {
+ children = null;
+ evaluators = null;
+ wildcardEvaluators = null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/35982827/agents-common/src/test/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/agents-common/src/test/resources/log4j.xml b/agents-common/src/test/resources/log4j.xml
index d1a6f1c..714d463 100644
--- a/agents-common/src/test/resources/log4j.xml
+++ b/agents-common/src/test/resources/log4j.xml
@@ -35,6 +35,10 @@
</layout>
</appender>
<!--
+ <logger name="org.apache.ranger.perf.resourcetrie" additivity="false">
+ <level value="debug" />
+ <appender-ref ref="ranger_perf_appender" />
+ </logger>
<logger name="org.apache.ranger.perf.policyengine.getResourceACLs" additivity="false">
<level value="debug" />
<appender-ref ref="ranger_perf_appender" />
[21/28] ranger git commit: RANGER-2167 - Upgrade to Apache parent pom
version 20
Posted by me...@apache.org.
RANGER-2167 - Upgrade to Apache parent pom version 20
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/b66e98dc
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/b66e98dc
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/b66e98dc
Branch: refs/heads/ranger-1.1
Commit: b66e98dc94c9a298d68cad13d120406e1c0802d0
Parents: 6dcf1a9
Author: Colm O hEigeartaigh <co...@apache.org>
Authored: Wed Aug 15 17:29:21 2018 +0100
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:43:44 2018 +0530
----------------------------------------------------------------------
.../plugin/model/RangerValiditySchedule.java | 4 -
.../validation/RangerServiceValidator.java | 18 ++--
.../model/validation/RangerValidator.java | 1 -
.../RangerValidityScheduleValidator.java | 1 -
.../plugin/policyengine/RangerResourceACLs.java | 10 +-
.../RangerDefaultPolicyEvaluator.java | 9 +-
.../RangerValidityScheduleEvaluator.java | 1 -
.../model/validation/TestRangerValidator.java | 3 -
dev-support/ranger-pmd-ruleset.xml | 4 +-
.../hadoop/RangerHdfsAuthorizer.java | 12 +--
.../hadoop/crypto/key/RangerKeyStore.java | 26 ++---
.../atlas/authorizer/RangerAtlasAuthorizer.java | 3 +-
.../KafkaRangerAuthorizerGSSTest.java | 3 +-
.../kms/authorizer/RangerKmsAuthorizer.java | 4 +-
.../ranger/services/kms/client/KMSClient.java | 7 +-
.../services/nifi/client/NiFiConnectionMgr.java | 1 -
pom.xml | 2 +-
.../RangerPolicyEnginePerformanceTest.java | 1 -
.../org/apache/ranger/rest/ServiceREST.java | 5 +-
.../filter/RangerKRBAuthenticationFilter.java | 6 +-
.../security/web/filter/RangerKrbFilter.java | 2 -
.../ranger/service/RangerPolicyService.java | 106 +++++++++----------
.../ranger/service/RangerServiceService.java | 28 +++--
.../apache/ranger/biz/TestServiceDBStore.java | 13 ---
24 files changed, 118 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerValiditySchedule.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerValiditySchedule.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerValiditySchedule.java
index 55f787d..d2271ae 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerValiditySchedule.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerValiditySchedule.java
@@ -24,8 +24,6 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
@@ -44,8 +42,6 @@ import java.util.List;
public class RangerValiditySchedule implements Serializable {
- private static final Log LOG = LogFactory.getLog(RangerValiditySchedule.class);
-
public static final String VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION = "yyyy/MM/dd HH:mm:ss";
private String startTime;
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java
index b64de32..9b5d8d5 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceValidator.java
@@ -37,12 +37,12 @@ import com.google.common.collect.Sets;
public class RangerServiceValidator extends RangerValidator {
private static final Log LOG = LogFactory.getLog(RangerServiceValidator.class);
static final public String VALIDATION_SERVICE_NAME = "^[a-zA-Z0-9_-][a-zA-Z0-9\\s_-]{0,254}";
-
- static Pattern serviceNameCompiledRegEx;
+
+ static Pattern serviceNameCompiledRegEx;
public RangerServiceValidator(ServiceStore store) {
super(store);
}
-
+
public void validate(RangerService service, Action action) throws Exception {
if(LOG.isDebugEnabled()) {
LOG.debug(String.format("==> RangerServiceValidator.validate(%s, %s)", service, action));
@@ -62,7 +62,7 @@ public class RangerServiceValidator extends RangerValidator {
}
}
}
-
+
boolean isValid(Long id, Action action, List<ValidationFailureDetails> failures) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerServiceValidator.isValid(" + id + ")");
@@ -97,7 +97,7 @@ public class RangerServiceValidator extends RangerValidator {
}
return valid;
}
-
+
boolean isValid(RangerService service, Action action, List<ValidationFailureDetails> failures) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerServiceValidator.isValid(" + service + ")");
@@ -105,7 +105,7 @@ public class RangerServiceValidator extends RangerValidator {
if (!(action == Action.CREATE || action == Action.UPDATE)) {
throw new IllegalArgumentException("isValid(RangerService, ...) is only supported for CREATE/UPDATE");
}
-
+
boolean valid = true;
if (service == null) {
ValidationErrorCode error = ValidationErrorCode.SERVICE_VALIDATION_ERR_NULL_SERVICE_OBJECT;
@@ -272,15 +272,15 @@ public class RangerServiceValidator extends RangerValidator {
}
return valid;
}
-
+
public boolean regExPatternMatch(String expression, String inputStr) {
Pattern pattern = serviceNameCompiledRegEx;
if (pattern == null) {
pattern = Pattern.compile(expression, Pattern.CASE_INSENSITIVE);
serviceNameCompiledRegEx = pattern;
}
-
- return pattern != null ? pattern.matcher(inputStr).matches() : false;
+
+ return pattern != null && pattern.matcher(inputStr).matches();
}
public boolean validateString(String regExStr, String str) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java
index ed5aa8d..c7062dd 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidator.java
@@ -45,7 +45,6 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef;
import org.apache.ranger.plugin.store.ServiceStore;
import org.apache.ranger.plugin.util.RangerObjectFactory;
-import org.apache.ranger.plugin.util.SearchFilter;
public abstract class RangerValidator {
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidityScheduleValidator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidityScheduleValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidityScheduleValidator.java
index f34d124..3bfdf93 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidityScheduleValidator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerValidityScheduleValidator.java
@@ -24,7 +24,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import java.io.Serializable;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceACLs.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceACLs.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceACLs.java
index 34098fa..adee37e 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceACLs.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceACLs.java
@@ -20,7 +20,6 @@
package org.apache.ranger.plugin.policyengine;
import org.apache.commons.lang.StringUtils;
-import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -33,6 +32,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator.ACCESS_ALLOWED;
import static org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator.ACCESS_DENIED;
public class RangerResourceACLs {
@@ -59,7 +59,7 @@ public class RangerResourceACLs {
AccessResult accessResult = entry.getValue();
int access = accessResult.getResult();
- if (access == RangerPolicyEvaluator.ACCESS_DENIED || access == RangerPolicyEvaluator.ACCESS_ALLOWED) {
+ if (access == ACCESS_DENIED || access == ACCESS_ALLOWED) {
for (Map.Entry<String, Map<String, AccessResult>> mapEntry : userACLs.entrySet()) {
Map<String, AccessResult> mapValue = mapEntry.getValue();
AccessResult savedAccessResult = mapValue.get(accessType);
@@ -223,10 +223,12 @@ public class RangerResourceACLs {
}
@Override
public String toString() {
- if (result == RangerPolicyEvaluator.ACCESS_ALLOWED)
+ if (result == ACCESS_ALLOWED) {
return "ALLOWED, final=" + isFinal;
- if (result == RangerPolicyEvaluator.ACCESS_DENIED)
+ }
+ if (result == ACCESS_DENIED) {
return "NOT_ALLOWED, final=" + isFinal;
+ }
return "CONDITIONAL_ALLOWED, final=" + isFinal;
}
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
index 05a9f40..2a5ee54 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
@@ -651,7 +651,6 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
int allowedAccessCount = 0;
int deniedAccessCount = 0;
- int deniedWithException = 0;
int undeterminedAccessCount = 0;
int accessesSize = 0;
@@ -665,12 +664,8 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
allowedAccessCount++;
} else if (accessResult.getResult() == RangerPolicyEvaluator.ACCESS_DENIED) {
deniedAccessCount++;
- } else if (accessResult.getResult() == RangerPolicyEvaluator.ACCESS_UNDETERMINED) {
- if (accessResult.getHasSeenDeny()) {
- deniedWithException++;
- } else {
- undeterminedAccessCount++;
- }
+ } else if (accessResult.getResult() == RangerPolicyEvaluator.ACCESS_UNDETERMINED && !accessResult.getHasSeenDeny()) {
+ undeterminedAccessCount++;
}
accessesSize++;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
index b48ff3b..6715e2b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
@@ -32,7 +32,6 @@ import org.apache.ranger.plugin.resourcematcher.ScheduledTimeRangeMatcher;
import org.apache.ranger.plugin.util.RangerPerfTracer;
import javax.annotation.Nonnull;
-import java.io.Serializable;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java
index f9b3428..5bdffda 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/model/validation/TestRangerValidator.java
@@ -21,7 +21,6 @@ package org.apache.ranger.plugin.model.validation;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
@@ -43,7 +42,6 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef;
import org.apache.ranger.plugin.model.validation.RangerValidator.Action;
import org.apache.ranger.plugin.store.ServiceStore;
-import org.apache.ranger.plugin.util.SearchFilter;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -59,7 +57,6 @@ public class TestRangerValidator {
}
boolean isValid(String behavior) {
- boolean valid;
return "valid".equals(behavior);
}
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/dev-support/ranger-pmd-ruleset.xml
----------------------------------------------------------------------
diff --git a/dev-support/ranger-pmd-ruleset.xml b/dev-support/ranger-pmd-ruleset.xml
index f03cda3..65a4f1d 100644
--- a/dev-support/ranger-pmd-ruleset.xml
+++ b/dev-support/ranger-pmd-ruleset.xml
@@ -31,7 +31,9 @@
<exclude name="OverrideBothEqualsAndHashcode" />
</rule>
<rule ref="rulesets/java/unusedcode.xml" />
- <rule ref="rulesets/java/imports.xml" />
+ <rule ref="rulesets/java/imports.xml">
+ <exclude name="TooManyStaticImports" />
+ </rule>
<rule ref="rulesets/java/braces.xml">
<exclude name="ForLoopsMustUseBraces" />
<exclude name="IfElseStmtsMustUseBraces" />
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
----------------------------------------------------------------------
diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
index cccc759..65a397d 100644
--- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
+++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
@@ -495,7 +495,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
}
if (RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH_ALT.equals(path)) {
- path = RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH;
+ path = HDFS_ROOT_FOLDER_PATH;
}
if (LOG.isDebugEnabled()) {
@@ -612,7 +612,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
}
if (RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH_ALT.equals(path)) {
- path = RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH;
+ path = HDFS_ROOT_FOLDER_PATH;
}
if(LOG.isDebugEnabled()) {
@@ -666,7 +666,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
}
if (RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH_ALT.equals(path)) {
- path = RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH;
+ path = HDFS_ROOT_FOLDER_PATH;
}
if (LOG.isDebugEnabled()) {
@@ -731,10 +731,10 @@ class RangerHdfsPlugin extends RangerBasePlugin {
public RangerHdfsPlugin() {
super("hdfs", "hdfs");
}
-
+
public void init() {
super.init();
-
+
RangerHdfsPlugin.hadoopAuthEnabled = RangerConfiguration.getInstance().getBoolean(RangerHadoopConstants.RANGER_ADD_HDFS_PERMISSION_PROP, RangerHadoopConstants.RANGER_ADD_HDFS_PERMISSION_DEFAULT);
RangerHdfsPlugin.fileNameExtensionSeparator = RangerConfiguration.getInstance().get(RangerHdfsAuthorizer.RANGER_FILENAME_EXTENSION_SEPARATOR_PROP, RangerHdfsAuthorizer.DEFAULT_FILENAME_EXTENSION_SEPARATOR);
RangerHdfsPlugin.optimizeSubAccessAuthEnabled = RangerConfiguration.getInstance().getBoolean(RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP, RangerHadoopConstants.RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT);
@@ -801,7 +801,7 @@ class RangerHdfsAccessRequest extends RangerAccessRequestImpl {
buildRequestContext(inode);
}
}
-
+
private static String getRemoteIp() {
String ret = null;
InetAddress ip = Server.getRemoteIp();
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
index cd5a8c2..86f1a29 100644
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
@@ -45,10 +45,13 @@ import java.security.SecureRandom;
import java.security.UnrecoverableKeyException;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
+import java.util.Collections;
import java.util.Date;
import java.util.Enumeration;
-import java.util.Hashtable;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -97,8 +100,8 @@ public class RangerKeyStore extends KeyStoreSpi {
int version;
}
- private Hashtable<String, Object> keyEntries = new Hashtable<String, Object>();
- private Hashtable<String, Object> deltaEntries = new Hashtable<String, Object>();
+ private Map<String, Object> keyEntries = new ConcurrentHashMap<>();
+ private Map<String, Object> deltaEntries = new ConcurrentHashMap<>();
RangerKeyStore() {
}
@@ -265,7 +268,7 @@ public class RangerKeyStore extends KeyStoreSpi {
@Override
public Enumeration<String> engineAliases() {
- return keyEntries.keys();
+ return Collections.enumeration(keyEntries.keySet());
}
@Override
@@ -293,24 +296,23 @@ public class RangerKeyStore extends KeyStoreSpi {
MessageDigest md = getKeyedMessageDigest(password);
byte digest[] = md.digest();
- for (Enumeration<String> e = deltaEntries.keys(); e.hasMoreElements(); ) {
+ for (Entry<String, Object> entry : deltaEntries.entrySet()) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(new DigestOutputStream(baos, md));
ObjectOutputStream oos = null;
try {
- String alias = e.nextElement();
- Object entry = deltaEntries.get(alias);
-
oos = new ObjectOutputStream(dos);
- oos.writeObject(((SecretKeyEntry) entry).sealedKey);
+ oos.writeObject(((SecretKeyEntry) entry.getValue()).sealedKey);
dos.write(digest);
dos.flush();
- Long creationDate = ((SecretKeyEntry) entry).date.getTime();
- SecretKeyEntry secretKey = (SecretKeyEntry) entry;
- XXRangerKeyStore xxRangerKeyStore = mapObjectToEntity(alias, creationDate, baos.toByteArray(), secretKey.cipher_field, secretKey.bit_length, secretKey.description, secretKey.version, secretKey.attributes);
+ Long creationDate = ((SecretKeyEntry) entry.getValue()).date.getTime();
+ SecretKeyEntry secretKey = (SecretKeyEntry) entry.getValue();
+ XXRangerKeyStore xxRangerKeyStore = mapObjectToEntity(entry.getKey(), creationDate, baos.toByteArray(),
+ secretKey.cipher_field, secretKey.bit_length, secretKey.description,
+ secretKey.version, secretKey.attributes);
dbOperationStore(xxRangerKeyStore);
} finally {
if (oos != null) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java
----------------------------------------------------------------------
diff --git a/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java b/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java
index c02e9e6..bf588e2 100644
--- a/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java
+++ b/plugin-atlas/src/main/java/org/apache/ranger/authorization/atlas/authorizer/RangerAtlasAuthorizer.java
@@ -44,7 +44,6 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
import org.apache.ranger.plugin.policyengine.RangerAccessResult;
import org.apache.ranger.plugin.service.RangerBasePlugin;
import org.apache.ranger.plugin.util.RangerPerfTracer;
-import org.apache.ranger.services.atlas.RangerServiceAtlas;
import java.util.*;
@@ -373,7 +372,7 @@ public class RangerAtlasAuthorizer implements AtlasAuthorizer {
}
}
} else {
- rangerResource.setValue(RESOURCE_ENTITY_CLASSIFICATION, RangerServiceAtlas.ENTITY_NOT_CLASSIFIED);
+ rangerResource.setValue(RESOURCE_ENTITY_CLASSIFICATION, ENTITY_NOT_CLASSIFIED);
ret = checkAccess(rangerRequest, auditHandler);
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
----------------------------------------------------------------------
diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
index 2624478..c1386fe 100644
--- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
+++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/KafkaRangerAuthorizerGSSTest.java
@@ -333,8 +333,7 @@ public class KafkaRangerAuthorizerGSSTest {
final Producer<String, String> producer = new KafkaProducer<>(producerProps);
// Send a message
- Future<RecordMetadata> record =
- producer.send(new ProducerRecord<String, String>("test", "somekey", "somevalue"));
+ producer.send(new ProducerRecord<String, String>("test", "somekey", "somevalue"));
producer.flush();
producer.close();
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java
----------------------------------------------------------------------
diff --git a/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java b/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java
index c3d75a1..07921a9 100755
--- a/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java
+++ b/plugin-kms/src/main/java/org/apache/ranger/authorization/kms/authorizer/RangerKmsAuthorizer.java
@@ -222,7 +222,7 @@ public class RangerKmsAuthorizer implements Runnable, KeyACLs {
if(plugin != null && ret) {
RangerKMSAccessRequest request = new RangerKMSAccessRequest("", rangerAccessType, ugi, clientIp, clusterName);
RangerAccessResult result = plugin.isAccessAllowed(request);
- ret = result == null ? false : result.getIsAllowed();
+ ret = result != null && result.getIsAllowed();
}
RangerPerfTracer.log(perf);
if(LOG.isDebugEnabled()) {
@@ -249,7 +249,7 @@ public class RangerKmsAuthorizer implements Runnable, KeyACLs {
if(plugin != null && ret) {
RangerKMSAccessRequest request = new RangerKMSAccessRequest(keyName, rangerAccessType, ugi, clientIp, clusterName);
RangerAccessResult result = plugin.isAccessAllowed(request);
- ret = result == null ? false : result.getIsAllowed();
+ ret = result != null && result.getIsAllowed();
}
if(LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSClient.java
----------------------------------------------------------------------
diff --git a/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSClient.java b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSClient.java
index af0ac71..4a958d4 100755
--- a/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSClient.java
+++ b/plugin-kms/src/main/java/org/apache/ranger/services/kms/client/KMSClient.java
@@ -41,7 +41,6 @@ import org.apache.log4j.Logger;
import org.apache.ranger.plugin.client.BaseClient;
import org.apache.ranger.plugin.util.PasswordUtils;
import org.apache.ranger.plugin.client.HadoopException;
-import org.apache.ranger.services.kms.client.KMSClient;
import com.google.common.base.Strings;
import com.google.gson.Gson;
@@ -113,7 +112,7 @@ public class KMSClient {
}
hostsPart = t[0];
}
- return createProvider(providerUri, origUrl, port, hostsPart);
+ return createProvider(origUrl, port, hostsPart);
}
private static Path extractKMSPath(URI uri) throws MalformedURLException,
@@ -121,7 +120,7 @@ public class KMSClient {
return ProviderUtils.unnestUri(uri);
}
- private String[] createProvider(URI providerUri, URL origUrl, int port,
+ private String[] createProvider(URL origUrl, int port,
String hostsPart) throws IOException {
String[] hosts = hostsPart.split(";");
String[] providers = new String[hosts.length];
@@ -305,7 +304,7 @@ public class KMSClient {
return lret;
}
- public static Map<String, Object> testConnection(String serviceName,
+ public static Map<String, Object> testConnection(String serviceName, //NOPMD
Map<String, String> configs) {
List<String> strList = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/plugin-nifi/src/main/java/org/apache/ranger/services/nifi/client/NiFiConnectionMgr.java
----------------------------------------------------------------------
diff --git a/plugin-nifi/src/main/java/org/apache/ranger/services/nifi/client/NiFiConnectionMgr.java b/plugin-nifi/src/main/java/org/apache/ranger/services/nifi/client/NiFiConnectionMgr.java
index 739bef6..19cb1a9 100644
--- a/plugin-nifi/src/main/java/org/apache/ranger/services/nifi/client/NiFiConnectionMgr.java
+++ b/plugin-nifi/src/main/java/org/apache/ranger/services/nifi/client/NiFiConnectionMgr.java
@@ -30,7 +30,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
-import java.net.URL;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 472199d..4d7eddc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
- <version>19</version>
+ <version>20</version>
</parent>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger</artifactId>
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java b/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java
index 11af0a8..7a39396 100644
--- a/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java
+++ b/ranger-tools/src/test/java/org/apache/ranger/policyengine/RangerPolicyEnginePerformanceTest.java
@@ -34,7 +34,6 @@ import java.util.concurrent.CountDownLatch;
import org.apache.commons.lang.text.StrSubstitutor;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
-import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl;
import org.apache.ranger.plugin.util.PerfDataRecorder;
import org.apache.ranger.plugin.util.PerfDataRecorder.PerfStatistic;
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
index 5f8a05a..59741d3 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
@@ -122,7 +122,6 @@ import org.apache.ranger.view.RangerPluginInfoList;
import org.apache.ranger.view.RangerPolicyList;
import org.apache.ranger.view.RangerServiceDefList;
import org.apache.ranger.view.RangerServiceList;
-import org.apache.ranger.view.VXPolicyLabelList;
import org.apache.ranger.view.VXResponse;
import org.apache.ranger.view.VXString;
import org.apache.ranger.view.VXUser;
@@ -2130,7 +2129,7 @@ public class ServiceREST {
LOG.debug("Deleting Policy from provided services in servicesMapJson file for specific resource...");
}
if (CollectionUtils.isNotEmpty(sourceServices) && CollectionUtils.isNotEmpty(destinationServices)){
- deletePoliciesForResource(sourceServices, destinationServices, polResource, request, policies);
+ deletePoliciesForResource(sourceServices, destinationServices, request, policies);
}
}
if (policies != null && !CollectionUtils.sizeIsEmpty(policies)){
@@ -2373,7 +2372,7 @@ public class ServiceREST {
}
}
- private void deletePoliciesForResource(List<String> sourceServices, List<String> destinationServices, String resource, HttpServletRequest request, List<RangerPolicy> exportPolicies) {
+ private void deletePoliciesForResource(List<String> sourceServices, List<String> destinationServices, HttpServletRequest request, List<RangerPolicy> exportPolicies) {
int totalDeletedPilicies = 0;
if (CollectionUtils.isNotEmpty(sourceServices)
&& CollectionUtils.isNotEmpty(destinationServices)) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java
index b4a3f93..d20a203 100644
--- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java
+++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKRBAuthenticationFilter.java
@@ -556,19 +556,19 @@ public class RangerKRBAuthenticationFilter extends RangerKrbFilter {
}
@Override
- public javax.servlet.ServletRegistration.Dynamic addServlet(
+ public ServletRegistration.Dynamic addServlet(
String servletName, Class<? extends Servlet> servletClass) {
return null;
}
@Override
- public javax.servlet.ServletRegistration.Dynamic addServlet(
+ public ServletRegistration.Dynamic addServlet(
String servletName, Servlet servlet) {
return null;
}
@Override
- public javax.servlet.ServletRegistration.Dynamic addServlet(
+ public ServletRegistration.Dynamic addServlet(
String servletName, String className) {
return null;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKrbFilter.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKrbFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKrbFilter.java
index ca0d17e..0be0e68 100644
--- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKrbFilter.java
+++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerKrbFilter.java
@@ -44,8 +44,6 @@ import java.io.IOException;
import java.security.Principal;
import java.text.SimpleDateFormat;
import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
@InterfaceAudience.Private
@InterfaceStability.Unstable
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java
index a3ff825..ad3984d 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java
@@ -157,7 +157,7 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
if (!trxLogAttrs.containsKey(field.getName())) {
continue;
}
- XXTrxLog xTrxLog = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, oldPolicy, action);
+ XXTrxLog xTrxLog = processFieldToCreateTrxLog(field, objectName, vObj, mObj, oldPolicy, action);
if (xTrxLog != null) {
trxLogList.add(xTrxLog);
}
@@ -167,7 +167,7 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
.getDeclaredFields();
for (Field field : superClassFields) {
if ("isEnabled".equalsIgnoreCase(field.getName())) {
- XXTrxLog xTrx = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, oldPolicy, action);
+ XXTrxLog xTrx = processFieldToCreateTrxLog(field, objectName, vObj, mObj, oldPolicy, action);
if (xTrx != null) {
trxLogList.add(xTrx);
}
@@ -184,7 +184,7 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
}
private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName,
- Field nameField, RangerPolicy vObj, XXPolicy mObj, RangerPolicy oldPolicy, int action) {
+ RangerPolicy vObj, XXPolicy mObj, RangerPolicy oldPolicy, int action) {
String actionString = "";
@@ -199,54 +199,54 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
String value = null;
boolean isEnum = vTrxLogAttr.isEnum();
- if (isEnum) {
-
- } else if (POLICY_RESOURCE_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processPolicyResourcesForTrxLog(field.get(vObj));
- } else if (POLICY_ITEM_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processPolicyItemsForTrxLog(field.get(vObj));
- } else if (DENYPOLICY_ITEM_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processPolicyItemsForTrxLog(field.get(vObj));
- } else if (POLICY_NAME_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processPolicyNameForTrxLog(field.get(vObj));
- } else if (ALLOW_EXCEPTIONS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processPolicyItemsForTrxLog(field.get(vObj));
- } else if (DENY_EXCEPTIONS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processPolicyItemsForTrxLog(field.get(vObj));
- } else if (DATAMASK_POLICY_ITEM_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processDataMaskPolicyItemsForTrxLog(field.get(vObj));
- if(vObj.getDataMaskPolicyItems() != null && CollectionUtils.isNotEmpty(vObj.getDataMaskPolicyItems())) {
- for(RangerDataMaskPolicyItem policyItem : vObj.getDataMaskPolicyItems()) {
- if(policyItem.getDataMaskInfo() != null && policyItem.getDataMaskInfo().getDataMaskType() != null) {
- List<XXDataMaskTypeDef> xDataMaskDef = daoMgr.getXXDataMaskTypeDef().getAll();
- if(CollectionUtils.isNotEmpty(xDataMaskDef) && xDataMaskDef != null ) {
- for (XXDataMaskTypeDef xxDataMaskTypeDef : xDataMaskDef) {
- if(xxDataMaskTypeDef.getName().equalsIgnoreCase(policyItem.getDataMaskInfo().getDataMaskType())) {
- String label = xxDataMaskTypeDef.getLabel();
- StringBuilder sbValue = new StringBuilder(value);
- label = ",\"DataMasklabel\":\""+label+"\"";
- int sbValueIndex = sbValue.lastIndexOf("}]");
- sbValue.insert(sbValueIndex, label);
- value = sbValue.toString();
- break;
- }
- }
- }
- }
- }
- }
- } else if (ROWFILTER_POLICY_ITEM_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processRowFilterPolicyItemForTrxLog(field.get(vObj));
- } else if (IS_ENABLED_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = String.valueOf(processIsEnabledClassFieldNameForTrxLog(field.get(vObj)));
- } else if (POLICY_LABELS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processPolicyLabelsClassFieldNameForTrxLog(field.get(vObj));
- } else if (POLICY_VALIDITYSCHEDULES_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processValiditySchedulesClassFieldNameForTrxLog(field.get(vObj));
- } else if (POLICY_PRIORITY_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processPriorityClassFieldNameForTrxLog(field.get(vObj));
- } else {
- value = "" + field.get(vObj);
+ if (!isEnum) {
+ if (POLICY_RESOURCE_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processPolicyResourcesForTrxLog(field.get(vObj));
+ } else if (POLICY_ITEM_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processPolicyItemsForTrxLog(field.get(vObj));
+ } else if (DENYPOLICY_ITEM_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processPolicyItemsForTrxLog(field.get(vObj));
+ } else if (POLICY_NAME_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processPolicyNameForTrxLog(field.get(vObj));
+ } else if (ALLOW_EXCEPTIONS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processPolicyItemsForTrxLog(field.get(vObj));
+ } else if (DENY_EXCEPTIONS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processPolicyItemsForTrxLog(field.get(vObj));
+ } else if (DATAMASK_POLICY_ITEM_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processDataMaskPolicyItemsForTrxLog(field.get(vObj));
+ if(vObj.getDataMaskPolicyItems() != null && CollectionUtils.isNotEmpty(vObj.getDataMaskPolicyItems())) {
+ for(RangerDataMaskPolicyItem policyItem : vObj.getDataMaskPolicyItems()) {
+ if(policyItem.getDataMaskInfo() != null && policyItem.getDataMaskInfo().getDataMaskType() != null) {
+ List<XXDataMaskTypeDef> xDataMaskDef = daoMgr.getXXDataMaskTypeDef().getAll();
+ if(CollectionUtils.isNotEmpty(xDataMaskDef) && xDataMaskDef != null ) {
+ for (XXDataMaskTypeDef xxDataMaskTypeDef : xDataMaskDef) {
+ if(xxDataMaskTypeDef.getName().equalsIgnoreCase(policyItem.getDataMaskInfo().getDataMaskType())) {
+ String label = xxDataMaskTypeDef.getLabel();
+ StringBuilder sbValue = new StringBuilder(value);
+ label = ",\"DataMasklabel\":\""+label+"\"";
+ int sbValueIndex = sbValue.lastIndexOf("}]");
+ sbValue.insert(sbValueIndex, label);
+ value = sbValue.toString();
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else if (ROWFILTER_POLICY_ITEM_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processRowFilterPolicyItemForTrxLog(field.get(vObj));
+ } else if (IS_ENABLED_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = String.valueOf(processIsEnabledClassFieldNameForTrxLog(field.get(vObj)));
+ } else if (POLICY_LABELS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processPolicyLabelsClassFieldNameForTrxLog(field.get(vObj));
+ } else if (POLICY_VALIDITYSCHEDULES_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processValiditySchedulesClassFieldNameForTrxLog(field.get(vObj));
+ } else if (POLICY_PRIORITY_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processPriorityClassFieldNameForTrxLog(field.get(vObj));
+ } else {
+ value = "" + field.get(vObj);
+ }
}
if (action == OPERATION_CREATE_CONTEXT) {
@@ -266,9 +266,7 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
mField.setAccessible(true);
String mFieldName = mField.getName();
if (fieldName.equalsIgnoreCase(mFieldName)) {
- if (isEnum) {
-
- } else {
+ if (!isEnum) {
oldValue = mField.get(mObj) + "";
}
break;
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java
index df3fdb5..2ddb5f3 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceService.java
@@ -151,7 +151,7 @@ public class RangerServiceService extends RangerServiceServiceBase<XXService, Ra
continue;
}
XXTrxLog xTrxLog = processFieldToCreateTrxLog(field,
- objectName, nameField, vObj, mObj, action);
+ objectName, vObj, mObj, action);
if (xTrxLog != null) {
trxLogList.add(xTrxLog);
}
@@ -159,7 +159,7 @@ public class RangerServiceService extends RangerServiceServiceBase<XXService, Ra
Field[] superClassFields = vObj.getClass().getSuperclass().getDeclaredFields();
for(Field field : superClassFields) {
if("isEnabled".equalsIgnoreCase(field.getName())) {
- XXTrxLog xTrx = processFieldToCreateTrxLog(field, objectName, nameField, vObj, mObj, action);
+ XXTrxLog xTrx = processFieldToCreateTrxLog(field, objectName, vObj, mObj, action);
if(xTrx != null) {
trxLogList.add(xTrx);
}
@@ -176,7 +176,7 @@ public class RangerServiceService extends RangerServiceServiceBase<XXService, Ra
@SuppressWarnings("unchecked")
private XXTrxLog processFieldToCreateTrxLog(Field field, String objectName,
- Field nameField, RangerService vObj, XXService mObj, int action) {
+ RangerService vObj, XXService mObj, int action) {
String actionString = "";
@@ -191,15 +191,15 @@ public class RangerServiceService extends RangerServiceServiceBase<XXService, Ra
String value = null;
boolean isEnum = vTrxLogAttr.isEnum();
- if (isEnum) {
-
- } else if ("configs".equalsIgnoreCase(fieldName)) {
- Map<String, String> configs = (field.get(vObj) != null) ? (Map<String, String>) field
- .get(vObj) : new HashMap<String, String>();
-
- value = jsonUtil.readMapToString(configs);
- } else {
- value = "" + field.get(vObj);
+ if (!isEnum) {
+ if ("configs".equalsIgnoreCase(fieldName)) {
+ Map<String, String> configs = (field.get(vObj) != null) ? (Map<String, String>) field
+ .get(vObj) : new HashMap<String, String>();
+
+ value = jsonUtil.readMapToString(configs);
+ } else {
+ value = "" + field.get(vObj);
+ }
}
if (action == OPERATION_CREATE_CONTEXT) {
@@ -219,9 +219,7 @@ public class RangerServiceService extends RangerServiceServiceBase<XXService, Ra
mField.setAccessible(true);
String mFieldName = mField.getName();
if (fieldName.equalsIgnoreCase(mFieldName)) {
- if (isEnum) {
-
- } else {
+ if (!isEnum) {
oldValue = mField.get(mObj) + "";
}
break;
http://git-wip-us.apache.org/repos/asf/ranger/blob/b66e98dc/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
index cb496ea..bd474cc 100644
--- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
+++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
@@ -24,18 +24,14 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.ranger.common.AppConstants;
import org.apache.ranger.common.ContextUtil;
-import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.RESTErrorUtil;
-import org.apache.ranger.common.RangerConstants;
import org.apache.ranger.common.RangerFactory;
import org.apache.ranger.common.SearchCriteria;
import org.apache.ranger.common.StringUtil;
import org.apache.ranger.common.UserSessionBase;
import org.apache.ranger.db.*;
import org.apache.ranger.entity.*;
-import org.apache.ranger.plugin.model.RangerBaseModelObject;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess;
@@ -53,7 +49,6 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef;
//import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
import org.apache.ranger.plugin.store.PList;
import org.apache.ranger.plugin.store.ServicePredicateUtil;
-import org.apache.ranger.plugin.store.ServiceStore;
import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.plugin.util.ServicePolicies;
import org.apache.ranger.security.context.RangerContextHolder;
@@ -69,12 +64,8 @@ import org.apache.ranger.service.XUserService;
import org.apache.ranger.view.RangerPolicyList;
import org.apache.ranger.view.RangerServiceDefList;
import org.apache.ranger.view.RangerServiceList;
-import org.apache.ranger.view.VXAccessAuditList;
-import org.apache.ranger.view.VXGroup;
import org.apache.ranger.view.VXGroupList;
-import org.apache.ranger.view.VXMetricUserGroupCount;
import org.apache.ranger.view.VXString;
-import org.apache.ranger.view.VXUser;
import org.apache.ranger.view.VXUserList;
import org.junit.Assert;
import org.junit.FixMethodOrder;
@@ -88,9 +79,6 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
@RunWith(MockitoJUnitRunner.class)
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class TestServiceDBStore {
@@ -1320,7 +1308,6 @@ public class TestServiceDBStore {
policyResourceMap.setValue("1L");
policyResourceMapList.add(policyResourceMap);
- List<XXPolicyLabelMap> xxPolicyLabelMapList = new ArrayList<>();
List<XXServiceConfigDef> xServiceConfigDefList = new ArrayList<XXServiceConfigDef>();
XXServiceConfigDef serviceConfigDefObj = new XXServiceConfigDef();
serviceConfigDefObj.setId(Id);
[14/28] ranger git commit: RANGER-2173: Optimize Trie constuction and
Policy lookup - Part II
Posted by me...@apache.org.
RANGER-2173: Optimize Trie constuction and Policy lookup
- Part II
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/27f56df1
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/27f56df1
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/27f56df1
Branch: refs/heads/ranger-1.1
Commit: 27f56df10c667b5f8de192d6acbc6968220afa57
Parents: c470225
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Sun Aug 5 09:01:24 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:38:30 2018 +0530
----------------------------------------------------------------------
.../org/apache/ranger/plugin/util/RangerResourceTrie.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/27f56df1/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index 1723d14..10c6faa 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -122,9 +122,11 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
RangerPerfTracer.logAlways(perf);
- if (PERF_TRIE_INIT_LOG.isTraceEnabled()) {
- PERF_TRIE_INIT_LOG.trace(toString());
+ if (PERF_TRIE_INIT_LOG.isDebugEnabled()) {
+ PERF_TRIE_INIT_LOG.debug(toString());
+ }
+ if (PERF_TRIE_INIT_LOG.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
root.toString("", sb);
PERF_TRIE_INIT_LOG.trace("Trie Dump:\n{" + sb.toString() + "}");
@@ -442,7 +444,9 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
final String childStr = child.getStr();
final int childStrLen = childStr.length();
- if (!StringUtils.equals(childStr, str)) {
+ final boolean isExactMatch = optIgnoreCase ? StringUtils.equalsIgnoreCase(childStr, str) : StringUtils.equals(childStr, str);
+
+ if (!isExactMatch) {
final int numOfCharactersToMatch = childStrLen < len ? childStrLen : len;
int index = 1;
for (; index < numOfCharactersToMatch; index++) {
[05/28] ranger git commit: RANGER-2166: A ClassNotFound exception is
thrown with atlasrest as a tag source
Posted by me...@apache.org.
RANGER-2166: A ClassNotFound exception is thrown with atlasrest as a tag source
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/0926baf7
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/0926baf7
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/0926baf7
Branch: refs/heads/ranger-1.1
Commit: 0926baf7f54eb17195640b9f16b1397ad590b35f
Parents: d3c10fc
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Fri Jul 27 10:59:29 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:28:33 2018 +0530
----------------------------------------------------------------------
src/main/assembly/tagsync.xml | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/0926baf7/src/main/assembly/tagsync.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml
index bc6e28b..05a6739 100644
--- a/src/main/assembly/tagsync.xml
+++ b/src/main/assembly/tagsync.xml
@@ -60,6 +60,7 @@
<include>com.fasterxml.jackson.core:jackson-databind:jar:${atlas.jackson.version}</include>
<include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${atlas.jackson.version}</include>
<include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${atlas.jackson.version}</include>
+ <include>com.fasterxml.jackson.jaxrs:jackson-module-jaxb-annotations:jar:${atlas.jackson.version}</include>
<include>org.codehaus.jackson:jackson-core-asl:jar:${codehaus.jackson.version}</include>
<include>org.codehaus.jackson:jackson-mapper-asl:jar:${codehaus.jackson.version}</include>
<include>org.codehaus.jackson:jackson-jaxrs:jar:${codehaus.jackson.version}</include>
@@ -68,6 +69,7 @@
<include>org.slf4j:slf4j-api</include>
<include>aopalliance:aopalliance:jar:${aopalliance.version}</include>
<include>commons-cli:commons-cli:jar:${commons.cli.version}</include>
+ <include>commons-codec:commons-codec</include>
<include>commons-collections:commons-collections:jar:${commons.collections.version}</include>
<include>commons-configuration:commons-configuration:jar:${commons.configuration.version}</include>
<include>commons-io:commons-io:jar:${commons.io.version}</include>
[19/28] ranger git commit: RANGER-2188: Support multiple threads to
build Trie and on-lookup post-setup for Trie nodes
Posted by me...@apache.org.
RANGER-2188: Support multiple threads to build Trie and on-lookup post-setup for Trie nodes
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/e37ca0be
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/e37ca0be
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/e37ca0be
Branch: refs/heads/ranger-1.1
Commit: e37ca0be33a635c3fbac2f2a79b0508f487489af
Parents: b163a48
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Aug 14 10:42:42 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:42:04 2018 +0530
----------------------------------------------------------------------
.../contextenricher/RangerTagEnricher.java | 9 +
.../policyengine/RangerPolicyEngineOptions.java | 15 +-
.../policyengine/RangerPolicyRepository.java | 25 +-
.../ranger/plugin/util/RangerResourceTrie.java | 383 ++++++++++++++++---
.../plugin/policyengine/TestPolicyEngine.java | 7 +
5 files changed, 372 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/e37ca0be/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index b12d8ff..d671b73 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -62,6 +62,8 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class);
private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init");
+ private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.retrieval");
+
public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval";
public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName";
@@ -401,6 +403,12 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
if (resource == null || resource.getKeys() == null || resource.getKeys().isEmpty() || serviceResourceTrie == null) {
ret = enrichedServiceTags.getServiceResourceMatchers();
} else {
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerTagEnricher.getEvaluators(resource=" + resource.getAsString() + ")");
+ }
+
Set<String> resourceKeys = resource.getKeys();
List<List<RangerServiceResourceMatcher>> serviceResourceMatchersList = null;
List<RangerServiceResourceMatcher> smallestList = null;
@@ -452,6 +460,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
ret = smallestList;
}
}
+ RangerPerfTracer.logAlways(perf);
}
if(ret == null) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/e37ca0be/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
index b76820c..01dbbe8 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
@@ -33,6 +33,7 @@ public class RangerPolicyEngineOptions {
public boolean evaluateDelegateAdminOnly = false;
public boolean enableTagEnricherWithLocalRefresher = false;
public boolean disableAccessEvaluationWithPolicyACLSummary = true;
+ public boolean optimizeTrieForRetrieval = true;
private RangerServiceDefHelper serviceDefHelper;
@@ -50,6 +51,8 @@ public class RangerPolicyEngineOptions {
evaluateDelegateAdminOnly = false;
enableTagEnricherWithLocalRefresher = false;
disableAccessEvaluationWithPolicyACLSummary = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.access.evaluation.with.policy.acl.summary", true);
+ optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", true);
+
}
public void configureDefaultRangerAdmin(Configuration conf, String propertyPrefix) {
@@ -62,6 +65,8 @@ public class RangerPolicyEngineOptions {
evaluateDelegateAdminOnly = false;
enableTagEnricherWithLocalRefresher = false;
disableAccessEvaluationWithPolicyACLSummary = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.access.evaluation.with.policy.acl.summary", true);
+ optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false);
+
}
public void configureDelegateAdmin(Configuration conf, String propertyPrefix) {
@@ -69,6 +74,8 @@ public class RangerPolicyEngineOptions {
disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true);
disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", true);
disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false);
+ optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false);
+
cacheAuditResults = false;
evaluateDelegateAdminOnly = true;
@@ -81,6 +88,8 @@ public class RangerPolicyEngineOptions {
disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true);
disableTagPolicyEvaluation = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tagpolicy.evaluation", false);
disableTrieLookupPrefilter = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false);
+ optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false);
+
cacheAuditResults = false;
evaluateDelegateAdminOnly = false;
@@ -111,7 +120,8 @@ public class RangerPolicyEngineOptions {
&& this.disableTrieLookupPrefilter == that.disableTrieLookupPrefilter
&& this.cacheAuditResults == that.cacheAuditResults
&& this.evaluateDelegateAdminOnly == that.evaluateDelegateAdminOnly
- && this.enableTagEnricherWithLocalRefresher == that.enableTagEnricherWithLocalRefresher;
+ && this.enableTagEnricherWithLocalRefresher == that.enableTagEnricherWithLocalRefresher
+ && this.optimizeTrieForRetrieval == that.optimizeTrieForRetrieval;
}
return ret;
}
@@ -133,6 +143,8 @@ public class RangerPolicyEngineOptions {
ret *= 2;
ret += enableTagEnricherWithLocalRefresher ? 1 : 0;
ret *= 2;
+ ret += optimizeTrieForRetrieval ? 1 : 0;
+ ret *= 2;
return ret;
}
@@ -144,6 +156,7 @@ public class RangerPolicyEngineOptions {
", disableContextEnrichers: " + disableContextEnrichers +
", disableCustomConditions: " + disableContextEnrichers +
", disableTrieLookupPrefilter: " + disableTrieLookupPrefilter +
+ ", optimizeTrieForRetrieval: " + optimizeTrieForRetrieval +
" }";
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/e37ca0be/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 088b729..f1b9764 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -54,6 +54,7 @@ class RangerPolicyRepository {
private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class);
private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init");
+ private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.retrieval");
enum AuditModeEnum {
AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT
@@ -145,9 +146,9 @@ class RangerPolicyRepository {
dataMaskResourceTrie = null;
rowFilterResourceTrie = null;
} else {
- policyResourceTrie = createResourceTrieMap(policyEvaluators);
- dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators);
- rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators);
+ policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval);
+ dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval);
+ rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval);
}
}
@@ -190,9 +191,9 @@ class RangerPolicyRepository {
dataMaskResourceTrie = null;
rowFilterResourceTrie = null;
} else {
- policyResourceTrie = createResourceTrieMap(policyEvaluators);
- dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators);
- rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators);
+ policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval);
+ dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval);
+ rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval);
}
}
@@ -383,6 +384,12 @@ class RangerPolicyRepository {
List<RangerPolicyEvaluator> ret = null;
Set<String> resourceKeys = resource == null ? null : resource.getKeys();
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerPolicyRepository.getLikelyMatchEvaluators(resource=" + resource.getAsString() + ")");
+ }
+
if(CollectionUtils.isNotEmpty(resourceKeys)) {
List<List<RangerPolicyEvaluator>> resourceEvaluatorsList = null;
List<RangerPolicyEvaluator> smallestList = null;
@@ -439,6 +446,8 @@ class RangerPolicyRepository {
ret = Collections.emptyList();
}
+ RangerPerfTracer.logAlways(perf);
+
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerPolicyRepository.getLikelyMatchPolicyEvaluators(" + resource.getAsString() + "): evaluatorCount=" + ret.size());
}
@@ -876,14 +885,14 @@ class RangerPolicyRepository {
return ret;
}
- private Map<String, RangerResourceTrie> createResourceTrieMap(List<RangerPolicyEvaluator> evaluators) {
+ private Map<String, RangerResourceTrie> createResourceTrieMap(List<RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval) {
final Map<String, RangerResourceTrie> ret;
if (CollectionUtils.isNotEmpty(evaluators) && serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) {
ret = new HashMap<>();
for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
- ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators, RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR));
+ ret.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, evaluators, RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR, optimizeTrieForRetrieval));
}
} else {
ret = null;
http://git-wip-us.apache.org/repos/asf/ranger/blob/e37ca0be/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index 10c6faa..e702684 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -24,6 +24,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator;
@@ -36,6 +37,8 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class);
@@ -43,6 +46,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.op");
private static final String DEFAULT_WILDCARD_CHARS = "*?";
+ private static final String TRIE_BUILDER_THREAD_COUNT = "ranger.policyengine.trie.builder.thread.count";
private final String resourceName;
private final boolean optIgnoreCase;
@@ -50,14 +54,15 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
private final String wildcardChars;
private final TrieNode<T> root;
private final Comparator<T> comparator;
+ private final boolean isOptimizedForRetrieval;
public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<T> evaluators) {
- this(resourceDef, evaluators, null);
+ this(resourceDef, evaluators, null, true);
}
- public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<T> evaluators, Comparator<T> comparator) {
+ public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<T> evaluators, Comparator<T> comparator, boolean isOptimizedForRetrieval) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ")");
+ LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + ")");
}
RangerPerfTracer perf = null;
@@ -66,6 +71,15 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + ")");
}
+ int builderThreadCount = RangerConfiguration.getInstance().getInt(TRIE_BUILDER_THREAD_COUNT, 1);
+
+ if (builderThreadCount < 1) {
+ builderThreadCount = 1;
+ }
+
+ LOG.info("builderThreadCount is set to ["+ builderThreadCount +"]");
+ PERF_TRIE_INIT_LOG.info("builderThreadCount is set to ["+ builderThreadCount +"]");
+
Map<String, String> matcherOptions = resourceDef.getMatcherOptions();
boolean optReplaceTokens = RangerAbstractResourceMatcher.getOptionReplaceTokens(matcherOptions);
@@ -86,40 +100,17 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
this.optIgnoreCase = RangerAbstractResourceMatcher.getOptionIgnoreCase(matcherOptions);
this.optWildcard = RangerAbstractResourceMatcher.getOptionWildCard(matcherOptions);
this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS + tokenReplaceSpecialChars : "" + tokenReplaceSpecialChars;
- this.root = new TrieNode<>(null);
this.comparator = comparator;
+ this.isOptimizedForRetrieval = isOptimizedForRetrieval;
- for(T evaluator : evaluators) {
- Map<String, RangerPolicyResource> policyResources = evaluator.getPolicyResource();
- RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null;
-
- if(policyResource == null) {
- if(evaluator.getLeafResourceLevel() != null && resourceDef.getLevel() != null && evaluator.getLeafResourceLevel() < resourceDef.getLevel()) {
- root.addWildcardEvaluator(evaluator);
- }
-
- continue;
- }
-
- if(policyResource.getIsExcludes()) {
- root.addWildcardEvaluator(evaluator);
- } else {
- RangerResourceMatcher resourceMatcher = evaluator.getResourceMatcher(resourceName);
+ TrieNode<T> tmpRoot = buildTrie(resourceDef, evaluators, comparator, builderThreadCount);
- if(resourceMatcher != null && (resourceMatcher.isMatchAny())) {
- root.addWildcardEvaluator(evaluator);
- } else {
- if(CollectionUtils.isNotEmpty(policyResource.getValues())) {
- for (String resource : policyResource.getValues()) {
- insert(resource, policyResource.getIsRecursive(), evaluator);
- }
- }
- }
- }
+ if (builderThreadCount > 1 && tmpRoot == null) { // if multi-threaded trie-creation failed, build using a single thread
+ this.root = buildTrie(resourceDef, evaluators, comparator, 1);
+ } else {
+ this.root = tmpRoot;
}
- root.postSetup(null, comparator);
-
RangerPerfTracer.logAlways(perf);
if (PERF_TRIE_INIT_LOG.isDebugEnabled()) {
@@ -133,7 +124,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + "): " + toString());
+ LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + "): " + toString());
}
}
@@ -158,6 +149,138 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
return null;
}
+ private TrieNode<T> buildTrie(RangerServiceDef.RangerResourceDef resourceDef, List<T> evaluators, Comparator<T> comparator, int builderThreadCount) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + (builderThreadCount > 1) + ")");
+ }
+
+ TrieNode<T> ret = new TrieNode<>(null);
+ final boolean isMultiThreaded = builderThreadCount > 1;
+ final List<ResourceTrieBuilderThread> builderThreads;
+ final Map<Character, Integer> builderThreadMap;
+ int lastUsedThreadIndex = 0;
+
+ if (isMultiThreaded) {
+ builderThreads = new ArrayList<>();
+ for (int i = 0; i < builderThreadCount; i++) {
+ ResourceTrieBuilderThread t = new ResourceTrieBuilderThread(isOptimizedForRetrieval);
+ builderThreads.add(t);
+ t.start();
+ }
+ builderThreadMap = new HashMap<>();
+ } else {
+ builderThreads = null;
+ builderThreadMap = null;
+ }
+
+ for (T evaluator : evaluators) {
+ Map<String, RangerPolicyResource> policyResources = evaluator.getPolicyResource();
+ RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null;
+
+ if (policyResource == null) {
+ if (evaluator.getLeafResourceLevel() != null && resourceDef.getLevel() != null && evaluator.getLeafResourceLevel() < resourceDef.getLevel()) {
+ ret.addWildcardEvaluator(evaluator);
+ }
+
+ continue;
+ }
+
+ if (policyResource.getIsExcludes()) {
+ ret.addWildcardEvaluator(evaluator);
+ } else {
+ RangerResourceMatcher resourceMatcher = evaluator.getResourceMatcher(resourceName);
+
+ if (resourceMatcher != null && (resourceMatcher.isMatchAny())) {
+ ret.addWildcardEvaluator(evaluator);
+ } else {
+ if (CollectionUtils.isNotEmpty(policyResource.getValues())) {
+ for (String resource : policyResource.getValues()) {
+ if (!isMultiThreaded) {
+ insert(ret, resource, policyResource.getIsRecursive(), evaluator);
+ } else {
+ try {
+ lastUsedThreadIndex = insert(ret, resource, policyResource.getIsRecursive(), evaluator, builderThreadMap, builderThreads, lastUsedThreadIndex);
+ } catch (InterruptedException ex) {
+ LOG.error("Failed to dispatch " + resource + " to " + builderThreads.get(lastUsedThreadIndex));
+ LOG.error("Failing and retrying with one thread");
+
+ ret = null;
+
+ break;
+ }
+ }
+ }
+ if (ret == null) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ if (ret != null) {
+ if (isMultiThreaded) {
+ ret.setup(null, comparator);
+
+ for (ResourceTrieBuilderThread t : builderThreads) {
+ t.setParentWildcardEvaluators(ret.wildcardEvaluators);
+ try {
+ // Send termination signal to each thread
+ t.add("", false, null);
+ // Wait for threads to finish work
+ t.join();
+ ret.getChildren().putAll(t.getSubtrees());
+ } catch (InterruptedException ex) {
+ LOG.error("BuilderThread " + t + " was interrupted:", ex);
+ LOG.error("Failing and retrying with one thread");
+
+ ret = null;
+
+ break;
+ }
+ }
+ } else {
+ if (isOptimizedForRetrieval) {
+ RangerPerfTracer postSetupPerf = null;
+
+ if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
+ postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + "-postSetup)");
+ }
+
+ ret.postSetup(null, comparator);
+
+ RangerPerfTracer.logAlways(postSetupPerf);
+ } else {
+ ret.setup(null, comparator);
+ }
+ }
+ }
+
+ if (isMultiThreaded) {
+ cleanUpThreads(builderThreads);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + isMultiThreaded + ") :" + ret);
+ }
+
+ return ret;
+ }
+
+ private void cleanUpThreads(List<ResourceTrieBuilderThread> builderThreads) {
+ if (CollectionUtils.isNotEmpty(builderThreads)) {
+ for (ResourceTrieBuilderThread t : builderThreads) {
+ try {
+ if (t.isAlive()) {
+ t.interrupt();
+ t.join();
+ }
+ } catch (InterruptedException ex) {
+ LOG.error("Could not terminate thread " + t);
+ }
+ }
+ }
+ }
+
private TrieData getTrieData() {
TrieData ret = new TrieData();
@@ -179,18 +302,37 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
return getLookupChar(str.charAt(index));
}
- private void insert(String resource, boolean isRecursive, T evaluator) {
+ private int insert(TrieNode<T> currentRoot, String resource, boolean isRecursive, T evaluator, Map<Character, Integer> builderThreadMap, List<ResourceTrieBuilderThread> builderThreads, int lastUsedThreadIndex) throws InterruptedException {
+ int ret = lastUsedThreadIndex;
+ final String prefix = getNonWildcardPrefix(resource);
+
+ if (StringUtils.isNotEmpty(prefix)) {
+ char c = getLookupChar(prefix.charAt(0));
+ Integer index = builderThreadMap.get(c);
+
+ if (index == null) {
+ ret = index = (lastUsedThreadIndex + 1) % builderThreads.size();
+ builderThreadMap.put(c, index);
+ }
+
+ builderThreads.get(index).add(resource, isRecursive, evaluator);
+ } else {
+ currentRoot.addWildcardEvaluator(evaluator);
+ }
+
+ return ret;
+ }
+ private void insert(TrieNode<T> currentRoot, String resource, boolean isRecursive, T evaluator) {
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.insert(resource=" + resource + ")");
}
- TrieNode<T> curr = root;
-
- final String prefix = getNonWildcardPrefix(resource);
- final boolean isWildcard = prefix.length() != resource.length();
+ TrieNode<T> curr = currentRoot;
+ final String prefix = getNonWildcardPrefix(resource);
+ final boolean isWildcard = prefix.length() != resource.length();
if (StringUtils.isNotEmpty(prefix)) {
curr = curr.getOrCreateChild(prefix);
@@ -206,14 +348,17 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
private String getNonWildcardPrefix(String str) {
- if (!optWildcard) return str;
+
int minIndex = str.length();
+
for (int i = 0; i < wildcardChars.length(); i++) {
int index = str.indexOf(wildcardChars.charAt(i));
+
if (index != -1 && index < minIndex) {
minIndex = index;
}
}
+
return str.substring(0, minIndex);
}
@@ -228,12 +373,16 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerResourceTrie.getEvaluatorsForResource(resource=" + resource + ")");
}
- TrieNode<T> curr = root;
-
- final int len = resource.length();
- int i = 0;
+ TrieNode<T> curr = root;
+ TrieNode<T> parent = null;
+ final int len = resource.length();
+ int i = 0;
while (i < len) {
+ if (!isOptimizedForRetrieval) {
+ curr.setupIfNeeded(parent, comparator);
+ }
+
final TrieNode<T> child = curr.getChild(getLookupChar(resource, i));
if (child == null) {
@@ -246,10 +395,15 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
break;
}
+ parent = curr;
curr = child;
i += childStr.length();
}
+ if (!isOptimizedForRetrieval) {
+ curr.setupIfNeeded(parent, comparator);
+ }
+
List<T> ret = i == len ? curr.getEvaluators() : curr.getWildcardEvaluators();
RangerPerfTracer.logAlways(perf);
@@ -334,6 +488,91 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
return sb.toString();
}
+ class ResourceTrieBuilderThread extends Thread {
+
+ class WorkItem {
+ final String resourceName;
+ final boolean isRecursive;
+ final T evaluator;
+
+ WorkItem(String resourceName, boolean isRecursive, T evaluator) {
+ this.resourceName = resourceName;
+ this.isRecursive = isRecursive;
+ this.evaluator = evaluator;
+ }
+ @Override
+ public String toString() {
+ return
+ "resourceName=" + resourceName +
+ "isRecursive=" + isRecursive +
+ "evaluator=" + (evaluator != null? evaluator.getId() : null);
+ }
+ }
+
+ private final TrieNode<T> thisRoot = new TrieNode<>(null);
+ private final BlockingQueue<WorkItem> workQueue = new LinkedBlockingQueue<>();
+ private final boolean isOptimizedForRetrieval;
+ private List<T> parentWildcardEvaluators;
+
+ ResourceTrieBuilderThread(boolean isOptimizedForRetrieval) {
+ this.isOptimizedForRetrieval = isOptimizedForRetrieval;
+ }
+
+ void add(String resourceName, boolean isRecursive, T evaluator) throws InterruptedException {
+ workQueue.put(new WorkItem(resourceName, isRecursive, evaluator));
+ }
+
+ void setParentWildcardEvaluators(List<T> parentWildcardEvaluators) {
+ this.parentWildcardEvaluators = parentWildcardEvaluators;
+ }
+
+ Map<Character, TrieNode<T>> getSubtrees() { return thisRoot.getChildren(); }
+
+ @Override
+ public void run() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Running " + this);
+ }
+
+ while (true) {
+ final WorkItem workItem;
+
+ try {
+ workItem = workQueue.take();
+ } catch (InterruptedException exception) {
+ LOG.error("Thread=" + this + " is interrupted", exception);
+
+ break;
+ }
+
+ if (workItem.evaluator != null) {
+ insert(thisRoot, workItem.resourceName, workItem.isRecursive, workItem.evaluator);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received termination signal. " + workItem);
+ }
+ break;
+ }
+ }
+
+ if (!isInterrupted() && isOptimizedForRetrieval) {
+ RangerPerfTracer postSetupPerf = null;
+
+ if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
+ postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(thread=" + this.getName() + "-postSetup)");
+ }
+
+ thisRoot.postSetup(parentWildcardEvaluators, comparator);
+
+ RangerPerfTracer.logAlways(postSetupPerf);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Exiting " + this);
+ }
+ }
+ }
+
class TrieData {
int nodeCount;
int leafNodeCount;
@@ -346,11 +585,12 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
class TrieNode<U extends RangerPolicyResourceEvaluator> {
- private String str;
- private Map<Character, TrieNode<U>> children = new HashMap<>();
- private List<U> evaluators;
- private List<U> wildcardEvaluators;
- private boolean isSharingParentWildcardEvaluators;
+ private String str;
+ private final Map<Character, TrieNode<U>> children = new HashMap<>();
+ private List<U> evaluators;
+ private List<U> wildcardEvaluators;
+ private boolean isSharingParentWildcardEvaluators;
+ private volatile boolean isSetup = false;
TrieNode(String str) {
this.str = str;
@@ -507,6 +747,38 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
void postSetup(List<U> parentWildcardEvaluators, Comparator<U> comparator) {
+
+ setup(parentWildcardEvaluators, comparator);
+
+ if (children != null) {
+ for (Map.Entry<Character, TrieNode<U>> entry : children.entrySet()) {
+ TrieNode<U> child = entry.getValue();
+
+ child.postSetup(wildcardEvaluators, comparator);
+ }
+ }
+ }
+
+ void setupIfNeeded(TrieNode<U> parent, Comparator<U> comparator) {
+ if (parent == null) {
+ return;
+ }
+
+ boolean setupNeeded = !isSetup;
+
+ if (setupNeeded) {
+ synchronized (this) {
+ setupNeeded = !isSetup;
+
+ if (setupNeeded) {
+ setup(parent.getWildcardEvaluators(), comparator);
+ isSetup = true;
+ }
+ }
+ }
+ }
+
+ void setup(List<U> parentWildcardEvaluators, Comparator<U> comparator) {
// finalize wildcard-evaluators list by including parent's wildcard evaluators
if (parentWildcardEvaluators != null) {
if (CollectionUtils.isEmpty(this.wildcardEvaluators)) {
@@ -539,14 +811,6 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
evaluators.sort(comparator);
}
}
-
- if (children != null) {
- for (Map.Entry<Character, TrieNode<U>> entry : children.entrySet()) {
- TrieNode<U> child = entry.getValue();
-
- child.postSetup(wildcardEvaluators, comparator);
- }
- }
}
public void toString(String prefix, StringBuilder sb) {
@@ -584,8 +848,11 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
public void clear() {
- children = null;
- evaluators = null;
+ if (children != null) {
+ children.clear();
+ }
+
+ evaluators = null;
wildcardEvaluators = null;
}
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/e37ca0be/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
----------------------------------------------------------------------
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
index 1d9b865..9d9be6c 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
@@ -160,6 +160,10 @@ public class TestPolicyEngine {
" <name>ranger.plugin.tag.attr.additional.date.formats</name>\n" +
" <value>abcd||xyz||yyyy/MM/dd'T'HH:mm:ss.SSS'Z'</value>\n" +
" </property>\n" +
+ " <property>\n" +
+ " <name>ranger.policyengine.trie.builder.thread.count</name>\n" +
+ " <value>3</value>\n" +
+ " </property>\n" +
"</configuration>\n");
writer.close();
@@ -368,6 +372,7 @@ public class TestPolicyEngine {
policyEngineOptions.disableTagPolicyEvaluation = false;
policyEngineOptions.disableAccessEvaluationWithPolicyACLSummary = false;
+ policyEngineOptions.optimizeTrieForRetrieval = false;
boolean useForwardedIPAddress = RangerConfiguration.getInstance().getBoolean("ranger.plugin.hive.use.x-forwarded-for.ipaddress", false);
String trustedProxyAddressString = RangerConfiguration.getInstance().get("ranger.plugin.hive.trusted.proxy.ipaddresses");
@@ -383,6 +388,8 @@ public class TestPolicyEngine {
policyEngine.setTrustedProxyAddresses(trustedProxyAddresses);
policyEngineOptions.disableAccessEvaluationWithPolicyACLSummary = true;
+ policyEngineOptions.optimizeTrieForRetrieval = false;
+
RangerPolicyEngine policyEngineForResourceAccessInfo = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions);
policyEngineForResourceAccessInfo.setUseForwardedIPAddress(useForwardedIPAddress);
[27/28] ranger git commit: RANGER-2196:Ensure that any explicit
threads used by Ranger are marked as daemon threads
Posted by me...@apache.org.
RANGER-2196:Ensure that any explicit threads used by Ranger are marked as daemon threads
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/d4b5722d
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/d4b5722d
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/d4b5722d
Branch: refs/heads/ranger-1.1
Commit: d4b5722d7b812fb18c46edf8e81d06e0f169c6a7
Parents: e8c5935
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Wed Aug 22 17:38:46 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:50:45 2018 +0530
----------------------------------------------------------------------
.../java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java | 2 ++
.../apache/ranger/plugin/policyengine/RangerPolicyRepository.java | 3 +++
.../java/org/apache/ranger/plugin/util/RangerResourceTrie.java | 1 +
.../main/java/org/apache/ranger/biz/RangerPolicyRetriever.java | 1 +
.../src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java | 1 +
.../apache/ranger/authentication/UnixAuthenticationService.java | 2 ++
6 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/d4b5722d/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java
index 8a14394..769b86c 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java
@@ -144,6 +144,8 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
mDispatcherThread = new DestinationDispatcherThread<T>(this, destination, mLogger);
+ mDispatcherThread.setDaemon(true);
+
mDispatcherThread.start();
mLogger.debug("<== LocalFileLogBuffer.start()");
http://git-wip-us.apache.org/repos/asf/ranger/blob/d4b5722d/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index f1b9764..f64e773 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -680,6 +680,9 @@ class RangerPolicyRepository {
}
}
}
+ if (LOG.isInfoEnabled()) {
+ LOG.info("This policy engine contains " + (policyEvaluators.size()+dataMaskPolicyEvaluators.size()+rowFilterPolicyEvaluators.size()) + " policy evaluators");
+ }
RangerPolicyEvaluator.PolicyEvalOrderComparator comparator = new RangerPolicyEvaluator.PolicyEvalOrderComparator();
Collections.sort(policyEvaluators, comparator);
this.policyEvaluators = Collections.unmodifiableList(policyEvaluators);
http://git-wip-us.apache.org/repos/asf/ranger/blob/d4b5722d/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index 9c7c775..a2d52a0 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -170,6 +170,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
builderThreads = new ArrayList<>();
for (int i = 0; i < builderThreadCount; i++) {
ResourceTrieBuilderThread t = new ResourceTrieBuilderThread(isOptimizedForRetrieval);
+ t.setDaemon(true);
builderThreads.add(t);
t.start();
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/d4b5722d/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index efbdfdf..c26f0a5 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -178,6 +178,7 @@ public class RangerPolicyRetriever {
}
PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService);
+ t.setDaemon(true);
t.start();
try {
t.join();
http://git-wip-us.apache.org/repos/asf/ranger/blob/d4b5722d/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index e5e9572..7875bc2 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -98,6 +98,7 @@ public class RangerTagDBRetriever {
}
TagLoaderThread t = new TagLoaderThread(txTemplate, xService);
+ t.setDaemon(true);
t.start();
try {
t.join();
http://git-wip-us.apache.org/repos/asf/ranger/blob/d4b5722d/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java
----------------------------------------------------------------------
diff --git a/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java b/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java
index 45834f4..1ee5e21 100644
--- a/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java
+++ b/unixauthservice/src/main/java/org/apache/ranger/authentication/UnixAuthenticationService.java
@@ -131,6 +131,8 @@ public class UnixAuthenticationService {
UserGroupSync syncProc = new UserGroupSync();
Thread newSyncProcThread = new Thread(syncProc);
newSyncProcThread.setName("UnixUserSyncThread");
+ // If this thread is set as daemon, then the entire process will terminate if enableUnixAuth is false
+ // Therefore this is marked as non-daemon thread. Don't change the following line
newSyncProcThread.setDaemon(false);
newSyncProcThread.start();
}
[09/28] ranger git commit: RANGER-2153 Supply the function of
reverting policy history version.
Posted by me...@apache.org.
RANGER-2153 Supply the function of reverting policy history version.
Signed-off-by: pengjianhua <35...@qq.com>
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/e902bef0
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/e902bef0
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/e902bef0
Branch: refs/heads/ranger-1.1
Commit: e902bef006690c7d5a92c8f48857a622f0293073
Parents: 3598282
Author: 洪一帆6186000050 <ho...@zte.com.cn>
Authored: Fri Jul 27 14:28:24 2018 +0800
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:34:41 2018 +0530
----------------------------------------------------------------------
.../scripts/views/policies/RangerPolicyRO.js | 27 ++++++++++++++++++++
.../views/policies/RangerPolicyTableLayout.js | 3 +++
2 files changed, 30 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/e902bef0/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js
----------------------------------------------------------------------
diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js
index a0b46fd..1e3554f 100644
--- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js
+++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js
@@ -30,6 +30,7 @@ define(function(require) {
var RangerPolicyROTmpl = require('hbs!tmpl/policies/RangerPolicyRO_tmpl');
var RangerService = require('models/RangerService');
+ var RangerPolicy = require('models/RangerPolicy');
var RangerPolicyRO = Backbone.Marionette.Layout.extend({
_viewName: 'RangerPolicyRO',
@@ -268,6 +269,7 @@ define(function(require) {
this.render();
var verEl = $(e.currentTarget).parent();
verEl.find('text').text('Version '+this.policy.get('version'));
+ var reverEl = verEl.next('#revert');
var prevEl = verEl.find('#preVer'),
nextEl = verEl.find('#nextVer');
if(this.policy.get('version')>1){
@@ -278,11 +280,36 @@ define(function(require) {
var policyVerIndexAt = this.policyVersionList.indexOf(this.policy.get('version').toString());
if(!_.isUndefined(this.policyVersionList[++policyVerIndexAt])){
nextEl.addClass('active');
+ reverEl.css('display','');
}else{
nextEl.removeClass('active');
+ reverEl.css('display','none');
}
},
+ revert : function(e){
+ e.preventDefault();
+ var policyId = this.policy.attributes.id;
+ var rangerPolicy = new RangerPolicy({id : policyId});
+ rangerPolicy.attributes = this.policy.attributes;
+ rangerPolicy.save({}, {
+ wait: true,
+ success: function() {
+ XAUtil.blockUI('unblock');
+ XAUtil.notifySuccess('Success', 'Policy reverted successfully.');
+ location.reload();
+ },
+ error: function(model, response, options) {
+ XAUtil.blockUI('unblock');
+ if(response && response.responseJSON && response.responseJSON.msgDesc) {
+ XAUtil.showErrorMsg(response.responseJSON.msgDesc);
+ } else {
+ XAUtil.notifyError('Error', 'Error reverting policy.');
+ }
+ }
+ });
+ },
+
/** on close */
onClose: function() {}
});
http://git-wip-us.apache.org/repos/asf/ranger/blob/e902bef0/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js
----------------------------------------------------------------------
diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js
index 206bb96..83253e4 100644
--- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js
+++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js
@@ -202,6 +202,9 @@ define(function(require){
policyVerEl.append('<i id="nextVer" class="icon-chevron-right ' + (!_.isUndefined(policyVersionList[++policyVerIndexAt]) ? 'active' : '') + '"></i>').find('#nextVer').click(function(e) {
view.nextVer(e);
});
+ policyVerEl.after('<a id="revert" href="#" class="btn btn-primary" style="display:none;">Revert</a>').next('#revert').click(function(e){
+ view.revert(e);
+ });
modal.$el.find('.cancel').hide();
});
},
[15/28] ranger git commit: RANGER-2177: Handle validations for
duplicate configuration item during service create/edit
Posted by me...@apache.org.
RANGER-2177: Handle validations for duplicate configuration item during service create/edit
Signed-off-by: zhangqiang2 <zh...@zte.com.cn>
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/a2de0ebf
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/a2de0ebf
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/a2de0ebf
Branch: refs/heads/ranger-1.1
Commit: a2de0ebff8dfb7d36f9e50ef027b404100be1f67
Parents: 27f56df
Author: xingpeng1 <xi...@zte.com.cn>
Authored: Fri Aug 3 15:58:24 2018 +0800
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:38:54 2018 +0530
----------------------------------------------------------------------
.../scripts/modules/globalize/message/en.js | 3 +-
.../scripts/views/service/ServiceCreate.js | 22 ++++++------
.../webapp/scripts/views/service/ServiceForm.js | 36 ++++++++++++++------
3 files changed, 39 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/a2de0ebf/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js
----------------------------------------------------------------------
diff --git a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js
index f4c3c27..99cbf55 100644
--- a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js
+++ b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js
@@ -437,7 +437,8 @@ define(function(require) {
noUserFoundText : 'No user associate with this group.',
showInitialHundredUser : 'Initially search filter is applied for first hundred users. To get more users click on ',
searchForUserSync :"Search for your user sync audits...",
- policyExpired :'Policy Expired'
+ policyExpired :'Policy Expired',
+ duplicateNewConfigValidationMsg :'Please make sure the new configuration has no duplicate names'
},
plcHldr : {
http://git-wip-us.apache.org/repos/asf/ranger/blob/a2de0ebf/security-admin/src/main/webapp/scripts/views/service/ServiceCreate.js
----------------------------------------------------------------------
diff --git a/security-admin/src/main/webapp/scripts/views/service/ServiceCreate.js b/security-admin/src/main/webapp/scripts/views/service/ServiceCreate.js
index c8ecdc3..7c2c778 100644
--- a/security-admin/src/main/webapp/scripts/views/service/ServiceCreate.js
+++ b/security-admin/src/main/webapp/scripts/views/service/ServiceCreate.js
@@ -17,7 +17,7 @@
* under the License.
*/
-
+
/*
* Repository/Service create view
*/
@@ -31,7 +31,7 @@ define(function(require){
var XAEnums = require('utils/XAEnums');
var XALinks = require('modules/XALinks');
var localization = require('utils/XALangSupport');
-
+
var ServiceForm = require('views/service/ServiceForm');
var RangerServiceDef = require('models/RangerServiceDef');
var ServiceCreateTmpl = require('hbs!tmpl/service/ServiceCreate_tmpl');
@@ -42,19 +42,19 @@ define(function(require){
_viewName : 'ServiceCreate',
template: ServiceCreateTmpl,
-
+
templateHelpers : function(){
return { editService : this.editService};
},
-
+
breadCrumbs :function(){
- var name = this.rangerServiceDefModel.get('name') != XAEnums.ServiceType.SERVICE_TAG.label ? 'ServiceManager' : 'TagBasedServiceManager';
+ var name = this.rangerServiceDefModel.get('name') != XAEnums.ServiceType.SERVICE_TAG.label ? 'ServiceManager' : 'TagBasedServiceManager';
if(this.model.isNew()){
return [XALinks.get(name), XALinks.get('ServiceCreate')];
} else {
return [XALinks.get(name), XALinks.get('ServiceEdit')];
}
- },
+ },
/** Layout sub regions */
regions: {
@@ -132,7 +132,9 @@ define(function(require){
if(! _.isEmpty(errors)){
return;
}
- this.form.formValidation();
+ if (!this.form.formValidation()) {
+ return;
+ }
this.saveService();
},
saveService : function(){
@@ -165,7 +167,7 @@ define(function(require){
msg :'Are you sure want to delete ?',
callback : function(){
XAUtil.blockUI();
-
+
that.model.destroy({
success: function(model, response) {
XAUtil.blockUI('unblock');
@@ -182,7 +184,7 @@ define(function(require){
}
}
});
-
+
}
});
},
@@ -242,7 +244,7 @@ define(function(require){
XAUtil.defaultErrorHandler(options , msResponse);
}
bootbox.alert("Connection Failed.");
- }
+ }
});
},
gotoResourceOrTagTab : function(){
http://git-wip-us.apache.org/repos/asf/ranger/blob/a2de0ebf/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js
----------------------------------------------------------------------
diff --git a/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js b/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js
index 079ce00..726fb5f 100644
--- a/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js
+++ b/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js
@@ -1,4 +1,4 @@
- /*
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,14 +17,14 @@
* under the License.
*/
-
+
define(function(require){
'use strict';
var Backbone = require('backbone');
var XAEnums = require('utils/XAEnums');
var XAUtil = require('utils/XAUtils');
-
+
var localization = require('utils/XALangSupport');
var BackboneFormDataType = require('models/BackboneFormDataType');
var ConfigurationList = require('views/service/ConfigurationList')
@@ -52,7 +52,7 @@ define(function(require){
serviceDetail += name+",";
}
});
-
+
return {
serviceDetail : serviceDetail.slice(0,-1),
serviceConfig : serviceConfig.slice(0,-1)
@@ -124,7 +124,7 @@ define(function(require){
} else {
this.fields.isEnabled.editor.setValue(XAEnums.ActiveStatus.STATUS_DISABLED.value);
}
- }
+ }
},
evIsEnabledChange : function(form, fieldEditor){
XAUtil.checkDirtyFieldForToggle(fieldEditor.$el);
@@ -143,8 +143,22 @@ define(function(require){
},
formValidation : function(){
- //return false;
- return true;
+ var valid = true;
+ var config = {};
+ this.extraConfigColl.each(function(obj){
+ if(!_.isEmpty(obj.attributes)) {
+ if (!_.isUndefined(config[obj.get('name')])) {
+ XAUtil.alertPopup({
+ msg : localization.tt('msg.duplicateNewConfigValidationMsg')
+ });
+ valid = false;
+ return;
+ } else {
+ config[obj.get('name')] = obj.get('value');
+ }
+ }
+ });
+ return valid;
},
beforeSave : function(){
@@ -165,11 +179,11 @@ define(function(require){
}
});
}
- this.extraConfigColl.each(function(obj){
+ this.extraConfigColl.each(function(obj){
if(!_.isEmpty(obj.attributes)) config[obj.get('name')] = obj.get('value');
});
- this.model.set('configs',config);
-
+ this.model.set('configs',config);
+
//Set service type
this.model.set('type',this.rangerServiceDefModel.get('name'))
//Set isEnabled
@@ -178,7 +192,7 @@ define(function(require){
} else {
this.model.set('isEnabled',false);
}
-
+
//Remove unwanted attributes from model
if(!this.model.isNew()){
_.each(this.model.attributes.configs, function(value, name){
[11/28] ranger git commit: RANGER-2180: Handle token replacement
correctly when token is not defined in the request context
Posted by me...@apache.org.
RANGER-2180: Handle token replacement correctly when token is not defined in the request context
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/87fccb54
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/87fccb54
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/87fccb54
Branch: refs/heads/ranger-1.1
Commit: 87fccb548a7c8499ca8d71bc6f306a4c93dd2f0a
Parents: ae728a7
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Thu Aug 2 08:26:28 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:35:37 2018 +0530
----------------------------------------------------------------------
.../org/apache/ranger/plugin/util/StringTokenReplacer.java | 6 ++----
.../resourcematcher/test_resourcematcher_dynamic.json | 4 ++--
.../test_resourcematcher_wildcards_as_delimiters.json | 2 +-
3 files changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/87fccb54/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java
index 2ec809c..ace04d6 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/StringTokenReplacer.java
@@ -73,9 +73,7 @@ public class StringTokenReplacer {
if (replaced != null) {
ret.append(replaced.toString());
} else {
- ret = null;
- token = null;
- break;
+ ret.append(startChar).append(token).append(endChar);
}
} else {
ret.append(startChar).append(token).append(endChar);
@@ -91,6 +89,6 @@ public class StringTokenReplacer {
ret.append(startChar).append(token);
}
- return ret != null ? ret.toString() : null;
+ return ret.toString();
}
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/87fccb54/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json
----------------------------------------------------------------------
diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json
index 5237d47..db19f76 100644
--- a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json
+++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_dynamic.json
@@ -15,7 +15,7 @@
,
{ "name":"exact-path","input":"/abcw", "evalContext": {"token:somestuff": "somethingelse"}, "result":false}
,
- { "name":"exact-path","input":"/abc%xyz%w", "evalContext": {"token:somestuff": "somethingelse"}, "result":false}
+ { "name":"exact-path","input":"/abc%xyz%w", "evalContext": {"token:somestuff": "somethingelse"}, "result":true}
,
{ "name":"exact-path","input":"/abc%xyz%w", "evalContext": {"token:somestuff": "somethingelse", "token:xyz":"abcd"}, "result":false}
,
@@ -39,7 +39,7 @@
,
{ "name":"hdfs-agent-test-11","input":"/tmp/tmpdir4/", "evalContext": {"token:FILENAME": ""}, "result":true}
,
- { "name":"hdfs-agent-test-12","input":"/tmp/tmpdir4/%FILENAME%", "evalContext": {"token:USER": "admin"}, "result":false}
+ { "name":"hdfs-agent-test-12","input":"/tmp/tmpdir4/%FILENAME%", "evalContext": {"token:USER": "admin"}, "result":true}
,
{ "name":"hdfs-agent-test-13","input":"/tmp/tmpdir5/data-file.txt", "evalContext": {"token:BASE_FILENAME": "data-file"}, "result":true}
,
http://git-wip-us.apache.org/repos/asf/ranger/blob/87fccb54/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json
----------------------------------------------------------------------
diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json
index c907f41..8e791d6 100644
--- a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json
+++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_wildcards_as_delimiters.json
@@ -17,7 +17,7 @@
,
{ "name":"exact-path","input":"/abcw", "evalContext": {"token:somestuff": "somethingelse"}, "result":false}
,
- { "name":"exact-path","input":"/abc*xyz?w", "evalContext": {"token:somestuff": "somethingelse"}, "result":false}
+ { "name":"exact-path","input":"/abc*xyz?w", "evalContext": {"token:somestuff": "somethingelse"}, "result":true}
,
{ "name":"exact-path","input":"/abcabcdw", "evalContext": {"token:somestuff": "somethingelse", "token:xyz":"abcd"}, "result":true}
,
[16/28] ranger git commit: RANGER-2183: Use INodeAttribute
information to authorize HDFS access
Posted by me...@apache.org.
RANGER-2183: Use INodeAttribute information to authorize HDFS access
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/4f7e4cbc
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/4f7e4cbc
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/4f7e4cbc
Branch: refs/heads/ranger-1.1
Commit: 4f7e4cbc853ee782fca08b32b4eb679503d8c273
Parents: a2de0eb
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Thu Aug 9 14:08:11 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:39:52 2018 +0530
----------------------------------------------------------------------
.../hadoop/RangerHdfsAuthorizer.java | 97 ++++++++++++++------
1 file changed, 69 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/4f7e4cbc/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
----------------------------------------------------------------------
diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
index 3697b8c..cccc759 100644
--- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
+++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java
@@ -20,6 +20,7 @@
package org.apache.ranger.authorization.hadoop;
import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.EXECUTE_ACCCESS_TYPE;
+import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH;
import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.READ_ACCCESS_TYPE;
import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.WRITE_ACCCESS_TYPE;
@@ -37,6 +38,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
@@ -196,6 +198,16 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
}
}
+ class SubAccessData {
+ final INodeDirectory dir;
+ final String resourcePath;
+
+ SubAccessData(INodeDirectory dir, String resourcePath) {
+ this.dir = dir;
+ this.resourcePath = resourcePath;
+ }
+ }
+
@Override
public void checkPermission(String fsOwner, String superGroup, UserGroupInformation ugi,
INodeAttributes[] inodeAttrs, INode[] inodes, byte[][] pathByNameArr,
@@ -207,11 +219,12 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
RangerHdfsAuditHandler auditHandler = null;
String user = ugi != null ? ugi.getShortUserName() : null;
Set<String> groups = ugi != null ? Sets.newHashSet(ugi.getGroupNames()) : null;
+ String resourcePath = path;
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerAccessControlEnforcer.checkPermission("
+ "fsOwner=" + fsOwner + "; superGroup=" + superGroup + ", inodesCount=" + (inodes != null ? inodes.length : 0)
- + ", snapshotId=" + snapshotId + ", user=" + user + ", path=" + path + ", ancestorIndex=" + ancestorIndex
+ + ", snapshotId=" + snapshotId + ", user=" + user + ", provided-path=" + path + ", ancestorIndex=" + ancestorIndex
+ ", doCheckOwner="+ doCheckOwner + ", ancestorAccess=" + ancestorAccess + ", parentAccess=" + parentAccess
+ ", access=" + access + ", subAccess=" + subAccess + ", ignoreEmptyDir=" + ignoreEmptyDir + ")");
}
@@ -219,7 +232,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_HDFSAUTH_REQUEST_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_HDFSAUTH_REQUEST_LOG, "RangerHdfsAuthorizer.checkPermission(path=" + path + ")");
+ perf = RangerPerfTracer.getPerfTracer(PERF_HDFSAUTH_REQUEST_LOG, "RangerHdfsAuthorizer.checkPermission(provided-path=" + path + ")");
}
try {
@@ -229,6 +242,29 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
INode inode = null;
if(plugin != null && !ArrayUtils.isEmpty(inodes)) {
+ int sz = inodeAttrs.length;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Size of INodeAttrs array:[" + sz + "]");
+ }
+ byte[][] components = new byte[sz][];
+
+ int i = 0;
+ for (; i < sz; i++) {
+ if (inodeAttrs[i] != null) {
+ components[i] = inodeAttrs[i].getLocalNameBytes();
+ } else {
+ break;
+ }
+ }
+ if (i != sz) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Input INodeAttributes array contains null at position " + i);
+ LOG.debug("Will use only first [" + i + "] components to build resourcePath");
+ }
+ }
+
+ resourcePath = DFSUtil.byteArray2PathString(components, 0, i);
+
if(ancestorIndex >= inodes.length) {
ancestorIndex = inodes.length - 1;
}
@@ -241,7 +277,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
parent = inodes.length > 1 ? inodes[inodes.length - 2] : null;
inode = inodes[inodes.length - 1]; // could be null while creating a new file
- auditHandler = new RangerHdfsAuditHandler(path, isTraverseOnlyCheck);
+ auditHandler = new RangerHdfsAuditHandler(resourcePath, isTraverseOnlyCheck);
/* Hadoop versions prior to 2.8.0 didn't ask for authorization of parent/ancestor traversal for
* reading or writing a file. However, Hadoop version 2.8.0 and later ask traversal authorization for
@@ -259,7 +295,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
* without requiring policy migration/update, for the changes in behaviour in Hadoop 2.8.0.
*/
if(isTraverseOnlyCheck) {
- authzStatus = traverseOnlyCheck(inode, inodeAttrs, parent, ancestor, ancestorIndex, user, groups, plugin, auditHandler);
+ authzStatus = traverseOnlyCheck(inode, inodeAttrs, resourcePath, components, parent, ancestor, ancestorIndex, user, groups, plugin, auditHandler);
}
// checkStickyBit
@@ -273,8 +309,9 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
// checkAncestorAccess
if(authzStatus == AuthzStatus.ALLOW && ancestorAccess != null && ancestor != null) {
INodeAttributes ancestorAttribs = inodeAttrs.length > ancestorIndex ? inodeAttrs[ancestorIndex] : null;
+ String ancestorPath = ancestorAttribs != null ? DFSUtil.byteArray2PathString(components, 0, ancestorIndex + 1) : null;
- authzStatus = isAccessAllowed(ancestor, ancestorAttribs, ancestorAccess, user, groups, plugin, auditHandler);
+ authzStatus = isAccessAllowed(ancestor, ancestorAttribs, ancestorPath, ancestorAccess, user, groups, plugin, auditHandler);
if (authzStatus == AuthzStatus.NOT_DETERMINED) {
authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes,
pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner,
@@ -286,8 +323,9 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
// checkParentAccess
if(authzStatus == AuthzStatus.ALLOW && parentAccess != null && parent != null) {
INodeAttributes parentAttribs = inodeAttrs.length > 1 ? inodeAttrs[inodeAttrs.length - 2] : null;
+ String parentPath = parentAttribs != null ? DFSUtil.byteArray2PathString(components, 0, inodeAttrs.length - 1) : null;
- authzStatus = isAccessAllowed(parent, parentAttribs, parentAccess, user, groups, plugin, auditHandler);
+ authzStatus = isAccessAllowed(parent, parentAttribs, parentPath, parentAccess, user, groups, plugin, auditHandler);
if (authzStatus == AuthzStatus.NOT_DETERMINED) {
authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes,
pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner,
@@ -300,7 +338,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
if(authzStatus == AuthzStatus.ALLOW && access != null && inode != null) {
INodeAttributes inodeAttribs = inodeAttrs.length > 0 ? inodeAttrs[inodeAttrs.length - 1] : null;
- authzStatus = isAccessAllowed(inode, inodeAttribs, access, user, groups, plugin, auditHandler);
+ authzStatus = isAccessAllowed(inode, inodeAttribs, resourcePath, access, user, groups, plugin, auditHandler);
if (authzStatus == AuthzStatus.NOT_DETERMINED) {
authzStatus = checkDefaultEnforcer(fsOwner, superGroup, ugi, inodeAttrs, inodes,
pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner,
@@ -311,16 +349,16 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
// checkSubAccess
if(authzStatus == AuthzStatus.ALLOW && subAccess != null && inode != null && inode.isDirectory()) {
- Stack<INodeDirectory> directories = new Stack<INodeDirectory>();
+ Stack<SubAccessData> directories = new Stack<>();
- for(directories.push(inode.asDirectory()); !directories.isEmpty(); ) {
- INodeDirectory dir = directories.pop();
- ReadOnlyList<INode> cList = dir.getChildrenList(snapshotId);
+ for(directories.push(new SubAccessData(inode.asDirectory(), resourcePath)); !directories.isEmpty(); ) {
+ SubAccessData data = directories.pop();
+ ReadOnlyList<INode> cList = data.dir.getChildrenList(snapshotId);
if (!(cList.isEmpty() && ignoreEmptyDir)) {
- INodeAttributes dirAttribs = dir.getSnapshotINode(snapshotId);
+ INodeAttributes dirAttribs = data.dir.getSnapshotINode(snapshotId);
- authzStatus = isAccessAllowed(dir, dirAttribs, subAccess, user, groups, plugin, auditHandler);
+ authzStatus = isAccessAllowed(data.dir, dirAttribs, data.resourcePath, subAccess, user, groups, plugin, auditHandler);
if(authzStatus != AuthzStatus.ALLOW) {
break;
@@ -331,13 +369,13 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
boolean optimizeSubAccessAuthEnabled = RangerHdfsPlugin.isOptimizeSubAccessAuthEnabled();
if (optimizeSubAccessAuthEnabled) {
- subDirAuthStatus = isAccessAllowedForHierarchy(dir, dirAttribs, subAccess, user, groups, plugin);
+ subDirAuthStatus = isAccessAllowedForHierarchy(data.dir, dirAttribs, data.resourcePath, subAccess, user, groups, plugin);
}
if (subDirAuthStatus != AuthzStatus.ALLOW) {
for(INode child : cList) {
if (child.isDirectory()) {
- directories.push(child.asDirectory());
+ directories.push(new SubAccessData(child.asDirectory(), resourcePath + org.apache.hadoop.fs.Path.SEPARATOR_CHAR + child.getLocalName()));
}
}
}
@@ -382,7 +420,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
}
}
- throw new RangerAccessControlException("Permission denied: user=" + user + ", access=" + action + ", inode=\"" + path + "\"");
+ throw new RangerAccessControlException("Permission denied: user=" + user + ", access=" + action + ", inode=\"" + resourcePath + "\"");
}
} finally {
if(auditHandler != null) {
@@ -392,7 +430,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
RangerPerfTracer.log(perf);
if(LOG.isDebugEnabled()) {
- LOG.debug("<== RangerAccessControlEnforcer.checkPermission(" + path + ", " + access + ", user=" + user + ") : " + authzStatus);
+ LOG.debug("<== RangerAccessControlEnforcer.checkPermission(" + resourcePath + ", " + access + ", user=" + user + ") : " + authzStatus);
}
}
}
@@ -401,11 +439,9 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
Check if parent or ancestor of the file being accessed is denied EXECUTE permission. If not, assume that Ranger-acls
allowed EXECUTE access. Do not audit this authorization check if resource is a file unless access is explicitly denied
*/
- private AuthzStatus traverseOnlyCheck(INode inode, INodeAttributes[] inodeAttrs, INode parent, INode ancestor, int ancestorIndex,
+ private AuthzStatus traverseOnlyCheck(INode inode, INodeAttributes[] inodeAttrs, String path, byte[][] components, INode parent, INode ancestor, int ancestorIndex,
String user, Set<String> groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler) {
- String path = inode != null ? inode.getFullPathName() : null;
-
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerAccessControlEnforcer.traverseOnlyCheck("
+ "path=" + path + ", user=" + user + ", groups=" + groups + ")");
@@ -416,32 +452,39 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
INodeAttributes nodeAttribs = inodeAttrs.length > 0 ? inodeAttrs[inodeAttrs.length - 1] : null;
boolean skipAuditOnAllow = false;
+ String resourcePath = path;
if (nodeToCheck == null || nodeToCheck.isFile()) {
skipAuditOnAllow = true;
if (parent != null) {
nodeToCheck = parent;
nodeAttribs = inodeAttrs.length > 1 ? inodeAttrs[inodeAttrs.length - 2] : null;
+ resourcePath = inodeAttrs.length > 0 ? DFSUtil.byteArray2PathString(components, 0, inodeAttrs.length - 1) : HDFS_ROOT_FOLDER_PATH;
} else if (ancestor != null) {
nodeToCheck = ancestor;
nodeAttribs = inodeAttrs.length > ancestorIndex ? inodeAttrs[ancestorIndex] : null;
+ resourcePath = nodeAttribs != null ? DFSUtil.byteArray2PathString(components, 0, ancestorIndex+1) : HDFS_ROOT_FOLDER_PATH;
}
}
if (nodeToCheck != null) {
- ret = isAccessAllowedForTraversal(nodeToCheck, nodeAttribs, user, groups, plugin, auditHandler, skipAuditOnAllow);
+ if (resourcePath.length() > 1) {
+ if (resourcePath.endsWith(HDFS_ROOT_FOLDER_PATH)) {
+ resourcePath = resourcePath.substring(0, resourcePath.length()-1);
+ }
+ }
+ ret = isAccessAllowedForTraversal(nodeToCheck, nodeAttribs, resourcePath, user, groups, plugin, auditHandler, skipAuditOnAllow);
} else {
ret = AuthzStatus.ALLOW;
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== RangerAccessControlEnforcer.traverseOnlyCheck("
- + "path=" + path + ", user=" + user + ", groups=" + groups + ") : " + ret);
+ + "path=" + path + ", resourcePath=" + resourcePath + ", user=" + user + ", groups=" + groups + ") : " + ret);
}
return ret;
}
- private AuthzStatus isAccessAllowedForTraversal(INode inode, INodeAttributes inodeAttribs, String user, Set<String> groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler, boolean skipAuditOnAllow) {
+ private AuthzStatus isAccessAllowedForTraversal(INode inode, INodeAttributes inodeAttribs, String path, String user, Set<String> groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler, boolean skipAuditOnAllow) {
final AuthzStatus ret;
- String path = inode.getFullPathName();
String pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null;
String clusterName = plugin.getClusterName();
FsAction access = FsAction.EXECUTE;
@@ -559,9 +602,8 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
return authzStatus;
}
- private AuthzStatus isAccessAllowed(INode inode, INodeAttributes inodeAttribs, FsAction access, String user, Set<String> groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler) {
+ private AuthzStatus isAccessAllowed(INode inode, INodeAttributes inodeAttribs, String path, FsAction access, String user, Set<String> groups, RangerHdfsPlugin plugin, RangerHdfsAuditHandler auditHandler) {
AuthzStatus ret = null;
- String path = inode != null ? inode.getFullPathName() : null;
String pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null;
String clusterName = plugin.getClusterName();
@@ -614,9 +656,8 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider {
return ret;
}
- private AuthzStatus isAccessAllowedForHierarchy(INode inode, INodeAttributes inodeAttribs, FsAction access, String user, Set<String> groups, RangerHdfsPlugin plugin) {
+ private AuthzStatus isAccessAllowedForHierarchy(INode inode, INodeAttributes inodeAttribs, String path, FsAction access, String user, Set<String> groups, RangerHdfsPlugin plugin) {
AuthzStatus ret = null;
- String path = inode != null ? inode.getFullPathName() : null;
String pathOwner = inodeAttribs != null ? inodeAttribs.getUserName() : null;
String clusterName = plugin.getClusterName();
[20/28] ranger git commit: RANGER-2165: Address JPA Cache issue when
policies Create,
Update and Delete are done via REST API in Apache Ranger admin
Posted by me...@apache.org.
RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/6dcf1a90
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/6dcf1a90
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/6dcf1a90
Branch: refs/heads/ranger-1.1
Commit: 6dcf1a90c4df682d8a4c9ace07a5345760610405
Parents: e37ca0b
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Aug 14 10:49:34 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:42:23 2018 +0530
----------------------------------------------------------------------
.../main/java/org/apache/ranger/biz/RangerPolicyRetriever.java | 1 +
.../src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/6dcf1a90/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index 3e291d5..efbdfdf 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -130,6 +130,7 @@ public class RangerPolicyRetriever {
@Override
public void run() {
try {
+ txTemplate.setReadOnly(true);
policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>() {
@Override
public List<RangerPolicy> doInTransaction(TransactionStatus status) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/6dcf1a90/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index 789068f..e5e9572 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -263,7 +263,8 @@ public class RangerTagDBRetriever {
@Override
public void run() {
try {
- Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
+ txTemplate.setReadOnly(true);
+ Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
boolean ret = initializeTagCache(xService);
[28/28] ranger git commit: RANGER-2204: Fixed issue where audit log
values for policy service are not processed properly
Posted by me...@apache.org.
RANGER-2204: Fixed issue where audit log values for policy service are not processed properly
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/8e19e67e
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/8e19e67e
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/8e19e67e
Branch: refs/heads/ranger-1.1
Commit: 8e19e67e20cdf9757df2d46a8fae30602376106e
Parents: d4b5722
Author: Sailaja Polavarapu <sp...@hortonworks.com>
Authored: Fri Aug 31 10:02:23 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:51:47 2018 +0530
----------------------------------------------------------------------
.../ranger/service/RangerPolicyService.java | 43 +++++++++++++-------
1 file changed, 28 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/8e19e67e/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java
index ad3984d..08baf89 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyService.java
@@ -147,7 +147,7 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
}
List<XXTrxLog> trxLogList = new ArrayList<XXTrxLog>();
Field[] fields = vObj.getClass().getDeclaredFields();
-
+
try {
Field nameField = vObj.getClass().getDeclaredField("name");
@@ -236,14 +236,16 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
}
} else if (ROWFILTER_POLICY_ITEM_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
value = processRowFilterPolicyItemForTrxLog(field.get(vObj));
- } else if (IS_ENABLED_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = String.valueOf(processIsEnabledClassFieldNameForTrxLog(field.get(vObj)));
- } else if (POLICY_LABELS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processPolicyLabelsClassFieldNameForTrxLog(field.get(vObj));
- } else if (POLICY_VALIDITYSCHEDULES_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- value = processValiditySchedulesClassFieldNameForTrxLog(field.get(vObj));
- } else if (POLICY_PRIORITY_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ } else if (IS_ENABLED_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processIsEnabledClassFieldNameForTrxLog(field.get(vObj));
+ } else if (POLICY_LABELS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processPolicyLabelsClassFieldNameForTrxLog(field.get(vObj));
+ } else if (POLICY_VALIDITYSCHEDULES_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processValiditySchedulesClassFieldNameForTrxLog(field.get(vObj));
+ } else if (POLICY_PRIORITY_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
value = processPriorityClassFieldNameForTrxLog(field.get(vObj));
+ } else if (IS_AUDIT_ENABLED_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ value = processIsAuditEnabledClassFieldNameForTrxLog(field.get(vObj));
} else {
value = "" + field.get(vObj);
}
@@ -330,9 +332,13 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
}
}else if (IS_ENABLED_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
if (oldPolicy != null) {
- oldValue = String.valueOf(processIsEnabledClassFieldNameForTrxLog(oldPolicy.getIsEnabled()));
+ oldValue = processIsEnabledClassFieldNameForTrxLog(oldPolicy.getIsEnabled());
}
- } else if (POLICY_LABELS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ } else if (IS_AUDIT_ENABLED_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ if (oldPolicy != null) {
+ oldValue = processIsAuditEnabledClassFieldNameForTrxLog(oldPolicy.getIsAuditEnabled());
+ }
+ }else if (POLICY_LABELS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
oldValue = processPolicyLabelsClassFieldNameForTrxLog(oldPolicy.getPolicyLabels());
} else if (POLICY_VALIDITYSCHEDULES_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
oldValue = processValiditySchedulesClassFieldNameForTrxLog(oldPolicy.getValiditySchedules());
@@ -399,11 +405,11 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
return null;
}
} else if (IS_AUDIT_ENABLED_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- if (compareTwoPolicyName(value, oldValue)) {
- return null;
- }
- } else if (POLICY_LABELS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
- if (compareTwoPolicyLabelList(value, oldValue)) {
+ if (compareTwoPolicyName(value, oldValue)) {
+ return null;
+ }
+ } else if (POLICY_LABELS_CLASS_FIELD_NAME.equalsIgnoreCase(fieldName)) {
+ if (compareTwoPolicyLabelList(value, oldValue)) {
return null;
}
}
@@ -664,6 +670,13 @@ public class RangerPolicyService extends RangerPolicyServiceBase<XXPolicy, Range
return isEnabled;
}
+ private String processIsAuditEnabledClassFieldNameForTrxLog(Object value) {
+ if(value == null)
+ return null;
+ String isAuditEnabled = String.valueOf(value);
+ return isAuditEnabled;
+ }
+
private boolean compareTwoDataMaskingPolicyItemList(String value, String oldValue) {
if (value == null && oldValue == null) {
return true;
[23/28] ranger git commit: RANGER-2191: Update ranger-tool with new
options to control Trie
Posted by me...@apache.org.
RANGER-2191: Update ranger-tool with new options to control Trie
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/383864c3
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/383864c3
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/383864c3
Branch: refs/heads/ranger-1.1
Commit: 383864c3ac5b0f6c013b39d04604d78895ead7bb
Parents: fbcaaaf
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Thu Aug 16 23:53:39 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:48:40 2018 +0530
----------------------------------------------------------------------
.../RangerDefaultPolicyEvaluator.java | 4 +--
.../ranger/plugin/util/RangerResourceTrie.java | 20 +++++++------
ranger-tools/scripts/README.txt | 9 ++++--
.../ranger/policyengine/CommandLineParser.java | 30 ++++++++++++++++++--
.../ranger/policyengine/PerfTestOptions.java | 14 ++++++++-
.../RangerPolicyenginePerfTester.java | 9 ++++++
ranger-tools/src/test/resources/commandline | 2 +-
.../test/resources/testdata/ranger-config.xml | 18 ++++++++++++
ranger-tools/testdata/ranger-config.xml | 18 ++++++++++++
9 files changed, 106 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
index 2a5ee54..289ec9b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
@@ -207,7 +207,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
@Override
public void evaluate(RangerAccessRequest request, RangerAccessResult result) {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")");
+ LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")");
}
RangerPerfTracer perf = null;
@@ -257,7 +257,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
RangerPerfTracer.log(perf);
if(LOG.isDebugEnabled()) {
- LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")");
+ LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(policyId=" + getPolicy().getId() + ", " + request + ", " + result + ")");
}
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index e702684..9c7c775 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -68,7 +68,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + ")");
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + ")");
}
int builderThreadCount = RangerConfiguration.getInstance().getInt(TRIE_BUILDER_THREAD_COUNT, 1);
@@ -154,6 +154,12 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
LOG.debug("==> buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + (builderThreadCount > 1) + ")");
}
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(resourceDef=" + resourceDef.getName() + ")");
+ }
+
TrieNode<T> ret = new TrieNode<>(null);
final boolean isMultiThreaded = builderThreadCount > 1;
final List<ResourceTrieBuilderThread> builderThreads;
@@ -243,7 +249,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
RangerPerfTracer postSetupPerf = null;
if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(name=" + resourceDef.getName() + "-postSetup)");
+ postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(name=" + resourceDef.getName() + "-postSetup)");
}
ret.postSetup(null, comparator);
@@ -259,6 +265,8 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
cleanUpThreads(builderThreads);
}
+ RangerPerfTracer.logAlways(perf);
+
if(LOG.isDebugEnabled()) {
LOG.debug("<== buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + isMultiThreaded + ") :" + ret);
}
@@ -324,11 +332,6 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
private void insert(TrieNode<T> currentRoot, String resource, boolean isRecursive, T evaluator) {
- RangerPerfTracer perf = null;
-
- if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.insert(resource=" + resource + ")");
- }
TrieNode<T> curr = currentRoot;
final String prefix = getNonWildcardPrefix(resource);
@@ -344,7 +347,6 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
curr.addEvaluator(evaluator);
}
- RangerPerfTracer.logAlways(perf);
}
private String getNonWildcardPrefix(String str) {
@@ -559,7 +561,7 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
RangerPerfTracer postSetupPerf = null;
if (RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie(thread=" + this.getName() + "-postSetup)");
+ postSetupPerf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.init(thread=" + this.getName() + "-postSetup)");
}
thisRoot.postSetup(parentWildcardEvaluators, comparator);
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/scripts/README.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/scripts/README.txt b/ranger-tools/scripts/README.txt
index 55170d5..81b5b66 100644
--- a/ranger-tools/scripts/README.txt
+++ b/ranger-tools/scripts/README.txt
@@ -51,6 +51,8 @@ This file describes how to build, setup, configure and run the performance testi
tag-definitions, and service-resources with their associated tags;
testdata/test_requests_hive.json - Contains access requests to be made to the policy-engine;
+
+ testdata/ranger-config.xml - Contains any required Ranger configuration variables
Please review the contents of these files and modify to suit your profiling needs.
@@ -58,10 +60,13 @@ This file describes how to build, setup, configure and run the performance testi
6. Run the tool with the following command
- % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run>
+ % ./ranger-perftester.sh -s <service-policies-file> -r <requests-file> -c <number-of-concurrent-clients> -n <number-of-times-requests-file-to-be-run> -t -d -f <ranger-configuration-file> -p <test-modules-file>
+
+ where, -t indicates enabling Trie,
+ -d indicates enabling lazy post-setup of Trie structure,
Example:
- % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1
+ % ./ranger-perftester.sh -s testdata/test_servicepolicies_hive.json -r testdata/test_requests_hive.json -c 2 -n 1 -t -d -f testdata/ranger-config.xml -p testdata/test_modules.txt
7. At the end of the run, the performance-statistics are printed on the console and in the log specified file in conf/log4j.properties file as shown below. This is for time spent in evaluating access by Ranger Policy Engine during the course of a test run. The time values shown are in milliseconds.
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
index e8edd9e..fe9ec17 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
@@ -55,6 +55,12 @@ public class CommandLineParser
private boolean isDynamicReorderingDisabled = true;
private boolean isTrieLookupPrefixDisabled = true;
+ private boolean isLazyTriePostSetupDisabled = true;
+
+ private String configurationFileName;
+ private URL configurationFileURL;
+
+
private Options options = new Options();
CommandLineParser() {}
@@ -63,7 +69,7 @@ public class CommandLineParser
PerfTestOptions ret = null;
if (parseArguments(args) && validateInputFiles()) {
// Instantiate a data-object and return
- ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingDisabled, isTrieLookupPrefixDisabled);
+ ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingDisabled, isTrieLookupPrefixDisabled, isLazyTriePostSetupDisabled, configurationFileURL);
} else {
showUsage();
}
@@ -78,7 +84,10 @@ public class CommandLineParser
-r request-file-name-list
-n number-of-iterations
-p modules-to-collect-stats
+ -f configuration-file-name
-o
+ -t
+ -d
If the concurrent-client-count is more than the number of files in the request-file-name-list,
then reuse the request-file-names in a round-robin way
@@ -98,8 +107,11 @@ public class CommandLineParser
options.addOption("p", "statistics", true, "Modules for stat collection File Name");
options.addOption("c", "clients", true, "Number of concurrent clients");
options.addOption("n", "cycles", true, "Number of iterations");
+ options.addOption("f", "configurations", true, "Configuration File Name");
options.addOption("o", "optimize", false, "Enable usage-based policy reordering");
options.addOption("t", "trie-prefilter", false, "Enable trie-prefilter");
+ options.addOption("d", "trie-lazy-setup", false, "Enable lazy trie-setup");
+
org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser();
@@ -133,11 +145,20 @@ public class CommandLineParser
isTrieLookupPrefixDisabled = false;
}
+ if (commandLine.hasOption("d")) {
+ isLazyTriePostSetupDisabled = false;
+ }
+
+ configurationFileName = commandLine.getOptionValue("f");
+
if (LOG.isDebugEnabled()) {
LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames));
LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount);
LOG.debug("isDynamicReorderingDisabled=" + isDynamicReorderingDisabled);
LOG.debug("isTrieLookupPrefixDisabled=" + isTrieLookupPrefixDisabled);
+ LOG.debug("isLazyTriePostSetupDisabled=" + isLazyTriePostSetupDisabled);
+ LOG.debug("configurationFileName=" + configurationFileName);
+
}
ret = true;
@@ -164,11 +185,14 @@ public class CommandLineParser
if (servicePoliciesFileURL != null) {
if (requestFileNames != null) {
if (validateRequestFiles()) {
+ ret = true;
if (statCollectionFileName != null) {
statCollectionFileURL = getInputFileURL(statCollectionFileName);
ret = statCollectionFileURL != null;
- } else {
- ret = true;
+ }
+ if (ret && configurationFileName != null) {
+ configurationFileURL = getInputFileURL(configurationFileName);
+ ret = configurationFileURL != null;
}
}
} else {
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
index d6e04ea..1486f52 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
@@ -28,12 +28,15 @@ public class PerfTestOptions {
private final URL statCollectionFileURL;
private final boolean isDynamicReorderingDisabled;
private final boolean isTrieLookupPrefixDisabled;
+ private final boolean isOnDemandTriePostSetupDisabled;
private final int concurrentClientCount;
private final int iterationsCount;
+ private final URL perfConfigurationFileURL;
- PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingDisabled, boolean isTrieLookupPrefixDisabled) {
+
+ PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingDisabled, boolean isTrieLookupPrefixDisabled, boolean isOnDemandTriePostSetupDisabled, URL perfConfigurationFileURL) {
this.servicePoliciesFileURL = servicePoliciesFileURL;
this.requestFileURLs = requestFileURLs;
this.statCollectionFileURL = statCollectionFileURL;
@@ -41,6 +44,8 @@ public class PerfTestOptions {
this.concurrentClientCount = concurrentClientCount;
this.isDynamicReorderingDisabled = isDynamicReorderingDisabled;
this.isTrieLookupPrefixDisabled = isTrieLookupPrefixDisabled;
+ this.isOnDemandTriePostSetupDisabled = isOnDemandTriePostSetupDisabled;
+ this.perfConfigurationFileURL = perfConfigurationFileURL;
}
public URL getServicePoliciesFileURL() {
@@ -66,4 +71,11 @@ public class PerfTestOptions {
public boolean getIsDynamicReorderingDisabled() { return isDynamicReorderingDisabled; }
public boolean getIsTrieLookupPrefixDisabled() { return isTrieLookupPrefixDisabled; }
+
+ public boolean getIsOnDemandTriePostSetupDisabled() { return isOnDemandTriePostSetupDisabled; }
+
+ public URL getPerfConfigurationFileURL() {
+ return this.perfConfigurationFileURL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
index 78cbe02..53e5cf0 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
@@ -21,6 +21,7 @@ package org.apache.ranger.policyengine;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import org.apache.ranger.plugin.util.PerfDataRecorder;
@@ -61,6 +62,14 @@ public class RangerPolicyenginePerfTester {
policyEngineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED;
policyEngineOptions.cacheAuditResults = false;
policyEngineOptions.disableTrieLookupPrefilter = perfTestOptions.getIsTrieLookupPrefixDisabled();
+ policyEngineOptions.optimizeTrieForRetrieval = perfTestOptions.getIsOnDemandTriePostSetupDisabled();
+
+ URL configurationFileURL = perfTestOptions.getPerfConfigurationFileURL();
+
+ if (configurationFileURL != null) {
+ RangerConfiguration config = RangerConfiguration.getInstance();
+ config.addResource(configurationFileURL);
+ }
PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions, perfTestOptions.getIsDynamicReorderingDisabled());
if (!perfTestEngine.init()) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/test/resources/commandline
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/commandline b/ranger-tools/src/test/resources/commandline
index 9ea690e..73338d5 100644
--- a/ranger-tools/src/test/resources/commandline
+++ b/ranger-tools/src/test/resources/commandline
@@ -17,4 +17,4 @@
# under the License.
#
--s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1
+-s /testdata/test_servicepolicies_hive.json -r /testdata/test_requests_hive.json -p /testdata/test_modules.txt -c 3 -n 1 -t -d -f /testdata/ranger-config.xml
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/src/test/resources/testdata/ranger-config.xml
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/testdata/ranger-config.xml b/ranger-tools/src/test/resources/testdata/ranger-config.xml
new file mode 100644
index 0000000..933d670
--- /dev/null
+++ b/ranger-tools/src/test/resources/testdata/ranger-config.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<configuration>
+ <property>
+ <name>ranger.policyengine.trie.builder.thread.count</name>
+ <value>1</value>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ranger/blob/383864c3/ranger-tools/testdata/ranger-config.xml
----------------------------------------------------------------------
diff --git a/ranger-tools/testdata/ranger-config.xml b/ranger-tools/testdata/ranger-config.xml
new file mode 100644
index 0000000..933d670
--- /dev/null
+++ b/ranger-tools/testdata/ranger-config.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<configuration>
+ <property>
+ <name>ranger.policyengine.trie.builder.thread.count</name>
+ <value>1</value>
+ </property>
+</configuration>
\ No newline at end of file
[10/28] ranger git commit: RANGER-2166: A ClassNotFound exception is
thrown with atlasrest as a tag source
Posted by me...@apache.org.
RANGER-2166: A ClassNotFound exception is thrown with atlasrest as a tag source
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/ae728a72
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/ae728a72
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/ae728a72
Branch: refs/heads/ranger-1.1
Commit: ae728a72066fe0ddfa65f77709ab4b470f01e46d
Parents: e902bef
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Wed Aug 1 08:31:41 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:35:03 2018 +0530
----------------------------------------------------------------------
src/main/assembly/tagsync.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/ae728a72/src/main/assembly/tagsync.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml
index 05a6739..d1b83df 100644
--- a/src/main/assembly/tagsync.xml
+++ b/src/main/assembly/tagsync.xml
@@ -60,7 +60,7 @@
<include>com.fasterxml.jackson.core:jackson-databind:jar:${atlas.jackson.version}</include>
<include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${atlas.jackson.version}</include>
<include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${atlas.jackson.version}</include>
- <include>com.fasterxml.jackson.jaxrs:jackson-module-jaxb-annotations:jar:${atlas.jackson.version}</include>
+ <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:${atlas.jackson.version}</include>
<include>org.codehaus.jackson:jackson-core-asl:jar:${codehaus.jackson.version}</include>
<include>org.codehaus.jackson:jackson-mapper-asl:jar:${codehaus.jackson.version}</include>
<include>org.codehaus.jackson:jackson-jaxrs:jar:${codehaus.jackson.version}</include>
[02/28] ranger git commit: RANGER-2147: Updated year in NOTICE.txt
Posted by me...@apache.org.
RANGER-2147: Updated year in NOTICE.txt
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/6eff731a
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/6eff731a
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/6eff731a
Branch: refs/heads/ranger-1.1
Commit: 6eff731ad38186250c1a9dd281a89bf86a8cf4ba
Parents: 7eb7839
Author: Velmurugan Periasamy <ve...@apache.org>
Authored: Thu Jul 19 14:39:43 2018 -0400
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:16:00 2018 +0530
----------------------------------------------------------------------
NOTICE.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/6eff731a/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 64a9599..4a9bf3e 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
Apache Ranger
-Copyright 2014-2017 The Apache Software Foundation
+Copyright 2014-2018 The Apache Software Foundation
This product includes software developed at The Apache Software Foundation (http://www.apache.org/).
This product includes software developed by Spring Security Project (http://www.springframework.org/security)
[12/28] ranger git commit: RANGER-2153 Supply the function of
reverting policy history version.
Posted by me...@apache.org.
RANGER-2153 Supply the function of reverting policy history version.
Signed-off-by: pengjianhua <35...@qq.com>
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/be3dc31a
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/be3dc31a
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/be3dc31a
Branch: refs/heads/ranger-1.1
Commit: be3dc31a4f64f7d00077850af8e75479ad551b9b
Parents: 87fccb5
Author: zhangqiang2 <zh...@zte.com.cn>
Authored: Thu Aug 2 15:33:55 2018 +0800
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:36:01 2018 +0530
----------------------------------------------------------------------
.../src/main/webapp/scripts/views/policies/RangerPolicyRO.js | 8 ++++++--
.../webapp/scripts/views/policies/RangerPolicyTableLayout.js | 2 +-
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/be3dc31a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js
----------------------------------------------------------------------
diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js
index 1e3554f..3a6a59e 100644
--- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js
+++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyRO.js
@@ -287,7 +287,7 @@ define(function(require) {
}
},
- revert : function(e){
+ revert : function(e, collection, modal){
e.preventDefault();
var policyId = this.policy.attributes.id;
var rangerPolicy = new RangerPolicy({id : policyId});
@@ -297,7 +297,11 @@ define(function(require) {
success: function() {
XAUtil.blockUI('unblock');
XAUtil.notifySuccess('Success', 'Policy reverted successfully.');
- location.reload();
+ collection.fetch({
+ cache : false,
+ reset : true
+ });
+ modal.close();
},
error: function(model, response, options) {
XAUtil.blockUI('unblock');
http://git-wip-us.apache.org/repos/asf/ranger/blob/be3dc31a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js
----------------------------------------------------------------------
diff --git a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js
index 83253e4..1150838 100644
--- a/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js
+++ b/security-admin/src/main/webapp/scripts/views/policies/RangerPolicyTableLayout.js
@@ -203,7 +203,7 @@ define(function(require){
view.nextVer(e);
});
policyVerEl.after('<a id="revert" href="#" class="btn btn-primary" style="display:none;">Revert</a>').next('#revert').click(function(e){
- view.revert(e);
+ view.revert(e, that.collection, modal);
});
modal.$el.find('.cancel').hide();
});
[03/28] ranger git commit: RANGER-2152: Incorrect debugging
information in RangerPluginClassLoader.java
Posted by me...@apache.org.
RANGER-2152: Incorrect debugging information in RangerPluginClassLoader.java
Signed-off-by: zhangqiang2 <zh...@zte.com.cn>
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/bfc1f190
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/bfc1f190
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/bfc1f190
Branch: refs/heads/ranger-1.1
Commit: bfc1f19038f2082903831784eac4ca33c750890c
Parents: 6eff731
Author: xingpeng1 <xi...@zte.com.cn>
Authored: Mon Jul 23 15:13:20 2018 +0800
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:18:28 2018 +0530
----------------------------------------------------------------------
.../apache/ranger/plugin/classloader/RangerPluginClassLoader.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/bfc1f190/ranger-plugin-classloader/src/main/java/org/apache/ranger/plugin/classloader/RangerPluginClassLoader.java
----------------------------------------------------------------------
diff --git a/ranger-plugin-classloader/src/main/java/org/apache/ranger/plugin/classloader/RangerPluginClassLoader.java b/ranger-plugin-classloader/src/main/java/org/apache/ranger/plugin/classloader/RangerPluginClassLoader.java
index 2855560..41f53c0 100644
--- a/ranger-plugin-classloader/src/main/java/org/apache/ranger/plugin/classloader/RangerPluginClassLoader.java
+++ b/ranger-plugin-classloader/src/main/java/org/apache/ranger/plugin/classloader/RangerPluginClassLoader.java
@@ -127,7 +127,7 @@ public class RangerPluginClassLoader extends URLClassLoader {
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== RangerPluginClassLoader.loadClass" + name + "): " + ret);
+ LOG.debug("<== RangerPluginClassLoader.loadClass(" + name + "): " + ret);
}
return ret;
[07/28] ranger git commit: RANGER-2165: Address JPA Cache issue when
policies Create,
Update and Delete are done via REST API in Apache Ranger admin
Posted by me...@apache.org.
RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/1a358571
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/1a358571
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/1a358571
Branch: refs/heads/ranger-1.1
Commit: 1a3585717f2c27b100a6dd3dda4198da4ee1ec33
Parents: 4a13ff7
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Jul 31 15:15:33 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:29:46 2018 +0530
----------------------------------------------------------------------
.../ranger/biz/RangerPolicyRetriever.java | 82 ++++++++++++++--
.../apache/ranger/biz/RangerTagDBRetriever.java | 99 ++++++++++++++++++--
.../org/apache/ranger/biz/ServiceDBStore.java | 2 +-
.../java/org/apache/ranger/biz/TagDBStore.java | 8 +-
4 files changed, 175 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
index 463957c..3e291d5 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java
@@ -47,17 +47,36 @@ import org.apache.ranger.plugin.model.RangerValiditySchedule;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator;
import org.apache.ranger.plugin.util.RangerPerfTracer;
import org.apache.ranger.service.RangerPolicyService;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
public class RangerPolicyRetriever {
static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class);
static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever");
- final RangerDaoManager daoMgr;
- final LookupCache lookupCache;
+ private final RangerDaoManager daoMgr;
+ private final LookupCache lookupCache = new LookupCache();
+
+ private final PlatformTransactionManager txManager;
+ private final TransactionTemplate txTemplate;
+
+ public RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) {
+ this.daoMgr = daoMgr;
+ this.txManager = txManager;
+ if (this.txManager != null) {
+ this.txTemplate = new TransactionTemplate(this.txManager);
+ this.txTemplate.setReadOnly(true);
+ } else {
+ this.txTemplate = null;
+ }
+ }
public RangerPolicyRetriever(RangerDaoManager daoMgr) {
this.daoMgr = daoMgr;
- this.lookupCache = new LookupCache();
+ this.txManager = null;
+ this.txTemplate = null;
}
public List<RangerPolicy> getServicePolicies(Long serviceId) {
@@ -96,7 +115,41 @@ public class RangerPolicyRetriever {
return ret;
}
- public List<RangerPolicy> getServicePolicies(XXService xService) {
+ private class PolicyLoaderThread extends Thread {
+ final TransactionTemplate txTemplate;
+ final XXService xService;
+ List<RangerPolicy> policies;
+
+ PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+ this.txTemplate = txTemplate;
+ this.xService = xService;
+ }
+
+ public List<RangerPolicy> getPolicies() { return policies; }
+
+ @Override
+ public void run() {
+ try {
+ policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>() {
+ @Override
+ public List<RangerPolicy> doInTransaction(TransactionStatus status) {
+ try {
+ RetrieverContext ctx = new RetrieverContext(xService);
+ return ctx.getAllPolicies();
+ } catch (Exception ex) {
+ LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex);
+ status.setRollbackOnly();
+ return null;
+ }
+ }
+ });
+ } catch (Throwable ex) {
+ LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex);
+ }
+ }
+ }
+
+ public List<RangerPolicy> getServicePolicies(final XXService xService) {
String serviceName = xService == null ? null : xService.getName();
Long serviceId = xService == null ? null : xService.getId();
@@ -112,9 +165,26 @@ public class RangerPolicyRetriever {
}
if(xService != null) {
- RetrieverContext ctx = new RetrieverContext(xService);
+ if (txTemplate == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction");
+ }
+ RetrieverContext ctx = new RetrieverContext(xService);
+ ret = ctx.getAllPolicies();
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Retrieving policies in a new, read-only transaction");
+ }
- ret = ctx.getAllPolicies();
+ PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService);
+ t.start();
+ try {
+ t.join();
+ ret = t.getPolicies();
+ } catch (InterruptedException ie) {
+ LOG.error("Failed to retrieve policies in a new, read-only thread.", ie);
+ }
+ }
} else {
if(LOG.isDebugEnabled()) {
LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter");
http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index 5550572..789068f 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -38,6 +38,10 @@ import org.apache.ranger.entity.*;
import org.apache.ranger.plugin.model.*;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallback;
+import org.springframework.transaction.support.TransactionTemplate;
public class RangerTagDBRetriever {
static final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class);
@@ -48,6 +52,9 @@ public class RangerTagDBRetriever {
private final XXService xService;
private final LookupCache lookupCache;
+ private final PlatformTransactionManager txManager;
+ private final TransactionTemplate txTemplate;
+
private List<RangerServiceResource> serviceResources;
private Map<Long, RangerTagDef> tagDefs;
private Map<Long, RangerTag> tags;
@@ -55,8 +62,15 @@ public class RangerTagDBRetriever {
private boolean filterForServicePlugin;
- public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xService) {
+ public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) {
this.daoMgr = daoMgr;
+ this.txManager = txManager;
+ if (this.txManager != null) {
+ this.txTemplate = new TransactionTemplate(this.txManager);
+ this.txTemplate.setReadOnly(true);
+ } else {
+ this.txTemplate = null;
+ }
this.xService = xService;
this.lookupCache = new LookupCache();
@@ -70,14 +84,27 @@ public class RangerTagDBRetriever {
}
filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false);
- TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService);
- TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService);
- TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService);
- serviceResources = serviceResourceContext.getAllServiceResources();
- tagDefs = tagDefContext.getAllTagDefs();
- tags = tagContext.getAllTags();
- tagResourceMaps = getAllTagResourceMaps();
+ if (this.txTemplate == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Load Tags in the same thread and using an existing transaction");
+ }
+ if (initializeTagCache(xService) == false) {
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction");
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Load Tags in a separate thread and using a new transaction");
+ }
+
+ TagLoaderThread t = new TagLoaderThread(txTemplate, xService);
+ t.start();
+ try {
+ t.join();
+ } catch (InterruptedException ie) {
+ LOG.error("Failed to get Tags in a separate thread and using a new transaction", ie);
+ }
+ }
RangerPerfTracer.log(perf);
@@ -100,6 +127,30 @@ public class RangerTagDBRetriever {
return tags;
}
+ private boolean initializeTagCache(XXService xService) {
+ boolean ret;
+ try {
+ TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService);
+ TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService);
+ TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService);
+
+ serviceResources = serviceResourceContext.getAllServiceResources();
+ tagDefs = tagDefContext.getAllTagDefs();
+ tags = tagContext.getAllTags();
+
+ tagResourceMaps = getAllTagResourceMaps();
+
+ ret = true;
+ } catch (Exception ex) {
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "]");
+ serviceResources = null;
+ tagDefs = null;
+ tags = null;
+ tagResourceMaps = null;
+ ret = false;
+ }
+ return ret;
+ }
private List<RangerTagResourceMap> getAllTagResourceMaps() {
List<XXTagResourceMap> xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId()) : daoMgr.getXXTagResourceMap().findByServiceId(xService.getId());
@@ -200,6 +251,38 @@ public class RangerTagDBRetriever {
}
}
+ private class TagLoaderThread extends Thread {
+ final TransactionTemplate txTemplate;
+ final XXService xService;
+
+ TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) {
+ this.txTemplate = txTemplate;
+ this.xService = xService;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() {
+ @Override
+ public Boolean doInTransaction(TransactionStatus status) {
+ boolean ret = initializeTagCache(xService);
+ if (!ret) {
+ status.setRollbackOnly();
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction");
+ }
+ return ret;
+ }
+ });
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("transaction result:[" + result +"]");
+ }
+ } catch (Throwable ex) {
+ LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction", ex);
+ }
+ }
+ }
+
private class TagRetrieverServiceResourceContext {
final XXService service;
http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index e75ea68..0773616 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -2462,7 +2462,7 @@ public class ServiceDBStore extends AbstractServiceStore {
LOG.debug("==> ServiceDBStore.getServicePoliciesFromDb(" + service.getName() + ")");
}
- RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr);
+ RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager);
List<RangerPolicy> ret = policyRetriever.getServicePolicies(service);
http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index 8341a73..d29df93 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -64,7 +64,9 @@ import org.apache.ranger.service.RangerTagResourceMapService;
import org.apache.ranger.service.RangerTagService;
import org.apache.ranger.service.RangerServiceResourceService;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletResponse;
@@ -89,6 +91,10 @@ public class TagDBStore extends AbstractTagStore {
RangerDaoManager daoManager;
@Autowired
+ @Qualifier(value = "transactionManager")
+ PlatformTransactionManager txManager;
+
+ @Autowired
RESTErrorUtil errorUtil;
@Autowired
@@ -991,7 +997,7 @@ public class TagDBStore extends AbstractTagStore {
throw new Exception("service-def does not exist. id=" + xxService.getType());
}
- RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService);
+ RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, txManager, xxService);
Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs();
Map<Long, RangerTag> tagMap = tagDBRetriever.getTags();
[13/28] ranger git commit: Update doc to reflect the Organization
associated with me(selva)
Posted by me...@apache.org.
Update doc to reflect the Organization associated with me(selva)
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/c4702256
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/c4702256
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/c4702256
Branch: refs/heads/ranger-1.1
Commit: c4702256b5ae38b17bd6f00e187d81a84e8118bf
Parents: be3dc31
Author: Selvamohan Neethiraj <sn...@apache.org>
Authored: Sat Aug 4 15:32:06 2018 -0400
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:38:00 2018 +0530
----------------------------------------------------------------------
docs/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/c4702256/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index 1426b75..2c38826 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -357,7 +357,7 @@
<role>Committer</role>
</roles>
<organization>
- Hortonworks
+ InfoTekies
</organization>
</developer>
<developer>
[17/28] ranger git commit: RANGER-2177: Handle validations for
duplicate configuration item during service create/edit
Posted by me...@apache.org.
RANGER-2177: Handle validations for duplicate configuration item during service create/edit
Signed-off-by: zhangqiang2 <zh...@zte.com.cn>
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/77c6e047
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/77c6e047
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/77c6e047
Branch: refs/heads/ranger-1.1
Commit: 77c6e0476170efea73ea086bf04978b0e4b99334
Parents: 4f7e4cb
Author: xingpeng1 <xi...@zte.com.cn>
Authored: Fri Aug 10 11:08:19 2018 +0800
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:40:30 2018 +0530
----------------------------------------------------------------------
.../src/main/webapp/scripts/views/service/ServiceForm.js | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/77c6e047/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js
----------------------------------------------------------------------
diff --git a/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js b/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js
index 726fb5f..b7215f2 100644
--- a/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js
+++ b/security-admin/src/main/webapp/scripts/views/service/ServiceForm.js
@@ -145,19 +145,22 @@ define(function(require){
formValidation : function(){
var valid = true;
var config = {};
- this.extraConfigColl.each(function(obj){
+
+ for (var i = 0; i < this.extraConfigColl.length; i++) {
+ var obj = this.extraConfigColl.at(i);
if(!_.isEmpty(obj.attributes)) {
if (!_.isUndefined(config[obj.get('name')])) {
XAUtil.alertPopup({
msg : localization.tt('msg.duplicateNewConfigValidationMsg')
});
valid = false;
- return;
+ break;
} else {
config[obj.get('name')] = obj.get('value');
}
}
- });
+ }
+
return valid;
},
[22/28] ranger git commit: RANGER-2186: Increment service-specific
policy and tag versions after update transaction is committed
Posted by me...@apache.org.
RANGER-2186: Increment service-specific policy and tag versions after update transaction is committed
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/fbcaaaf6
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/fbcaaaf6
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/fbcaaaf6
Branch: refs/heads/ranger-1.1
Commit: fbcaaaf6d780d959ee5fe6b5aae6c71ba289050f
Parents: b66e98d
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Wed Aug 15 21:03:41 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:45:03 2018 +0530
----------------------------------------------------------------------
.../plugin/store/AbstractServiceStore.java | 2 +-
.../org/apache/ranger/biz/ServiceDBStore.java | 148 ++++++++----------
...RangerTransactionSynchronizationAdapter.java | 154 ++++++++++++++-----
.../org/apache/ranger/db/RangerDaoManager.java | 8 +
.../apache/ranger/db/RangerDaoManagerBase.java | 6 -
.../ranger/db/XXServiceVersionInfoDao.java | 17 +-
.../apache/ranger/biz/TestServiceDBStore.java | 64 ++++----
7 files changed, 236 insertions(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/fbcaaaf6/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
index 69ded6d..b0dd283 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
@@ -138,7 +138,7 @@ public abstract class AbstractServiceStore implements ServiceStore {
}
}
- protected final long getNextVersion(Long currentVersion) {
+ public static long getNextVersion(Long currentVersion) {
return currentVersion == null ? 1L : currentVersion + 1;
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/fbcaaaf6/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index 0773616..cb5f240 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -57,6 +57,7 @@ import org.apache.ranger.common.AppConstants;
import org.apache.ranger.common.ContextUtil;
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.RangerCommonEnums;
+import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
@@ -304,6 +305,9 @@ public class ServiceDBStore extends AbstractServiceStore {
@Autowired
AssetMgr assetMgr;
+ @Autowired
+ RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter;
+
private static volatile boolean legacyServiceDefsInitDone = false;
private Boolean populateExistingBaseFields = false;
@@ -2032,7 +2036,7 @@ public class ServiceDBStore extends AbstractServiceStore {
dataHistService.createObjectDataHistory(updPolicy, RangerDataHistService.ACTION_UPDATE);
bizUtil.createTrxLog(trxLogList);
-
+
return updPolicy;
}
@@ -2814,6 +2818,8 @@ public class ServiceDBStore extends AbstractServiceStore {
updatePolicyVersion(service, isTagVersionUpdateNeeded);
}
+ public enum VERSION_TYPE { POLICY_VERSION, TAG_VERSION, POLICY_AND_TAG_VERSION }
+
private void updatePolicyVersion(RangerService service, boolean isTagVersionUpdateNeeded) throws Exception {
if(service == null || service.getId() == null) {
return;
@@ -2830,28 +2836,14 @@ public class ServiceDBStore extends AbstractServiceStore {
return;
}
- XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo();
-
- XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(service.getId());
-
- if(serviceVersionInfoDbObj != null) {
- serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion()));
- serviceVersionInfoDbObj.setPolicyUpdateTime(new Date());
-
- serviceVersionInfoDao.update(serviceVersionInfoDbObj);
-
- } else {
- LOG.warn("updatePolicyVersion(service=" + serviceDbObj.getName() + "): serviceVersionInfo not found, creating it..");
+ Runnable commitWork = new Runnable() {
+ @Override
+ public void run() {
+ persistVersionChange(daoMgr, serviceDbObj.getId(), VERSION_TYPE.POLICY_VERSION);
+ }
+ };
- serviceVersionInfoDbObj = new XXServiceVersionInfo();
- serviceVersionInfoDbObj.setServiceId(serviceDbObj.getId());
- serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceDbObj.getPolicyVersion()));
- serviceVersionInfoDbObj.setPolicyUpdateTime(new Date());
- serviceVersionInfoDbObj.setTagVersion(serviceDbObj.getTagVersion());
- serviceVersionInfoDbObj.setTagUpdateTime(serviceDbObj.getTagUpdateTime());
-
- serviceVersionInfoDao.create(serviceVersionInfoDbObj);
- }
+ transactionSynchronizationAdapter.executeOnTransactionCommit(commitWork);
// if this is a tag service, update all services that refer to this tag service
// so that next policy-download from plugins will get updated tag policies
@@ -2861,37 +2853,50 @@ public class ServiceDBStore extends AbstractServiceStore {
if(CollectionUtils.isNotEmpty(referringServices)) {
for(XXService referringService : referringServices) {
- serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(referringService.getId());
- if (serviceVersionInfoDbObj != null) {
-
- serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion()));
- serviceVersionInfoDbObj.setPolicyUpdateTime(new Date());
-
- if (filterForServicePlugin && isTagVersionUpdateNeeded) {
- serviceVersionInfoDbObj.setTagVersion(getNextVersion(serviceVersionInfoDbObj.getTagVersion()));
- serviceVersionInfoDbObj.setTagUpdateTime(new Date());
- }
- serviceVersionInfoDao.update(serviceVersionInfoDbObj);
- } else {
- LOG.warn("updatePolicyVersion(service=" + referringService.getName() + "): serviceVersionInfo not found, creating it..");
- serviceVersionInfoDbObj = new XXServiceVersionInfo();
- serviceVersionInfoDbObj.setServiceId(referringService.getId());
- serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(referringService.getPolicyVersion()));
- serviceVersionInfoDbObj.setPolicyUpdateTime(new Date());
- if (filterForServicePlugin && isTagVersionUpdateNeeded) {
- serviceVersionInfoDbObj.setTagVersion(getNextVersion(referringService.getTagVersion()));
- serviceVersionInfoDbObj.setTagUpdateTime(new Date());
- } else {
- serviceVersionInfoDbObj.setTagVersion(referringService.getTagVersion());
- serviceVersionInfoDbObj.setTagUpdateTime(referringService.getTagUpdateTime());
+ commitWork = new Runnable() {
+ @Override
+ public void run() {
+ persistVersionChange(daoMgr, referringService.getId(),
+ filterForServicePlugin && isTagVersionUpdateNeeded ? VERSION_TYPE.POLICY_AND_TAG_VERSION : VERSION_TYPE.POLICY_VERSION);
}
- serviceVersionInfoDao.create(serviceVersionInfoDbObj);
- }
+ };
+ transactionSynchronizationAdapter.executeOnTransactionCommit(commitWork);
}
}
}
}
+ public static void persistVersionChange(RangerDaoManager daoMgr, Long id, VERSION_TYPE versionType) {
+ XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo();
+
+ XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(id);
+
+ if(serviceVersionInfoDbObj != null) {
+ if (versionType == VERSION_TYPE.POLICY_VERSION || versionType == VERSION_TYPE.POLICY_AND_TAG_VERSION) {
+ serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion()));
+ serviceVersionInfoDbObj.setPolicyUpdateTime(new Date());
+ }
+ if (versionType == VERSION_TYPE.TAG_VERSION || versionType == VERSION_TYPE.POLICY_AND_TAG_VERSION) {
+
+ serviceVersionInfoDbObj.setTagVersion(getNextVersion(serviceVersionInfoDbObj.getTagVersion()));
+ serviceVersionInfoDbObj.setTagUpdateTime(new Date());
+ }
+
+ serviceVersionInfoDao.update(serviceVersionInfoDbObj);
+
+ } else {
+ XXService service = daoMgr.getXXService().getById(id);
+ serviceVersionInfoDbObj = new XXServiceVersionInfo();
+ serviceVersionInfoDbObj.setServiceId(service.getId());
+ serviceVersionInfoDbObj.setPolicyVersion(1L);
+ serviceVersionInfoDbObj.setPolicyUpdateTime(new Date());
+ serviceVersionInfoDbObj.setTagVersion(1L);
+ serviceVersionInfoDbObj.setTagUpdateTime(new Date());
+
+ serviceVersionInfoDao.create(serviceVersionInfoDbObj);
+ }
+ }
+
private XXPolicyItem createNewPolicyItemForPolicy(RangerPolicy policy, XXPolicy xPolicy, RangerPolicyItem policyItem, XXServiceDef xServiceDef, int itemOrder, int policyItemType) throws Exception {
XXPolicyItem xPolicyItem = new XXPolicyItem();
@@ -3308,52 +3313,31 @@ public class ServiceDBStore extends AbstractServiceStore {
boolean isTagServiceDef = StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME);
XXServiceDao serviceDao = daoMgr.getXXService();
- XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo();
List<XXService> services = serviceDao.findByServiceDefId(serviceDef.getId());
if(CollectionUtils.isNotEmpty(services)) {
for(XXService service : services) {
- XXServiceVersionInfo serviceVersionInfo = serviceVersionInfoDao.findByServiceId(service.getId());
- if (serviceVersionInfo != null) {
- serviceVersionInfo.setPolicyVersion(getNextVersion(serviceVersionInfo.getPolicyVersion()));
- serviceVersionInfo.setPolicyUpdateTime(serviceDef.getUpdateTime());
-
- serviceVersionInfoDao.update(serviceVersionInfo);
- } else {
- LOG.warn("updateServicesForServiceDefUpdate(service=" + service.getName() + "): serviceVersionInfo not found, creating it..");
- serviceVersionInfo = new XXServiceVersionInfo();
- serviceVersionInfo.setServiceId(service.getId());
- serviceVersionInfo.setPolicyVersion(getNextVersion(service.getPolicyVersion()));
- serviceVersionInfo.setTagVersion(service.getTagVersion());
- serviceVersionInfo.setPolicyUpdateTime(new Date());
- serviceVersionInfo.setTagUpdateTime(service.getTagUpdateTime());
-
- serviceVersionInfoDao.create(serviceVersionInfo);
- }
+ Runnable commitWork = new Runnable() {
+ @Override
+ public void run() {
+ persistVersionChange(daoMgr, service.getId(), VERSION_TYPE.POLICY_VERSION);
+ }
+ };
+ transactionSynchronizationAdapter.executeOnTransactionCommit(commitWork);
if(isTagServiceDef) {
List<XXService> referrringServices = serviceDao.findByTagServiceId(service.getId());
if(CollectionUtils.isNotEmpty(referrringServices)) {
for(XXService referringService : referrringServices) {
- serviceVersionInfo = serviceVersionInfoDao.findByServiceId(referringService.getId());
- if (serviceVersionInfo != null) {
- serviceVersionInfo.setPolicyVersion(getNextVersion(serviceVersionInfo.getPolicyVersion()));
- serviceVersionInfo.setPolicyUpdateTime(serviceDef.getUpdateTime());
-
- serviceVersionInfoDao.update(serviceVersionInfo);
- } else {
- LOG.warn("updateServicesForServiceDefUpdate(service=" + referringService.getName() + "): serviceVersionInfo not found, creating it..");
- serviceVersionInfo = new XXServiceVersionInfo();
- serviceVersionInfo.setServiceId(referringService.getId());
- serviceVersionInfo.setPolicyVersion(getNextVersion(referringService.getPolicyVersion()));
- serviceVersionInfo.setTagVersion(referringService.getTagVersion());
- serviceVersionInfo.setPolicyUpdateTime(new Date());
- serviceVersionInfo.setTagUpdateTime(referringService.getTagUpdateTime());
-
- serviceVersionInfoDao.create(serviceVersionInfo);
- }
+ commitWork = new Runnable() {
+ @Override
+ public void run() {
+ persistVersionChange(daoMgr, referringService.getId(), VERSION_TYPE.POLICY_VERSION);
+ }
+ };
+ transactionSynchronizationAdapter.executeOnTransactionCommit(commitWork);
}
}
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/fbcaaaf6/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
index 2a62fb4..536ca29 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java
@@ -22,6 +22,7 @@ package org.apache.ranger.common.db;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -46,6 +47,7 @@ public class RangerTransactionSynchronizationAdapter extends TransactionSynchron
private static final Log LOG = LogFactory.getLog(RangerTransactionSynchronizationAdapter.class);
private static final ThreadLocal<List<Runnable>> RUNNABLES = new ThreadLocal<List<Runnable>>();
+ private static final ThreadLocal<List<Runnable>> RUNNABLES_AFTER_COMMIT = new ThreadLocal<List<Runnable>>();
public void executeOnTransactionCompletion(Runnable runnable) {
if (LOG.isDebugEnabled()) {
@@ -64,7 +66,7 @@ public class RangerTransactionSynchronizationAdapter extends TransactionSynchron
TransactionSynchronizationAdapter
*/
- if (!TransactionSynchronizationManager.isSynchronizationActive()) {
+ if (!registerSynchronization()) {
LOG.info("Transaction synchronization is NOT ACTIVE. Executing right now runnable {" + runnable + "}");
runnable.run();
return;
@@ -73,9 +75,36 @@ public class RangerTransactionSynchronizationAdapter extends TransactionSynchron
if (threadRunnables == null) {
threadRunnables = new ArrayList<Runnable>();
RUNNABLES.set(threadRunnables);
- // Register a new transaction synchronization for the current thread.
- // TransactionSynchronizationManage will call afterCompletion() when current transaction completes.
- TransactionSynchronizationManager.registerSynchronization(this);
+ }
+ threadRunnables.add(runnable);
+ }
+
+ public void executeOnTransactionCommit(Runnable runnable) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Submitting new runnable {" + runnable + "} to run after transaction is committed");
+ }
+
+ /*
+ From TransactionSynchronizationManager documentation:
+ TransactionSynchronizationManager is a central helper that manages resources and transaction synchronizations per thread.
+ Resource management code should only register synchronizations when this manager is active,
+ which can be checked via isSynchronizationActive(); it should perform immediate resource cleanup else.
+ If transaction synchronization isn't active, there is either no current transaction,
+ or the transaction manager doesn't support transaction synchronization.
+
+ Note: Synchronization is an Interface for transaction synchronization callbacks which is implemented by
+ TransactionSynchronizationAdapter
+ */
+
+ if (!registerSynchronization()) {
+ LOG.info("Transaction synchronization is NOT ACTIVE. Executing right now runnable {" + runnable + "}");
+ runnable.run();
+ return;
+ }
+ List<Runnable> threadRunnables = RUNNABLES_AFTER_COMMIT.get();
+ if (threadRunnables == null) {
+ threadRunnables = new ArrayList<Runnable>();
+ RUNNABLES_AFTER_COMMIT.set(threadRunnables);
}
threadRunnables.add(runnable);
}
@@ -83,48 +112,93 @@ public class RangerTransactionSynchronizationAdapter extends TransactionSynchron
@Override
public void afterCompletion(int status) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Transaction completed with status {" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + "}");
+ LOG.debug("==> RangerTransactionSynchronizationAdapter.afterCompletion(status=" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + ")");
}
- /* Thread runnables are expected to be executed only when the status is STATUS_ROLLED_BACK. Currently, executeOnTransactionCompletion()
- * is called only for those changes that are going to be rolled-back by TransactionSynchronizationManager - such
- * as when the operation returns HttpServletResponse.SC_NOT_MODIFIED status.
- */
- //if (status == STATUS_ROLLED_BACK) {
- final List<Runnable> threadRunnables = RUNNABLES.get();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Transaction completed, executing {" + threadRunnables.size() + "} runnables");
+
+ List<Runnable> allRunnables = null;
+
+ if (status == STATUS_COMMITTED) {
+ final List<Runnable> postCommitRunnables = RUNNABLES_AFTER_COMMIT.get();
+ if (CollectionUtils.isNotEmpty(postCommitRunnables)) {
+ allRunnables = postCommitRunnables;
}
- if (threadRunnables != null) {
- try {
- //Create new transaction
- TransactionTemplate txTemplate = new TransactionTemplate(txManager);
- txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
-
- txTemplate.execute(new TransactionCallback<Object>() {
- public Object doInTransaction(TransactionStatus status) {
- for (Runnable runnable : threadRunnables) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Executing runnable {" + runnable + "}");
- }
- try {
- runnable.run();
- } catch (RuntimeException e) {
- LOG.error("Failed to execute runnable " + runnable, e);
- break;
- }
- }
+ }
- return null;
- }
- });
- } catch (Exception e) {
- LOG.error("Failed to commit TransactionService transaction", e);
- LOG.error("Ignoring...");
- }
+ final List<Runnable> postCompletionRunnables = RUNNABLES.get();
+
+ if (CollectionUtils.isNotEmpty(postCompletionRunnables)) {
+ if (allRunnables == null) {
+ allRunnables = postCompletionRunnables;
+ } else {
+ allRunnables.addAll(postCompletionRunnables);
}
+ }
- //}
+ runRunnables(allRunnables);
+
+ RUNNABLES_AFTER_COMMIT.remove();
RUNNABLES.remove();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerTransactionSynchronizationAdapter.afterCompletion(status=" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + ")");
+ }
}
+ private boolean registerSynchronization() {
+ final boolean ret = TransactionSynchronizationManager.isSynchronizationActive();
+ if (ret) {
+ List<Runnable> threadRunnablesOnCompletion = RUNNABLES.get();
+ List<Runnable> threadRunnablesOnCommit = RUNNABLES_AFTER_COMMIT.get();
+ if (threadRunnablesOnCompletion == null && threadRunnablesOnCommit == null) {
+ TransactionSynchronizationManager.registerSynchronization(this);
+ }
+ }
+ return ret;
+ }
+
+ private void runRunnables(final List<Runnable> runnables) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerTransactionSynchronizationAdapter.runRunnables()");
+ }
+
+ if (runnables != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Executing {" + runnables.size() + "} runnables");
+ }
+ try {
+ //Create new transaction
+ TransactionTemplate txTemplate = new TransactionTemplate(txManager);
+ txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
+
+ txTemplate.execute(new TransactionCallback<Object>() {
+ public Object doInTransaction(TransactionStatus status) {
+ for (Runnable runnable : runnables) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Executing runnable {" + runnable + "}");
+ }
+ try {
+ runnable.run();
+ } catch (RuntimeException e) {
+ LOG.error("Failed to execute runnable " + runnable, e);
+ break;
+ }
+ }
+
+ return null;
+ }
+ });
+ } catch (Exception e) {
+ LOG.error("Failed to commit TransactionService transaction", e);
+ LOG.error("Ignoring...");
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No runnables to execute");
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerTransactionSynchronizationAdapter.runRunnables()");
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ranger/blob/fbcaaaf6/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java
index 2788a61..8c1cfd8 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManager.java
@@ -26,6 +26,7 @@ import javax.persistence.PersistenceContext;
import org.apache.log4j.Logger;
import org.apache.ranger.common.StringUtil;
+import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -42,6 +43,9 @@ public class RangerDaoManager extends RangerDaoManagerBase {
@Autowired
StringUtil stringUtil;
+ @Autowired
+ RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter;
+
@Override
public EntityManager getEntityManager() {
return em;
@@ -66,4 +70,8 @@ public class RangerDaoManager extends RangerDaoManagerBase {
return stringUtil;
}
+ public RangerTransactionSynchronizationAdapter getRangerTransactionSynchronizationAdapter() {
+ return transactionSynchronizationAdapter;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/fbcaaaf6/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
index da89e04..5dffc0e 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
@@ -25,14 +25,8 @@
import javax.persistence.EntityManager;
-import org.apache.ranger.common.RESTErrorUtil;
-import org.springframework.beans.factory.annotation.Autowired;
-
-
public abstract class RangerDaoManagerBase {
- @Autowired
- protected RESTErrorUtil restErrorUtil;
abstract public EntityManager getEntityManager();
public RangerDaoManagerBase() {
http://git-wip-us.apache.org/repos/asf/ranger/blob/fbcaaaf6/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java
index e100329..d7e5a86 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java
@@ -23,6 +23,7 @@ import java.util.List;
import javax.persistence.NoResultException;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.ranger.biz.ServiceDBStore;
import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.entity.XXServiceVersionInfo;
import org.springframework.stereotype.Service;
@@ -31,6 +32,7 @@ import org.springframework.stereotype.Service;
*/
@Service
public class XXServiceVersionInfoDao extends BaseDao<XXServiceVersionInfo> {
+
/**
* Default Constructor
*/
@@ -122,14 +124,17 @@ public class XXServiceVersionInfoDao extends BaseDao<XXServiceVersionInfo> {
}
for(XXServiceVersionInfo serviceVersionInfo : serviceVersionInfos) {
- Long currentTagVersion = serviceVersionInfo.getTagVersion();
- if(currentTagVersion == null) {
- currentTagVersion = Long.valueOf(0);
- }
+ Runnable commitWork = new Runnable() {
+ @Override
+ public void run() {
+ ServiceDBStore.persistVersionChange(daoManager, serviceVersionInfo.getId(), ServiceDBStore.VERSION_TYPE.TAG_VERSION);
+ }
+ };
+
+ daoManager.getRangerTransactionSynchronizationAdapter().executeOnTransactionCommit(commitWork);
- serviceVersionInfo.setTagVersion(currentTagVersion + 1);
- serviceVersionInfo.setTagUpdateTime(updateTime);
}
+
}
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/fbcaaaf6/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
index bd474cc..c9db90a 100644
--- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
+++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
@@ -30,6 +30,7 @@ import org.apache.ranger.common.RangerFactory;
import org.apache.ranger.common.SearchCriteria;
import org.apache.ranger.common.StringUtil;
import org.apache.ranger.common.UserSessionBase;
+import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter;
import org.apache.ranger.db.*;
import org.apache.ranger.entity.*;
import org.apache.ranger.plugin.model.RangerPolicy;
@@ -138,9 +139,10 @@ public class TestServiceDBStore {
@Mock
AssetMgr assetMgr;
-
-
-
+
+ @Mock
+ RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter;
+
@Rule
public ExpectedException thrown = ExpectedException.none();
@@ -1056,8 +1058,8 @@ public class TestServiceDBStore {
public void test20updateService() throws Exception {
XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class);
XXService xService = Mockito.mock(XXService.class);
- XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
- XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
+ //XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
+ //XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
XXServiceConfigMapDao xServiceConfigMapDao = Mockito
.mock(XXServiceConfigMapDao.class);
XXServiceConfigDefDao xServiceConfigDefDao = Mockito
@@ -1134,9 +1136,9 @@ public class TestServiceDBStore {
Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn(
rangerService);
- Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
- Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
- Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
+ //Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
RangerService dbRangerService = serviceDBStore
.updateService(rangerService, options);
@@ -1159,8 +1161,8 @@ public class TestServiceDBStore {
XXPolicyDao xPolicyDao = Mockito.mock(XXPolicyDao.class);
XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class);
XXService xService = Mockito.mock(XXService.class);
- XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
- XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
+ //XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
+ //XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
XXPolicyItemDao xPolicyItemDao = Mockito.mock(XXPolicyItemDao.class);
XXPolicyItemDataMaskInfoDao xxPolicyItemDataMaskInfoDao = Mockito.mock(XXPolicyItemDataMaskInfoDao.class);
XXPolicyItemRowFilterInfoDao xxPolicyItemRowFilterInfoDao = Mockito.mock(XXPolicyItemRowFilterInfoDao.class);
@@ -1327,9 +1329,9 @@ public class TestServiceDBStore {
Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn(
rangerService);
- Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
- Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
- Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
+ //Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
Mockito.when(daoManager.getXXPolicyItem()).thenReturn(xPolicyItemDao);
Mockito.when(xPolicyItemDao.findByPolicyId(policyItem.getId()))
@@ -1524,9 +1526,9 @@ public class TestServiceDBStore {
XXPolicy xPolicy = Mockito.mock(XXPolicy.class);
XXPolicyDao xPolicyDao = Mockito.mock(XXPolicyDao.class);
XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class);
- XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
+ //XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
XXService xService = Mockito.mock(XXService.class);
- XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
+ //XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
XXPolicyItemDao xPolicyItemDao = Mockito.mock(XXPolicyItemDao.class);
XXServiceDef xServiceDef = serviceDef();
@@ -1682,9 +1684,9 @@ public class TestServiceDBStore {
Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao);
Mockito.when(xServiceDao.findByName(name)).thenReturn(xService);
- Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
- Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
- Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
+ //Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn(
rangerService);
@@ -1763,6 +1765,9 @@ public class TestServiceDBStore {
.thenReturn(policyConditionDefObj);
Mockito.when(!bizUtil.hasAccess(xService, null)).thenReturn(true);
+ //RangerTransactionSynchronizationAdapter spy = Mockito.spy(transactionSynchronizationAdapter);
+ //Mockito.doNothing().when(spy).executeOnTransactionCommit(Mockito.any(Runnable.class));
+
RangerPolicy dbRangerPolicy = serviceDBStore.createPolicy(rangerPolicy);
Assert.assertNull(dbRangerPolicy);
Assert.assertEquals(Id, rangerPolicy.getId());
@@ -1813,10 +1818,10 @@ public class TestServiceDBStore {
XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class);
XXPolicyLabelMapDao xPolicyLabelMapDao = Mockito.mock(XXPolicyLabelMapDao.class);
XXService xService = Mockito.mock(XXService.class);
- XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
+ //XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
XXServiceDefDao xServiceDefDao = Mockito.mock(XXServiceDefDao.class);
XXServiceDef xServiceDef = Mockito.mock(XXServiceDef.class);
- XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
+ //XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
XXPolicyResourceDao xPolicyResourceDao = Mockito
.mock(XXPolicyResourceDao.class);
XXPolicyResourceMapDao xPolicyResourceMapDao = Mockito
@@ -1883,9 +1888,9 @@ public class TestServiceDBStore {
Mockito.when(svcService.getPopulatedViewObject(xService)).thenReturn(
rangerService);
- Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
- Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
- Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
+ //Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
Mockito.when(daoManager.getXXServiceDef()).thenReturn(xServiceDefDao);
Mockito.when(xServiceDefDao.findByName(rangerService.getType()))
@@ -1960,8 +1965,8 @@ public class TestServiceDBStore {
setup();
XXServiceDao xServiceDao = Mockito.mock(XXServiceDao.class);
XXService xService = Mockito.mock(XXService.class);
- XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
- XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
+ //XXServiceVersionInfoDao xServiceVersionInfoDao = Mockito.mock(XXServiceVersionInfoDao.class);
+ //XXServiceVersionInfo xServiceVersionInfo = Mockito.mock(XXServiceVersionInfo.class);
XXPolicyLabelMapDao xPolicyLabelMapDao = Mockito.mock(XXPolicyLabelMapDao.class);
XXPolicyItemDao xPolicyItemDao = Mockito.mock(XXPolicyItemDao.class);
XXPolicyItemDataMaskInfoDao xPolicyItemDataMaskInfoDao = Mockito.mock(XXPolicyItemDataMaskInfoDao.class);
@@ -2136,9 +2141,9 @@ public class TestServiceDBStore {
Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao);
Mockito.when(xServiceDao.getById(Id)).thenReturn(xService);
- Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
- Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
- Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(daoManager.getXXServiceVersionInfo()).thenReturn(xServiceVersionInfoDao);
+ //Mockito.when(xServiceVersionInfoDao.findByServiceId(Id)).thenReturn(xServiceVersionInfo);
+ //Mockito.when(xServiceVersionInfoDao.update(xServiceVersionInfo)).thenReturn(xServiceVersionInfo);
Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao);
Mockito.when(xServiceDao.getById(rangerService.getId())).thenReturn(
@@ -2150,6 +2155,9 @@ public class TestServiceDBStore {
xPolicyLabelMapDao);
Mockito.when(xPolicyLabelMapDao.findByPolicyId(rangerPolicy.getId()))
.thenReturn(xxPolicyLabelMapList);
+ //RangerTransactionSynchronizationAdapter spy = Mockito.spy(transactionSynchronizationAdapter);
+ //Mockito.doNothing().when(spy).executeOnTransactionCommit(Mockito.any(Runnable.class));
+
serviceDBStore.deletePolicy(Id);
}
[04/28] ranger git commit: RANGER-1951: fix the script,
so it works even outside a git repository
Posted by me...@apache.org.
RANGER-1951: fix the script, so it works even outside a git repository
Conflicts:
ranger-util/src/scripts/saveVersion.py
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/d3c10fc7
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/d3c10fc7
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/d3c10fc7
Branch: refs/heads/ranger-1.1
Commit: d3c10fc7c8fcd8a8e10355f82bf6522da7975fdd
Parents: bfc1f19
Author: Zsombor Gegesy <zs...@apache.org>
Authored: Thu Jul 5 23:07:57 2018 +0200
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:22:25 2018 +0530
----------------------------------------------------------------------
ranger-util/src/scripts/saveVersion.py | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/d3c10fc7/ranger-util/src/scripts/saveVersion.py
----------------------------------------------------------------------
diff --git a/ranger-util/src/scripts/saveVersion.py b/ranger-util/src/scripts/saveVersion.py
index 40c0267..cceea41 100644
--- a/ranger-util/src/scripts/saveVersion.py
+++ b/ranger-util/src/scripts/saveVersion.py
@@ -29,7 +29,7 @@ import datetime
import getpass
import socket
import subprocess
-from subprocess import Popen,PIPE
+from subprocess import Popen,PIPE,CalledProcessError
from time import gmtime, strftime
import platform
@@ -37,9 +37,12 @@ def isWindowsSystem():
return 'Windows' in platform.system()
def check_output(query):
- p = subprocess.Popen(query, stdout=subprocess.PIPE)
- output = p.communicate ()[0]
- return output
+ try:
+ output = subprocess.check_output(query)
+ return output.decode("UTF-8")
+ except CalledProcessError:
+ # Not a git repository, or no git is installed
+ return ''
def hashfile(afile, hasher, blocksize=65536):
buf = afile.read(blocksize)
@@ -97,10 +100,10 @@ def main():
pass
sortedList = sorted(fileList, key = lambda x: x[:-4])
- for i, val in enumerate(sortedList):
+ for _, val in enumerate(sortedList):
m = hashfile(open(val,'rb'), hashlib.md5())
f = m +" "+ val + "\n"
- c.append(f);
+ c.append(f)
srcChecksum = hashlib.md5(''.join(c)).hexdigest()
[24/28] ranger git commit: Updated organization details for few PMC
members
Posted by me...@apache.org.
Updated organization details for few PMC members
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/d5417308
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/d5417308
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/d5417308
Branch: refs/heads/ranger-1.1
Commit: d5417308eb4f481c1406b0df8612409a9a05c87a
Parents: 383864c
Author: Selvamohan Neethiraj <sn...@apache.org>
Authored: Sun Aug 19 15:56:54 2018 -0400
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:49:05 2018 +0530
----------------------------------------------------------------------
docs/pom.xml | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/d5417308/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index 2c38826..9c6426d 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -134,6 +134,7 @@
<role>Committer</role>
</roles>
<organization>
+ Privacera
</organization>
</developer>
<developer>
@@ -208,6 +209,7 @@
<role>Committer</role>
</roles>
<organization>
+ Privacera
</organization>
</developer>
<developer>
[25/28] ranger git commit: RANGER-2188: Support multiple threads to
build Trie and on-lookup post-setup for Trie nodes - set default to lazy
post-setup
Posted by me...@apache.org.
RANGER-2188: Support multiple threads to build Trie and on-lookup post-setup for Trie nodes - set default to lazy post-setup
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/c9a0afaf
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/c9a0afaf
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/c9a0afaf
Branch: refs/heads/ranger-1.1
Commit: c9a0afaf645118efa5309d96630fe7956b015291
Parents: d541730
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Aug 21 15:20:45 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:49:21 2018 +0530
----------------------------------------------------------------------
.../ranger/plugin/policyengine/RangerPolicyEngineOptions.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/c9a0afaf/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
index 01dbbe8..5498545 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
@@ -33,7 +33,7 @@ public class RangerPolicyEngineOptions {
public boolean evaluateDelegateAdminOnly = false;
public boolean enableTagEnricherWithLocalRefresher = false;
public boolean disableAccessEvaluationWithPolicyACLSummary = true;
- public boolean optimizeTrieForRetrieval = true;
+ public boolean optimizeTrieForRetrieval = false;
private RangerServiceDefHelper serviceDefHelper;
@@ -51,7 +51,7 @@ public class RangerPolicyEngineOptions {
evaluateDelegateAdminOnly = false;
enableTagEnricherWithLocalRefresher = false;
disableAccessEvaluationWithPolicyACLSummary = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.access.evaluation.with.policy.acl.summary", true);
- optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", true);
+ optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false);
}
[06/28] ranger git commit: Change organization
Posted by me...@apache.org.
Change organization
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/4a13ff77
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/4a13ff77
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/4a13ff77
Branch: refs/heads/ranger-1.1
Commit: 4a13ff77383f75f49a4598221617ffee3a0a6e7b
Parents: 0926baf
Author: pengjianhua <35...@qq.com>
Authored: Tue Jul 31 00:20:29 2018 +0800
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:29:08 2018 +0530
----------------------------------------------------------------------
docs/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/4a13ff77/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index 4cbd26e..1426b75 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -394,7 +394,7 @@
<role>Committer</role>
</roles>
<organization>
- ZTE
+ NIIT
</organization>
</developer>
<developer>
[18/28] ranger git commit: RANGER-2114 : Internal Exception:
com.mysql.jdbc.MysqlDataTruncation: Data truncation: Data too long for
Posted by me...@apache.org.
RANGER-2114 : Internal Exception: com.mysql.jdbc.MysqlDataTruncation: Data truncation: Data too long for
column 'content' at row 1
Signed-off-by: Gautam Borad <ga...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/b163a48c
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/b163a48c
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/b163a48c
Branch: refs/heads/ranger-1.1
Commit: b163a48c93d798f9c3baa2f8400d435f5d4315ef
Parents: 77c6e04
Author: fatimaawez <fa...@gmail.com>
Authored: Fri Aug 10 15:37:40 2018 +0530
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:41:32 2018 +0530
----------------------------------------------------------------------
.../optimized/current/ranger_core_db_mysql.sql | 3 +-
.../patches/034-x_data_histContentSize.sql | 31 ++++++++++++++++++++
2 files changed, 33 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/b163a48c/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
index 9d200ba..3f23b00 100644
--- a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
+++ b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
@@ -923,7 +923,7 @@ CREATE TABLE `x_data_hist` (
`action` varchar(512) NOT NULL,
`from_time` datetime NOT NULL,
`to_time` datetime DEFAULT NULL,
-`content` text NOT NULL,
+`content` MEDIUMTEXT NOT NULL,
primary key (`id`)
)ROW_FORMAT=DYNAMIC;
@@ -1345,6 +1345,7 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('031',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('032',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('033',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y');
+INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('034',UTC_TIMESTAMP(),'Ranger 2.0.0',UTC_TIMESTAMP(),'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y');
INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,3,UTC_TIMESTAMP(),UTC_TIMESTAMP(),1,1,1);
INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (1,1,UTC_TIMESTAMP(),UTC_TIMESTAMP(),1,1,1);
http://git-wip-us.apache.org/repos/asf/ranger/blob/b163a48c/security-admin/db/mysql/patches/034-x_data_histContentSize.sql
----------------------------------------------------------------------
diff --git a/security-admin/db/mysql/patches/034-x_data_histContentSize.sql b/security-admin/db/mysql/patches/034-x_data_histContentSize.sql
new file mode 100644
index 0000000..18aa20f
--- /dev/null
+++ b/security-admin/db/mysql/patches/034-x_data_histContentSize.sql
@@ -0,0 +1,31 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+drop procedure if exists change_content_column_datatype_of_x_data_hist_table;
+
+delimiter ;;
+create procedure change_content_column_datatype_of_x_data_hist_table() begin
+
+ /* change content data type to longtext if not exist */
+ if exists (select * from information_schema.columns where table_schema=database() and table_name = 'x_data_hist' and column_name = 'content' and data_type='text') then
+ ALTER TABLE `x_data_hist` CHANGE `content` `content` MEDIUMTEXT NOT NULL;
+ end if;
+
+end;;
+
+delimiter ;
+call change_content_column_datatype_of_x_data_hist_table();
+
+drop procedure if exists change_content_column_datatype_of_x_data_hist_table;
[26/28] ranger git commit: RANGER-2195:TagPolicy not working due to
failure to update tag policy version
Posted by me...@apache.org.
RANGER-2195:TagPolicy not working due to failure to update tag policy version
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/e8c5935a
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/e8c5935a
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/e8c5935a
Branch: refs/heads/ranger-1.1
Commit: e8c5935ac1a0b4959797fec6b5093285bb493343
Parents: c9a0afa
Author: rmani <rm...@hortonworks.com>
Authored: Wed Aug 22 11:55:40 2018 -0700
Committer: Mehul Parikh <me...@apache.org>
Committed: Tue Sep 4 11:49:43 2018 +0530
----------------------------------------------------------------------
.../org/apache/ranger/biz/ServiceDBStore.java | 81 +++++++++++---------
.../ranger/db/XXServiceVersionInfoDao.java | 13 ++--
2 files changed, 51 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/e8c5935a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index cb5f240..32af4cd 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -2836,31 +2836,26 @@ public class ServiceDBStore extends AbstractServiceStore {
return;
}
- Runnable commitWork = new Runnable() {
- @Override
- public void run() {
- persistVersionChange(daoMgr, serviceDbObj.getId(), VERSION_TYPE.POLICY_VERSION);
- }
- };
+ final RangerDaoManager daoManager = daoMgr;
+ final Long serviceId = serviceDbObj.getId();
+ final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION;
- transactionSynchronizationAdapter.executeOnTransactionCommit(commitWork);
+ Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType);
+ transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater);
// if this is a tag service, update all services that refer to this tag service
// so that next policy-download from plugins will get updated tag policies
boolean isTagService = serviceDbObj.getType() == EmbeddedServiceDefsUtil.instance().getTagServiceDefId();
if(isTagService) {
- List<XXService> referringServices = serviceDao.findByTagServiceId(serviceDbObj.getId());
+ List<XXService> referringServices = serviceDao.findByTagServiceId(serviceId);
if(CollectionUtils.isNotEmpty(referringServices)) {
for(XXService referringService : referringServices) {
- commitWork = new Runnable() {
- @Override
- public void run() {
- persistVersionChange(daoMgr, referringService.getId(),
- filterForServicePlugin && isTagVersionUpdateNeeded ? VERSION_TYPE.POLICY_AND_TAG_VERSION : VERSION_TYPE.POLICY_VERSION);
- }
- };
- transactionSynchronizationAdapter.executeOnTransactionCommit(commitWork);
+ final Long referringServiceId = referringService.getId();
+ final VERSION_TYPE tagServiceversionType = filterForServicePlugin && isTagVersionUpdateNeeded ? VERSION_TYPE.POLICY_AND_TAG_VERSION : VERSION_TYPE.POLICY_VERSION;
+
+ Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceversionType);
+ transactionSynchronizationAdapter.executeOnTransactionCommit(tagServiceVersionUpdater);
}
}
}
@@ -2877,7 +2872,6 @@ public class ServiceDBStore extends AbstractServiceStore {
serviceVersionInfoDbObj.setPolicyUpdateTime(new Date());
}
if (versionType == VERSION_TYPE.TAG_VERSION || versionType == VERSION_TYPE.POLICY_AND_TAG_VERSION) {
-
serviceVersionInfoDbObj.setTagVersion(getNextVersion(serviceVersionInfoDbObj.getTagVersion()));
serviceVersionInfoDbObj.setTagUpdateTime(new Date());
}
@@ -3310,6 +3304,9 @@ public class ServiceDBStore extends AbstractServiceStore {
if(serviceDef == null) {
return;
}
+
+ final RangerDaoManager daoManager = daoMgr;
+
boolean isTagServiceDef = StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME);
XXServiceDao serviceDao = daoMgr.getXXService();
@@ -3318,26 +3315,24 @@ public class ServiceDBStore extends AbstractServiceStore {
if(CollectionUtils.isNotEmpty(services)) {
for(XXService service : services) {
- Runnable commitWork = new Runnable() {
- @Override
- public void run() {
- persistVersionChange(daoMgr, service.getId(), VERSION_TYPE.POLICY_VERSION);
- }
- };
- transactionSynchronizationAdapter.executeOnTransactionCommit(commitWork);
+
+ final Long serviceId = service.getId();
+ final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION;
+
+ Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType);
+ transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater);
if(isTagServiceDef) {
- List<XXService> referrringServices = serviceDao.findByTagServiceId(service.getId());
-
- if(CollectionUtils.isNotEmpty(referrringServices)) {
- for(XXService referringService : referrringServices) {
- commitWork = new Runnable() {
- @Override
- public void run() {
- persistVersionChange(daoMgr, referringService.getId(), VERSION_TYPE.POLICY_VERSION);
- }
- };
- transactionSynchronizationAdapter.executeOnTransactionCommit(commitWork);
+ List<XXService> referringServices = serviceDao.findByTagServiceId(service.getId());
+
+ if(CollectionUtils.isNotEmpty(referringServices)) {
+ for(XXService referringService : referringServices) {
+
+ final Long referringServiceId = referringService.getId();
+ final VERSION_TYPE tagServiceVersionType = VERSION_TYPE.POLICY_VERSION;
+
+ Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceVersionType);
+ transactionSynchronizationAdapter.executeOnTransactionCommit(tagServiceVersionUpdater);
}
}
}
@@ -4787,4 +4782,20 @@ public class ServiceDBStore extends AbstractServiceStore {
}
return ret;
}
+
+ public static class ServiceVersionUpdater implements Runnable {
+ final Long serviceId;
+ final RangerDaoManager daoManager;
+ final VERSION_TYPE versionType;
+
+ public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType ) {
+ this.serviceId = serviceId;
+ this.daoManager = daoManager;
+ this.versionType = versionType;
+ }
+ @Override
+ public void run() {
+ ServiceDBStore.persistVersionChange(this.daoManager, this.serviceId, this.versionType);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ranger/blob/e8c5935a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java
index d7e5a86..8081703 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceVersionInfoDao.java
@@ -124,16 +124,13 @@ public class XXServiceVersionInfoDao extends BaseDao<XXServiceVersionInfo> {
}
for(XXServiceVersionInfo serviceVersionInfo : serviceVersionInfos) {
+ final RangerDaoManager finaldaoManager = daoManager;
+ final Long finalServiceId = serviceVersionInfo.getServiceId();
+ final ServiceDBStore.VERSION_TYPE versionType = ServiceDBStore.VERSION_TYPE.TAG_VERSION;
- Runnable commitWork = new Runnable() {
- @Override
- public void run() {
- ServiceDBStore.persistVersionChange(daoManager, serviceVersionInfo.getId(), ServiceDBStore.VERSION_TYPE.TAG_VERSION);
- }
- };
-
- daoManager.getRangerTransactionSynchronizationAdapter().executeOnTransactionCommit(commitWork);
+ Runnable serviceVersionUpdater = new ServiceDBStore.ServiceVersionUpdater(finaldaoManager, finalServiceId, versionType);
+ daoManager.getRangerTransactionSynchronizationAdapter().executeOnTransactionCommit(serviceVersionUpdater);
}
}