You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/03/21 06:13:20 UTC
[1/2] incubator-ranger git commit: RANGER-323: Policy evaluation
optimization: cache results of resource-match in policy
Repository: incubator-ranger
Updated Branches:
refs/heads/master 87b6d4b59 -> b3dc07ce8
RANGER-323: Policy evaluation optimization: cache results of resource-match in policy
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/04536c6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/04536c6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/04536c6c
Branch: refs/heads/master
Commit: 04536c6c74cd54378156f9c365973b3d53033f25
Parents: 85d5658
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Fri Mar 20 19:03:59 2015 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Mar 20 22:12:49 2015 -0700
----------------------------------------------------------------------
.../RangerPolicyEvaluatorFacade.java | 24 +++-
.../policyengine/RangerPolicyRepository.java | 4 +-
.../RangerCachedPolicyEvaluator.java | 81 +++++++++++++
.../RangerResourceAccessCache.java | 39 ++++++
.../RangerResourceAccessCacheImpl.java | 118 +++++++++++++++++++
5 files changed, 257 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/04536c6c/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java
index 92dedba..755f553 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java
@@ -19,11 +19,14 @@
package org.apache.ranger.plugin.policyengine;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.conditionevaluator.RangerConditionEvaluator;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.policyevaluator.RangerCachedPolicyEvaluator;
import org.apache.ranger.plugin.policyevaluator.RangerDefaultPolicyEvaluator;
import org.apache.ranger.plugin.policyevaluator.RangerOptimizedPolicyEvaluator;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
@@ -33,13 +36,19 @@ import java.util.Map;
public class RangerPolicyEvaluatorFacade implements RangerPolicyEvaluator, Comparable<RangerPolicyEvaluatorFacade> {
private static final Log LOG = LogFactory.getLog(RangerPolicyEvaluatorFacade.class);
- RangerDefaultPolicyEvaluator delegate = null;
- int computedPolicyEvalOrder = 0;
+ RangerDefaultPolicyEvaluator delegate = null;
+ int computedPolicyEvalOrder = 0;
- RangerPolicyEvaluatorFacade(boolean useCachePolicyEngine) {
+ RangerPolicyEvaluatorFacade() {
super();
- delegate = new RangerOptimizedPolicyEvaluator();
+ String evaluatorType = RangerConfiguration.getInstance().get("ranger.policyengine.evaluator.type", "cached");
+
+ if(StringUtils.isEmpty(evaluatorType) || StringUtils.equalsIgnoreCase(evaluatorType, "cached")) {
+ delegate = new RangerCachedPolicyEvaluator();
+ } else {
+ delegate = new RangerOptimizedPolicyEvaluator();
+ }
}
RangerPolicyEvaluator getPolicyEvaluator() {
@@ -78,12 +87,12 @@ public class RangerPolicyEvaluatorFacade implements RangerPolicyEvaluator, Compa
@Override
public boolean isMatch(RangerAccessResource resource) {
- return false;
+ return delegate.isMatch(resource);
}
@Override
public boolean isSingleAndExactMatch(RangerAccessResource resource) {
- return false;
+ return delegate.isSingleAndExactMatch(resource);
}
@Override
@@ -121,10 +130,13 @@ public class RangerPolicyEvaluatorFacade implements RangerPolicyEvaluator, Compa
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyEvaluatorFacade.computePolicyEvalOrder()");
}
+
int result = delegate.computePolicyEvalOrder();
+
if(LOG.isDebugEnabled()) {
LOG.debug("<==RangerPolicyEvaluatorFacade.computePolicyEvalOrder(), result:" + result);
}
+
return result;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/04536c6c/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 4ed11c1..2e4a79e 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -42,7 +42,6 @@ public class RangerPolicyRepository {
private List<RangerContextEnricher> contextEnrichers = null;
private RangerServiceDef serviceDef = null;
// Not used at this time
- private boolean useCachePolicyEngine = false;
private Map<String, Boolean> accessAuditCache = null;
private static int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64*1024;
@@ -149,7 +148,7 @@ public class RangerPolicyRepository {
RangerPolicyEvaluatorFacade ret = null;
- ret = new RangerPolicyEvaluatorFacade(useCachePolicyEngine);
+ ret = new RangerPolicyEvaluatorFacade();
ret.init(policy, serviceDef);
if(LOG.isDebugEnabled()) {
@@ -230,7 +229,6 @@ public class RangerPolicyRepository {
}
}
}
- sb.append("useCachePolicyEngine={").append(useCachePolicyEngine).append("} ");
sb.append("} ");
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/04536c6c/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java
new file mode 100644
index 0000000..f4db52b
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.policyevaluator;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.policyengine.RangerAccessResource;
+
+public class RangerCachedPolicyEvaluator extends RangerOptimizedPolicyEvaluator {
+ private static final Log LOG = LogFactory.getLog(RangerCachedPolicyEvaluator.class);
+
+ private RangerResourceAccessCache cache = null;
+
+ @Override
+ public void init(RangerPolicy policy, RangerServiceDef serviceDef) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerCachedPolicyEvaluator.init()");
+ }
+
+ super.init(policy, serviceDef);
+
+ cache = RangerResourceAccessCacheImpl.getInstance(serviceDef, policy);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerCachedPolicyEvaluator.init()");
+ }
+ }
+
+ @Override
+ public boolean isMatch(RangerAccessResource resource) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerCachedPolicyEvaluator.isMatch(" + resource + ")");
+ }
+
+ boolean result = false;
+
+ // Check in the evaluator-owned cache for the match, if found return. else call super.isMatch(), add result to cache
+ RangerResourceAccessCache.LookupResult lookup = cache.lookup(resource);
+
+ if (lookup != RangerResourceAccessCache.LookupResult.IN_NOTMATCHED_CACHE) {
+ // We dont know definitely that this previously not matched
+ if (lookup != RangerResourceAccessCache.LookupResult.IN_MATCHED_CACHE) {
+ result = super.isMatch(resource);
+
+ // update the cache with the result of the match
+ if(result) {
+ cache.add(resource, RangerResourceAccessCache.CacheType.MATCHED_CACHE);
+ } else {
+ cache.add(resource, RangerResourceAccessCache.CacheType.NOTMATCHED_CACHE);
+ }
+ } else {
+ result = true;
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerCachedPolicyEvaluator.isMatch(" + resource + "): " + result);
+ }
+
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/04536c6c/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCache.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCache.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCache.java
new file mode 100644
index 0000000..55f3a04
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCache.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ranger.plugin.policyevaluator;
+
+import org.apache.ranger.plugin.policyengine.RangerAccessResource;
+
+public interface RangerResourceAccessCache {
+ public enum LookupResult {
+ IN_MATCHED_CACHE,
+ IN_NOTMATCHED_CACHE,
+ NOT_FOUND,
+ ERROR
+ }
+
+ public enum CacheType {
+ MATCHED_CACHE,
+ NOTMATCHED_CACHE
+ }
+
+ LookupResult lookup(RangerAccessResource resource);
+
+ void add(RangerAccessResource resource, CacheType cacheType);
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/04536c6c/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCacheImpl.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCacheImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCacheImpl.java
new file mode 100644
index 0000000..3388361
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCacheImpl.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ranger.plugin.policyevaluator;
+
+
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.policyengine.CacheMap;
+import org.apache.ranger.plugin.policyengine.RangerAccessResource;
+
+
+public class RangerResourceAccessCacheImpl implements RangerResourceAccessCache {
+ private static final Log LOG = LogFactory.getLog(RangerResourceAccessCacheImpl.class);
+
+ public synchronized static RangerResourceAccessCache getInstance(RangerServiceDef serviceDef, RangerPolicy policy) {
+ return new RangerResourceAccessCacheImpl(serviceDef, policy);
+ }
+
+ private RangerServiceDef serviceDef = null;
+
+ private Map<String, String> matchedResourceCache = null;
+ private Map<String, String> notMatchedResourceCache = null;
+
+ private RangerResourceAccessCacheImpl(RangerServiceDef serviceDef, RangerPolicy policy) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerResourceAccessCacheImpl.constructor(), policyName:" + policy.getName());
+ }
+
+ int matchedCacheSize = RangerConfiguration.getInstance().getInt("ranger.policyengine.matched.cached.count", 1000);
+ int notMatchedCacheSize = RangerConfiguration.getInstance().getInt("ranger.policyengine.not.matched.cached.count", matchedCacheSize * 10);
+
+ this.serviceDef = serviceDef;
+
+ matchedResourceCache = new CacheMap<String, String>(matchedCacheSize);
+ notMatchedResourceCache = new CacheMap<String, String>(notMatchedCacheSize);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerResourceAccessCacheImpl.constructor(), policyName:" + policy.getName());
+ }
+ }
+
+ @Override
+ public LookupResult lookup(RangerAccessResource resource) {
+ String strResource = resource.getAsString(serviceDef);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerResourceAccessCacheImpl.lookup(" + strResource + ")");
+ }
+
+ LookupResult result = LookupResult.NOT_FOUND;
+
+ try {
+ synchronized(this) {
+ if (matchedResourceCache.containsKey(strResource)) {
+ result = LookupResult.IN_MATCHED_CACHE;
+ } else if(notMatchedResourceCache.containsKey(strResource)) {
+ result = LookupResult.IN_NOTMATCHED_CACHE;
+ }
+ }
+ } catch (Exception exception) {
+ result = LookupResult.ERROR;
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerResourceAccessCacheImpl.lookup(" + strResource + "): " + result);
+ }
+
+ return result;
+ }
+
+ @Override
+ public void add(RangerAccessResource resource, CacheType cacheType) {
+ String strResource = resource.getAsString(serviceDef);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerResourceAccessCacheImpl.add(" + strResource + ", " + cacheType + ")");
+ }
+
+ synchronized(this) {
+ switch (cacheType) {
+ case MATCHED_CACHE:
+ matchedResourceCache.put(strResource, strResource);
+ break;
+
+ case NOTMATCHED_CACHE:
+ notMatchedResourceCache.put(strResource, strResource);
+ break;
+ default:
+ break;
+ }
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerResourceAccessCacheImpl.add(" + strResource + ", " + cacheType + ")");
+ }
+ }
+}
[2/2] incubator-ranger git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-ranger
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-ranger
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/b3dc07ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/b3dc07ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/b3dc07ce
Branch: refs/heads/master
Commit: b3dc07ce8523af3aeedd671668abef773f60e897
Parents: 04536c6 87b6d4b
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Fri Mar 20 22:13:11 2015 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Mar 20 22:13:11 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------