You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ab...@apache.org on 2019/03/06 21:20:16 UTC
[ranger] branch master updated: RANGER-2341: Support for
Incremental policy updates to improve performance of ranger-admin and
plugins by optimal building of policy-engine
This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 0f229b0 RANGER-2341: Support for Incremental policy updates to improve performance of ranger-admin and plugins by optimal building of policy-engine
0f229b0 is described below
commit 0f229b01e23b12d0c9e0c4ee3de817ce80d68a17
Author: Abhay Kulkarni <>
AuthorDate: Wed Mar 6 13:20:09 2019 -0800
RANGER-2341: Support for Incremental policy updates to improve performance of ranger-admin and plugins by optimal building of policy-engine
---
.../ranger/admin/client/RangerAdminRESTClient.java | 26 +-
.../plugin/contextenricher/RangerTagEnricher.java | 7 +-
.../apache/ranger/plugin/model/RangerPolicy.java | 9 +
.../ranger/plugin/model/RangerPolicyDelta.java | 100 ++++
.../model/validation/RangerServiceDefHelper.java | 13 +-
.../plugin/policyengine/RangerPolicyEngine.java | 3 +
.../policyengine/RangerPolicyEngineCache.java | 26 +-
.../policyengine/RangerPolicyEngineImpl.java | 251 +++++++---
.../policyengine/RangerPolicyEngineOptions.java | 20 +-
.../policyengine/RangerPolicyRepository.java | 512 +++++++++++++++++++--
.../RangerDefaultPolicyEvaluator.java | 12 +-
.../RangerDefaultPolicyResourceMatcher.java | 7 +-
.../ranger/plugin/service/RangerAuthContext.java | 6 +
.../ranger/plugin/service/RangerBasePlugin.java | 99 +++-
.../apache/ranger/plugin/store/ServiceStore.java | 6 +-
.../apache/ranger/plugin/util/PolicyRefresher.java | 10 +-
.../ranger/plugin/util/RangerPolicyDeltaUtil.java | 156 +++++++
.../ranger/plugin/util/RangerRESTClient.java | 2 +-
.../apache/ranger/plugin/util/RangerRESTUtils.java | 1 +
.../ranger/plugin/util/RangerResourceTrie.java | 272 ++++++++++-
.../apache/ranger/plugin/util/ServicePolicies.java | 71 ++-
.../plugin/policyengine/TestPolicyEngine.java | 47 +-
.../test_policyengine_hive_incremental_add.json | 345 ++++++++++++++
.../test_policyengine_hive_incremental_delete.json | 352 ++++++++++++++
.../test_policyengine_hive_incremental_update.json | 351 ++++++++++++++
.../admin/client/RangerAdminJersey2RESTClient.java | 9 +-
.../optimized/current/ranger_core_db_mysql.sql | 19 +-
.../patches/038-add-policy-change-log-table.sql | 33 ++
.../optimized/current/ranger_core_db_oracle.sql | 53 ++-
.../patches/038-add-policy-change-log-table.sql | 57 +++
.../optimized/current/ranger_core_db_postgres.sql | 28 +-
.../patches/038-add-policy-change-log-table.sql | 35 ++
.../current/ranger_core_db_sqlanywhere.sql | 21 +
.../patches/038-add-policy-change-log-table.sql | 64 +++
.../optimized/current/ranger_core_db_sqlserver.sql | 35 ++
.../patches/038-add-policy-change-log-table.sql | 56 +++
.../java/org/apache/ranger/biz/ServiceDBStore.java | 447 +++++++++++++++---
.../ranger/common/RangerServicePoliciesCache.java | 278 +++++++----
.../apache/ranger/common/db/JPABeanCallbacks.java | 8 +-
.../org/apache/ranger/db/RangerDaoManagerBase.java | 3 +
.../org/apache/ranger/db/XXPolicyChangeLogDao.java | 161 +++++++
.../java/org/apache/ranger/db/XXServiceDao.java | 9 +
.../apache/ranger/entity/XXPolicyChangeLog.java | 233 ++++++++++
.../java/org/apache/ranger/rest/AssetREST.java | 2 +-
.../java/org/apache/ranger/rest/PublicAPIsv2.java | 15 +
.../java/org/apache/ranger/rest/ServiceREST.java | 143 ++++--
.../ranger/service/RangerSecurityZoneService.java | 3 +-
.../main/resources/META-INF/jpa_named_queries.xml | 32 +-
.../org/apache/ranger/biz/TestServiceDBStore.java | 2 +-
.../java/org/apache/ranger/rest/TestAssetREST.java | 2 +-
.../org/apache/ranger/rest/TestServiceREST.java | 20 +-
51 files changed, 4085 insertions(+), 387 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
index dddfbc7..b6a9380 100644
--- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
+++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
@@ -53,6 +53,7 @@ public class RangerAdminRESTClient implements RangerAdminClient {
private String clusterName;
private RangerRESTClient restClient;
private RangerRESTUtils restUtils = new RangerRESTUtils();
+ private String supportsPolicyDeltas = "true";
public static <T> GenericType<List<T>> getGenericType(final T clazz) {
@@ -84,26 +85,32 @@ public class RangerAdminRESTClient implements RangerAdminClient {
clusterName = RangerConfiguration.getInstance().get(propertyPrefix + ".ambari.cluster.name", "");
int restClientConnTimeOutMs = RangerConfiguration.getInstance().getInt(propertyPrefix + ".policy.rest.client.connection.timeoutMs", 120 * 1000);
int restClientReadTimeOutMs = RangerConfiguration.getInstance().getInt(propertyPrefix + ".policy.rest.client.read.timeoutMs", 30 * 1000);
+ supportsPolicyDeltas = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.supports.policy.deltas", "false");
if (!StringUtil.isEmpty(tmpUrl)) {
url = tmpUrl.trim();
}
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
+ if (!"true".equalsIgnoreCase(supportsPolicyDeltas)) {
+ supportsPolicyDeltas = "false";
+ }
init(url, sslConfigFileName, restClientConnTimeOutMs , restClientReadTimeOutMs);
}
@Override
public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception {
- if(LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerAdminRESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")");
}
- ServicePolicies ret = null;
- UserGroupInformation user = MiscUtil.getUGILoginUser();
- boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled();
- ClientResponse response = null;
+ final ServicePolicies ret;
+
+ final UserGroupInformation user = MiscUtil.getUGILoginUser();
+ final boolean isSecureMode = user != null && UserGroupInformation.isSecurityEnabled();
+ final ClientResponse response;
+
if (isSecureMode) {
if (LOG.isDebugEnabled()) {
LOG.debug("Checking Service policy if updated as user : " + user);
@@ -114,7 +121,8 @@ public class RangerAdminRESTClient implements RangerAdminClient {
.queryParam(RangerRESTUtils.REST_PARAM_LAST_KNOWN_POLICY_VERSION, Long.toString(lastKnownVersion))
.queryParam(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis))
.queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId)
- .queryParam(RangerRESTUtils.REST_PARAM_CLUSTER_NAME, clusterName);
+ .queryParam(RangerRESTUtils.REST_PARAM_CLUSTER_NAME, clusterName)
+ .queryParam(RangerRESTUtils.REST_PARAM_SUPPORTS_POLICY_DELTAS, supportsPolicyDeltas);
return secureWebResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class);
}
};
@@ -127,7 +135,8 @@ public class RangerAdminRESTClient implements RangerAdminClient {
.queryParam(RangerRESTUtils.REST_PARAM_LAST_KNOWN_POLICY_VERSION, Long.toString(lastKnownVersion))
.queryParam(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis))
.queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId)
- .queryParam(RangerRESTUtils.REST_PARAM_CLUSTER_NAME, clusterName);
+ .queryParam(RangerRESTUtils.REST_PARAM_CLUSTER_NAME, clusterName)
+ .queryParam(RangerRESTUtils.REST_PARAM_SUPPORTS_POLICY_DELTAS, supportsPolicyDeltas);
response = webResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class);
}
@@ -148,6 +157,7 @@ public class RangerAdminRESTClient implements RangerAdminClient {
+ ", response=" + response.getStatus() + ", serviceName=" + serviceName
+ ", " + "lastKnownVersion=" + lastKnownVersion
+ ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis);
+ ret = null;
String exceptionMsg = response.hasEntity() ? response.getEntity(String.class) : null;
RangerServiceNotFoundException.throwExceptionIfServiceNotFound(serviceName, exceptionMsg);
@@ -159,7 +169,7 @@ public class RangerAdminRESTClient implements RangerAdminClient {
ret = null;
}
- if(LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
LOG.debug("<== RangerAdminRESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + ret);
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index 2a0797c..028efe8 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -178,6 +178,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
LOG.debug("<== RangerTagEnricher.enrich(" + request + ") with dataStore:[" + dataStore + "]): tags count=" + (matchedTags == null ? 0 : matchedTags.size()));
}
}
+
/*
* This class implements a cache of result of look-up of keyset of policy-resources for each of the collections of hierarchies
* for policy types: access, datamask and rowfilter. If a keyset is examined for validity in a hierarchy of a policy-type,
@@ -314,12 +315,12 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
public boolean preCleanup() {
boolean ret = true;
- super.preCleanup();
-
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerTagEnricher.preCleanup()");
}
+ super.preCleanup();
+
if (tagRefresher != null) {
tagRefresher.cleanup();
tagRefresher = null;
@@ -548,7 +549,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
this.cacheFile = cacheFile;
this.pollingIntervalMs = pollingIntervalMs;
try {
- gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create();
+ gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create();
} catch(Throwable excp) {
LOG.fatal("failed to create GsonBuilder object", excp);
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java
index 3bafd5c..327d37b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicy.java
@@ -20,6 +20,7 @@
package org.apache.ranger.plugin.model;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import java.util.HashMap;
import java.util.Map;
@@ -61,6 +62,8 @@ public class RangerPolicy extends RangerBaseModelObject implements java.io.Seria
public static final String POLICY_PRIORITY_NAME_NORMAL = "NORMAL";
public static final String POLICY_PRIORITY_NAME_OVERRIDE = "OVERRIDE";
+ public static final Comparator<RangerPolicy> POLICY_ID_COMPARATOR = new PolicyIdComparator();
+
// For future use
private static final long serialVersionUID = 1L;
@@ -616,6 +619,12 @@ public class RangerPolicy extends RangerBaseModelObject implements java.io.Seria
return sb;
}
+ static class PolicyIdComparator implements Comparator<RangerPolicy>, java.io.Serializable {
+ @Override
+ public int compare(RangerPolicy me, RangerPolicy other) {
+ return Long.compare(me.getId(), other.getId());
+ }
+ }
@JsonAutoDetect(fieldVisibility=Visibility.ANY)
@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java
new file mode 100644
index 0000000..74e7add
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.model;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonAutoDetect(fieldVisibility=Visibility.ANY)
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown=true)
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class RangerPolicyDelta implements java.io.Serializable {
+
+ public static final int CHANGE_TYPE_POLICY_CREATE = 0;
+ public static final int CHANGE_TYPE_POLICY_UPDATE = 1;
+ public static final int CHANGE_TYPE_POLICY_DELETE = 2;
+ public static final int CHANGE_TYPE_SERVICE_CHANGE = 3;
+ public static final int CHANGE_TYPE_SERVICE_DEF_CHANGE = 4;
+ public static final int CHANGE_TYPE_RANGER_ADMIN_START = 5;
+ public static final int CHANGE_TYPE_LOG_ERROR = 6;
+
+ private static String[] changeTypeNames = { "POLICY_CREATE", "POLICY_UPDATE", "POLICY_DELETE", "SERVICE_CHANGE", "SERVICE_DEF_CHANGE", "RANGER_ADMIN_START", "LOG_ERROR" };
+
+ private Long id;
+ private Integer changeType;
+ private RangerPolicy policy;
+
+ public RangerPolicyDelta() {
+ this(null, null, null);
+ }
+
+ public RangerPolicyDelta(final Long id, final Integer changeType, final RangerPolicy policy) {
+ setId(id);
+ setChangeType(changeType);
+ setPolicy(policy);
+ }
+ public Long getId() { return id; }
+
+ public Integer getChangeType() { return changeType; }
+
+ @JsonIgnore
+ public Long getPolicyVersion() { return policy != null ? policy.getVersion() : null; }
+
+ @JsonIgnore
+ public String getServiceType() { return policy != null ? policy.getServiceType() : null; }
+
+ @JsonIgnore
+ public Integer getPolicyType() { return policy != null ? policy.getPolicyType() : null; }
+
+ @JsonIgnore
+ public Long getPolicyId() { return policy != null ? policy.getId() : null; }
+
+ @JsonIgnore
+ public String getZoneName() { return policy != null ? policy.getZoneName() : null; }
+
+ public RangerPolicy getPolicy() { return policy; }
+
+ public void setId(Long id) { this.id = id;}
+
+ private void setChangeType(Integer changeType) { this.changeType = changeType; }
+
+ public void setPolicy(RangerPolicy policy) { this.policy = policy; }
+
+ @Override
+ public String toString() {
+ return "id:" + id
+ + ", changeType:" + changeTypeNames[changeType]
+ + ", policyVersion:" + getPolicyVersion()
+ + ", serviceType:" + getServiceType()
+ + ", policyType:" + getPolicyType()
+ + ", policyId:[" + getPolicyId() + "]"
+ + ", policy:[" + policy +"]";
+ }
+
+}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
index 6df5d8d..0954beb 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
@@ -196,6 +196,9 @@ public class RangerServiceDefHelper {
}
public Set<List<RangerResourceDef>> getResourceHierarchies(Integer policyType, Collection<String> keys) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> getResourceHierarchies(policyType=" + policyType + ", keys=" + StringUtils.join(keys, ",") + ")");
+ }
Set<List<RangerResourceDef>> ret = new HashSet<List<RangerResourceDef>>();
@@ -205,10 +208,16 @@ public class RangerServiceDefHelper {
}
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== getResourceHierarchies(policyType=" + policyType + ", keys=" + StringUtils.join(keys, ",") + ") : " + StringUtils.join(ret, ","));
+ }
return ret;
}
public boolean hierarchyHasAllResources(List<RangerResourceDef> hierarchy, Collection<String> resourceNames) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> hierarchyHasAllResources(hierarchy=" + StringUtils.join(hierarchy, ",") + ", resourceNames=" + StringUtils.join(resourceNames, ",") + ")");
+ }
boolean foundAllResourceKeys = true;
for (String resourceKey : resourceNames) {
@@ -226,7 +235,9 @@ public class RangerServiceDefHelper {
break;
}
}
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== hierarchyHasAllResources(hierarchy=" + StringUtils.join(hierarchy, ",") + ", resourceNames=" + StringUtils.join(resourceNames, ",") + "): " + foundAllResourceKeys);
+ }
return foundAllResourceKeys;
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
index 63fcbd0..9ed500c 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
@@ -29,6 +29,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.util.GrantRevokeRequest;
import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
+import org.apache.ranger.plugin.util.ServicePolicies;
public interface RangerPolicyEngine {
String GROUP_PUBLIC = "public";
@@ -90,4 +91,6 @@ public interface RangerPolicyEngine {
List<RangerPolicy> getAllowedPolicies(String user, Set<String> userGroups, String accessType);
+ RangerPolicyEngine cloneWithDelta(ServicePolicies servicePolicies);
+
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java
index 8642dbe..c1a7977 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineCache.java
@@ -22,6 +22,7 @@ package org.apache.ranger.plugin.policyengine;
import java.util.HashMap;
import java.util.Map;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.plugin.store.ServiceStore;
@@ -42,13 +43,13 @@ class RangerPolicyEngineCache {
if(svcStore != null) {
try {
- ServicePolicies policies = svcStore.getServicePoliciesIfUpdated(serviceName, policyVersion);
+ ServicePolicies policies = svcStore.getServicePoliciesIfUpdated(serviceName, policyVersion, false);
if(policies != null) {
if(ret == null) {
ret = addPolicyEngine(policies, options);
} else if(policies.getPolicyVersion() != null && !policies.getPolicyVersion().equals(policyVersion)) {
- ret = addPolicyEngine(policies, options);
+ ret = updatePolicyEngine(ret, policies, options);
}
}
} catch(Exception excp) {
@@ -68,4 +69,25 @@ class RangerPolicyEngineCache {
return ret;
}
+
+ private RangerPolicyEngine updatePolicyEngine(RangerPolicyEngine policyEngine, ServicePolicies policies, RangerPolicyEngineOptions options) {
+ final RangerPolicyEngine ret;
+
+
+ if (CollectionUtils.isNotEmpty(policies.getPolicyDeltas())) {
+ RangerPolicyEngine updatedEngine = policyEngine.cloneWithDelta(policies);
+ if (updatedEngine != null) {
+ policyEngineCache.put(policies.getServiceName(), updatedEngine);
+ ret = updatedEngine;
+ } else {
+ LOG.warn("Could not cloneWithDelta policyEngine to policyVersion:[" + policies.getPolicyVersion() + "]");
+ LOG.warn("Retaining old policyEngine with policyVersion:[" + policyEngine.getPolicyVersion() + "]");
+ ret = policyEngine;
+ }
+ } else {
+ ret = addPolicyEngine(policies, options);
+ }
+
+ return ret;
+ }
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
index b29f152..e239c89 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
@@ -28,6 +28,7 @@ import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.contextenricher.RangerContextEnricher;
import org.apache.ranger.plugin.contextenricher.RangerTagForEval;
import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.validation.RangerZoneResourceMatcher;
@@ -39,11 +40,12 @@ import org.apache.ranger.plugin.util.GrantRevokeRequest;
import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
import org.apache.ranger.plugin.util.RangerPerfTracer;
import org.apache.ranger.plugin.util.RangerResourceTrie;
+import org.apache.ranger.plugin.util.RangerPolicyDeltaUtil;
import org.apache.ranger.plugin.util.ServicePolicies;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@@ -68,10 +70,11 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
private final RangerPolicyRepository policyRepository;
private final RangerPolicyRepository tagPolicyRepository;
-
- private List<RangerContextEnricher> allContextEnrichers;
- private final Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
+ private boolean isPolicyRepositoryShared = false;
+ private boolean isTagPolicyRepositoryShared = false;
+
+ private List<RangerContextEnricher> allContextEnrichers;
private boolean useForwardedIPAddress;
private String[] trustedProxyAddresses;
@@ -80,6 +83,127 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
private Map<String, RangerResourceTrie> trieMap;
+ public RangerPolicyEngineImpl(final RangerPolicyEngineImpl other, ServicePolicies servicePolicies) {
+
+ List<RangerPolicyDelta> deltas = servicePolicies.getPolicyDeltas();
+ long policyVersion = servicePolicies.getPolicyVersion();
+
+ this.useForwardedIPAddress = other.useForwardedIPAddress;
+ this.trustedProxyAddresses = other.trustedProxyAddresses;
+
+ List<RangerPolicyDelta> defaultZoneDeltas = new ArrayList<>();
+ List<RangerPolicyDelta> defaultZoneDeltasForTagPolicies = new ArrayList<>();
+
+ if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) {
+ Map<String, List<RangerPolicyDelta>> zoneDeltasMap = new HashMap<>();
+
+ buildZoneTrie(servicePolicies);
+
+ for (Map.Entry<String, ServicePolicies.SecurityZoneInfo> zone : servicePolicies.getSecurityZones().entrySet()) {
+ zoneDeltasMap.put(zone.getKey(), new ArrayList<>());
+ }
+ for (RangerPolicyDelta delta : deltas) {
+ String zoneName = delta.getZoneName();
+
+ if (StringUtils.isNotEmpty(zoneName)) {
+ List<RangerPolicyDelta> zoneDeltas = zoneDeltasMap.get(zoneName);
+ if (zoneDeltas != null) {
+ zoneDeltas.add(delta);
+ }
+ } else {
+ if (servicePolicies.getServiceDef().getName().equals(delta.getServiceType())) {
+ defaultZoneDeltas.add(delta);
+ } else {
+ defaultZoneDeltasForTagPolicies.add(delta);
+ }
+ }
+ }
+ for (Map.Entry<String, ServicePolicies.SecurityZoneInfo> zone : servicePolicies.getSecurityZones().entrySet()) {
+ final String zoneName = zone.getKey();
+ List<RangerPolicyDelta> zoneDeltas = zoneDeltasMap.get(zoneName);
+
+ RangerPolicyRepository otherRepository = other.policyRepositories.get(zoneName);
+ final RangerPolicyRepository policyRepository;
+
+ if (CollectionUtils.isNotEmpty(zoneDeltas)) {
+ if (otherRepository == null) {
+ List<RangerPolicy> policies = new ArrayList<>();
+ for (RangerPolicyDelta delta : zoneDeltas) {
+ if (delta.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE) {
+ policies.add(delta.getPolicy());
+ } else {
+ LOG.warn("Expected changeType:[" + RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE + "], found policy-change-delta:[" + delta +"]");
+ }
+ }
+ servicePolicies.getSecurityZones().get(zoneName).setPolicies(policies);
+
+ policyRepository = new RangerPolicyRepository(other.policyRepository.getAppId(), servicePolicies, other.policyRepository.getOptions(), zoneName);
+ } else {
+ policyRepository = new RangerPolicyRepository(otherRepository, zoneDeltas, policyVersion);
+ }
+ } else {
+ policyRepository = otherRepository;
+ }
+
+ policyRepositories.put(zoneName, policyRepository);
+ }
+ } else {
+ for (RangerPolicyDelta delta : deltas) {
+ if (servicePolicies.getServiceDef().getName().equals(delta.getServiceType())) {
+ defaultZoneDeltas.add(delta);
+ } else {
+ defaultZoneDeltasForTagPolicies.add(delta);
+ }
+ }
+ }
+
+ if (other.policyRepository != null && CollectionUtils.isNotEmpty(defaultZoneDeltas)) {
+ this.policyRepository = new RangerPolicyRepository(other.policyRepository, defaultZoneDeltas, policyVersion);
+ } else {
+ this.policyRepository = other.policyRepository;
+ other.isPolicyRepositoryShared = true;
+ }
+ if (CollectionUtils.isNotEmpty(defaultZoneDeltasForTagPolicies)) {
+ if (other.tagPolicyRepository != null) {
+ this.tagPolicyRepository = new RangerPolicyRepository(other.tagPolicyRepository, defaultZoneDeltasForTagPolicies, policyVersion);
+ } else {
+ // Only creates are expected
+ List<RangerPolicy> tagPolicies = new ArrayList<>();
+ for (RangerPolicyDelta delta : defaultZoneDeltasForTagPolicies) {
+ if (delta.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE) {
+ tagPolicies.add(delta.getPolicy());
+ } else {
+ LOG.warn("Expected changeType:[" + RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE + "], found policy-change-delta:[" + delta +"]");
+ }
+ }
+ servicePolicies.getTagPolicies().setPolicies(tagPolicies);
+ this.tagPolicyRepository = new RangerPolicyRepository(other.policyRepository.getAppId(), servicePolicies.getTagPolicies(), other.policyRepository.getOptions(), servicePolicies.getServiceDef(), servicePolicies.getServiceName());
+
+ }
+ } else {
+ this.tagPolicyRepository = other.tagPolicyRepository;
+ other.isTagPolicyRepositoryShared = true;
+ }
+
+ List<RangerContextEnricher> tmpList;
+
+ List<RangerContextEnricher> tagContextEnrichers = tagPolicyRepository == null ? null :tagPolicyRepository.getContextEnrichers();
+ List<RangerContextEnricher> resourceContextEnrichers = policyRepository.getContextEnrichers();
+
+ if (CollectionUtils.isEmpty(tagContextEnrichers)) {
+ tmpList = resourceContextEnrichers;
+ } else if (CollectionUtils.isEmpty(resourceContextEnrichers)) {
+ tmpList = tagContextEnrichers;
+ } else {
+ tmpList = new ArrayList<>(tagContextEnrichers);
+ tmpList.addAll(resourceContextEnrichers);
+ }
+ this.allContextEnrichers = tmpList;
+
+ reorderPolicyEvaluators();
+
+ }
+
public RangerPolicyEngineImpl(String appId, ServicePolicies servicePolicies, RangerPolicyEngineOptions options) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyEngineImpl(" + appId + ", " + servicePolicies + ", " + options + ")");
@@ -158,7 +282,13 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
this.allContextEnrichers = tmpList;
- policyEvaluatorsMap = createPolicyEvaluatorsMap();
+ if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) {
+ buildZoneTrie(servicePolicies);
+ for (Map.Entry<String, ServicePolicies.SecurityZoneInfo> zone : servicePolicies.getSecurityZones().entrySet()) {
+ RangerPolicyRepository policyRepository = new RangerPolicyRepository(appId, servicePolicies, options, zone.getKey());
+ policyRepositories.put(zone.getKey(), policyRepository);
+ }
+ }
if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) {
buildZoneTrie(servicePolicies);
@@ -182,6 +312,33 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
}
@Override
+ public RangerPolicyEngine cloneWithDelta(ServicePolicies servicePolicies) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> cloneWithDelta(" + Arrays.toString(servicePolicies.getPolicyDeltas().toArray()) + ", " + servicePolicies.getPolicyVersion() + ")");
+ }
+ final RangerPolicyEngineImpl ret;
+
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_INIT_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_INIT_LOG, "RangerPolicyEngine.cloneWithDelta()");
+ }
+
+ if (CollectionUtils.isNotEmpty(servicePolicies.getPolicyDeltas()) && RangerPolicyDeltaUtil.isValidDeltas(servicePolicies.getPolicyDeltas(), this.getServiceDef().getName())) {
+ ret = new RangerPolicyEngineImpl(this, servicePolicies);
+ } else {
+ ret = null;
+ }
+
+ RangerPerfTracer.log(perf);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== cloneWithDelta(" + Arrays.toString(servicePolicies.getPolicyDeltas().toArray()) + ", " + servicePolicies.getPolicyVersion() + ")");
+ }
+ return ret;
+ }
+
+ @Override
protected void finalize() throws Throwable {
try {
cleanup();
@@ -519,14 +676,11 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
LOG.debug("==> RangerPolicyEngineImpl.preCleanup()");
}
- if (CollectionUtils.isNotEmpty(allContextEnrichers)) {
- for (RangerContextEnricher contextEnricher : allContextEnrichers) {
- boolean readyForCleanup = contextEnricher.preCleanup();
- if (!readyForCleanup) {
- LOG.warn("contextEnricher.preCleanup() failed for contextEnricher=" + contextEnricher.getName());
- ret = false;
- }
- }
+ if (policyRepository != null && !isPolicyRepositoryShared) {
+ policyRepository.preCleanup();
+ }
+ if (tagPolicyRepository != null && !isTagPolicyRepositoryShared) {
+ tagPolicyRepository.preCleanup();
}
if (LOG.isDebugEnabled()) {
@@ -550,13 +704,12 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
}
preCleanup();
- if (CollectionUtils.isNotEmpty(allContextEnrichers)) {
- for (RangerContextEnricher contextEnricher : allContextEnrichers) {
- contextEnricher.cleanup();
- }
+ if (policyRepository != null && !isPolicyRepositoryShared) {
+ policyRepository.cleanup();
+ }
+ if (tagPolicyRepository != null && !isTagPolicyRepositoryShared) {
+ tagPolicyRepository.cleanup();
}
-
- this.allContextEnrichers = null;
RangerPerfTracer.log(perf);
@@ -575,8 +728,13 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REBALANCE_LOG)) {
perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderEvaluators()");
}
- if (MapUtils.isNotEmpty(policyEvaluatorsMap)) {
- for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) {
+ if (tagPolicyRepository != null && MapUtils.isNotEmpty(tagPolicyRepository.getPolicyEvaluatorsMap())) {
+ for (Map.Entry<Long, RangerPolicyEvaluator> entry : tagPolicyRepository.getPolicyEvaluatorsMap().entrySet()) {
+ entry.getValue().setUsageCountImmutable();
+ }
+ }
+ if (policyRepository != null && MapUtils.isNotEmpty(policyRepository.getPolicyEvaluatorsMap())) {
+ for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyRepository.getPolicyEvaluatorsMap().entrySet()) {
entry.getValue().setUsageCountImmutable();
}
}
@@ -588,8 +746,13 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
policyRepository.reorderPolicyEvaluators();
}
- if (MapUtils.isNotEmpty(policyEvaluatorsMap)) {
- for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) {
+ if (tagPolicyRepository != null && MapUtils.isNotEmpty(tagPolicyRepository.getPolicyEvaluatorsMap())) {
+ for (Map.Entry<Long, RangerPolicyEvaluator> entry : tagPolicyRepository.getPolicyEvaluatorsMap().entrySet()) {
+ entry.getValue().resetUsageCount();
+ }
+ }
+ if (policyRepository != null && MapUtils.isNotEmpty(policyRepository.getPolicyEvaluatorsMap())) {
+ for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyRepository.getPolicyEvaluatorsMap().entrySet()) {
entry.getValue().resetUsageCount();
}
}
@@ -883,7 +1046,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
for (RangerPolicyEvaluator evaluator : likelyEvaluators) {
RangerPolicyResourceMatcher matcher = evaluator.getPolicyResourceMatcher();
if (matcher != null &&
- (request.isAccessTypeAny() ? matcher.isMatch(tagResource, RangerPolicyResourceMatcher.MatchScope.ANY, null) : matcher.isMatch(tagResource, null))) {
+ (request.isAccessTypeAny() ? matcher.isMatch(tagResource, RangerPolicyResourceMatcher.MatchScope.ANY, null) : matcher.isMatch(tagResource, null))) {
ret.add(evaluator.getPolicy());
}
}
@@ -900,11 +1063,10 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
for (RangerPolicyEvaluator evaluator : likelyEvaluators) {
RangerPolicyResourceMatcher matcher = evaluator.getPolicyResourceMatcher();
if (matcher != null &&
- (request.isAccessTypeAny() ? matcher.isMatch(request.getResource(), RangerPolicyResourceMatcher.MatchScope.ANY, null) : matcher.isMatch(request.getResource(), null))) {
+ (request.isAccessTypeAny() ? matcher.isMatch(request.getResource(), RangerPolicyResourceMatcher.MatchScope.ANY, null) : matcher.isMatch(request.getResource(), null))) {
ret.add(evaluator.getPolicy());
}
}
-
}
}
@@ -1055,6 +1217,10 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
return ret;
}
+ public List<RangerPolicy> getResourcePolicies() { return policyRepository == null ? null : policyRepository.getPolicies(); }
+
+ public List<RangerPolicy> getTagPolicies() { return tagPolicyRepository == null ? null : tagPolicyRepository.getPolicies(); }
+
private RangerAccessResult zoneAwareAccessEvaluationWithNoAudit(RangerAccessRequest request, int policyType) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyEngineImpl.zoneAwareAccessEvaluationWithNoAudit(" + request + ", policyType =" + policyType + ")");
@@ -1250,7 +1416,11 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
}
private RangerPolicyEvaluator getPolicyEvaluator(Long id) {
- return policyEvaluatorsMap.get(id);
+ RangerPolicyEvaluator ret = policyRepository.getPolicyEvaluator(id);
+ if (ret == null && tagPolicyRepository != null) {
+ ret = tagPolicyRepository.getPolicyEvaluator(id);
+ }
+ return ret;
}
private RangerAccessResult createAccessResult(RangerAccessRequest request, int policyType) {
@@ -1292,33 +1462,6 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies());
}
- private Map<Long, RangerPolicyEvaluator> createPolicyEvaluatorsMap() {
- Map<Long, RangerPolicyEvaluator> tmpPolicyEvaluatorMap = new HashMap<>();
-
- if (tagPolicyRepository != null) {
- for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getPolicyEvaluators()) {
- tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
- }
- for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getDataMaskPolicyEvaluators()) {
- tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
- }
- for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getRowFilterPolicyEvaluators()) {
- tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
- }
- }
- for (RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) {
- tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
- }
- for (RangerPolicyEvaluator evaluator : policyRepository.getDataMaskPolicyEvaluators()) {
- tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
- }
- for (RangerPolicyEvaluator evaluator : policyRepository.getRowFilterPolicyEvaluators()) {
- tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
- }
-
- return Collections.unmodifiableMap(tmpPolicyEvaluatorMap);
- }
-
private void updatePolicyUsageCounts(RangerAccessRequest accessRequest, RangerAccessResult accessResult) {
boolean auditCountUpdated = false;
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
index 5498545..1f6aed9 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
@@ -37,6 +37,21 @@ public class RangerPolicyEngineOptions {
private RangerServiceDefHelper serviceDefHelper;
+ public RangerPolicyEngineOptions() {}
+
+ public RangerPolicyEngineOptions(final RangerPolicyEngineOptions other) {
+ this.disableContextEnrichers = other.disableContextEnrichers;
+ this.disableCustomConditions = other.disableCustomConditions;
+ this.disableTagPolicyEvaluation = other.disableTagPolicyEvaluation;
+ this.disableTrieLookupPrefilter = other.disableTrieLookupPrefilter;
+ this.cacheAuditResults = other.cacheAuditResults;
+ this.evaluateDelegateAdminOnly = other.evaluateDelegateAdminOnly;
+ this.enableTagEnricherWithLocalRefresher = other.enableTagEnricherWithLocalRefresher;
+ this.disableAccessEvaluationWithPolicyACLSummary = other.disableAccessEvaluationWithPolicyACLSummary;
+ this.optimizeTrieForRetrieval = other.optimizeTrieForRetrieval;
+ this.serviceDefHelper = null;
+ }
+
public void configureForPlugin(Configuration conf, String propertyPrefix) {
disableContextEnrichers = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false);
disableCustomConditions = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false);
@@ -152,11 +167,14 @@ public class RangerPolicyEngineOptions {
public String toString() {
return "PolicyEngineOptions: {" +
" evaluatorType: " + evaluatorType +
- ", cacheAuditResult: " + cacheAuditResults +
+ ", evaluateDelegateAdminOnly: " + evaluateDelegateAdminOnly +
", disableContextEnrichers: " + disableContextEnrichers +
", disableCustomConditions: " + disableContextEnrichers +
+ ", disableTagPolicyEvaluation: " + disableTagPolicyEvaluation +
+ ", enableTagEnricherWithLocalRefresher: " + enableTagEnricherWithLocalRefresher +
", disableTrieLookupPrefilter: " + disableTrieLookupPrefilter +
", optimizeTrieForRetrieval: " + optimizeTrieForRetrieval +
+ ", cacheAuditResult: " + cacheAuditResults +
" }";
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index e5c8d0c..8b51c63 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -29,6 +29,7 @@ import org.apache.ranger.plugin.contextenricher.RangerTagEnricher;
import org.apache.ranger.plugin.contextenricher.RangerTagForEval;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
import org.apache.ranger.plugin.policyevaluator.RangerCachedPolicyEvaluator;
@@ -77,14 +78,17 @@ class RangerPolicyRepository {
}
private final String serviceName;
+ private final String zoneName;
private final String appId;
+ private final RangerPolicyEngineOptions options;
private final RangerServiceDef serviceDef;
private final List<RangerPolicy> policies;
private final long policyVersion;
- private List<RangerContextEnricher> contextEnrichers;
+ private final List<RangerContextEnricher> contextEnrichers;
private List<RangerPolicyEvaluator> policyEvaluators;
private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators;
private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators;
+ private Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
private final AuditModeEnum auditModeEnum;
private final Map<String, AuditInfo> accessAuditCache;
@@ -94,6 +98,156 @@ class RangerPolicyRepository {
private final Map<String, RangerResourceTrie> dataMaskResourceTrie;
private final Map<String, RangerResourceTrie> rowFilterResourceTrie;
+ private boolean isContextEnrichersShared = false;
+
+ RangerPolicyRepository(final RangerPolicyRepository other, final List<RangerPolicyDelta> deltas, long policyVersion) {
+
+ this.serviceName = other.serviceName;
+ this.zoneName = other.zoneName;
+ this.appId = other.appId;
+ this.options = other.options;
+ this.serviceDef = other.serviceDef;
+ this.policies = new ArrayList<>(other.policies);
+ this.policyEvaluators = new ArrayList<>(other.policyEvaluators);
+ this.dataMaskPolicyEvaluators = new ArrayList<>(other.dataMaskPolicyEvaluators);
+ this.rowFilterPolicyEvaluators = new ArrayList<>(other.rowFilterPolicyEvaluators);
+ this.auditModeEnum = other.auditModeEnum;
+ this.componentServiceName = other.componentServiceName;
+ this.componentServiceDef = other.componentServiceDef;
+ this.policyEvaluatorsMap = new HashMap<>(other.policyEvaluatorsMap);
+
+ if (other.policyResourceTrie != null) {
+ this.policyResourceTrie = new HashMap<>();
+ for (Map.Entry<String, RangerResourceTrie> entry : other.policyResourceTrie.entrySet()) {
+ policyResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
+ }
+ } else {
+ this.policyResourceTrie = null;
+ }
+
+ if (other.dataMaskResourceTrie != null) {
+ this.dataMaskResourceTrie = new HashMap<>();
+ for (Map.Entry<String, RangerResourceTrie> entry : other.dataMaskResourceTrie.entrySet()) {
+ dataMaskResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
+ }
+ } else {
+ this.dataMaskResourceTrie = null;
+ }
+
+ if (other.rowFilterResourceTrie != null) {
+ this.rowFilterResourceTrie = new HashMap<>();
+ for (Map.Entry<String, RangerResourceTrie> entry : other.rowFilterResourceTrie.entrySet()) {
+ rowFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
+ }
+ } else {
+ this.rowFilterResourceTrie = null;
+ }
+
+ if (other.accessAuditCache != null) {
+ int auditResultCacheSize = other.accessAuditCache.size();
+ this.accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize));
+ } else {
+ this.accessAuditCache = null;
+ }
+
+ boolean[] flags = new boolean[RangerPolicy.POLICY_TYPES.length];
+
+ for (RangerPolicyDelta delta : deltas) {
+
+ final Integer changeType = delta.getChangeType();
+ final String serviceType = delta.getServiceType();
+ final Long policyId = delta.getPolicyId();
+ final Integer policyType = delta.getPolicyType();
+
+ if (!serviceType.equals(this.serviceDef.getName())) {
+ continue;
+ }
+
+ RangerPolicyEvaluator evaluator = null;
+
+ switch (changeType) {
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE:
+ if (delta.getPolicy() == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not find policy for policy-id:[" + policyId + "]");
+ }
+ continue;
+ }
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE:
+ evaluator = getPolicyEvaluator(policyId);
+ if (evaluator == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not find evaluator for policy-id:[" + policyId + "]");
+ }
+ }
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE:
+ evaluator = getPolicyEvaluator(policyId);
+ if (evaluator == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not find evaluator for policy-id:[" + policyId + "]");
+ }
+ }
+ break;
+ default:
+ LOG.error("Unknown changeType:[" + changeType + "], Ignoring");
+ break;
+ }
+
+ evaluator = update(delta, evaluator);
+
+ if (evaluator != null) {
+ switch (changeType) {
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE:
+ policyEvaluatorsMap.put(policyId, evaluator);
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE:
+ policyEvaluatorsMap.put(policyId, evaluator);
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE:
+ policyEvaluatorsMap.remove(policyId);
+ break;
+ default:
+ break;
+ }
+
+ flags[policyType] = true;
+ }
+ }
+
+ for (int policyType = 0; policyType < flags.length; policyType++) {
+
+ if (flags[policyType]) {
+ Map<String, RangerResourceTrie> trie = getTrie(policyType);
+
+ if (trie != null) {
+ for (Map.Entry<String, RangerResourceTrie> entry : trie.entrySet()) {
+ entry.getValue().wrapUpUpdate();
+ }
+ }
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(other.getPolicies())) {
+ if (CollectionUtils.isNotEmpty(this.getPolicies())) {
+ this.contextEnrichers = other.contextEnrichers;
+ other.isContextEnrichersShared = true;
+ } else {
+ this.contextEnrichers = null;
+ }
+ } else {
+ if (CollectionUtils.isNotEmpty(this.policies)) {
+ this.contextEnrichers = Collections.unmodifiableList(buildContextEnrichers(options));
+ } else {
+ this.contextEnrichers = null;
+ }
+ }
+
+ this.policyVersion = policyVersion;
+
+ }
+
RangerPolicyRepository(String appId, ServicePolicies servicePolicies, RangerPolicyEngineOptions options) {
this(appId, servicePolicies, options, null);
}
@@ -104,7 +258,10 @@ class RangerPolicyRepository {
this.componentServiceName = this.serviceName = servicePolicies.getServiceName();
this.componentServiceDef = this.serviceDef = ServiceDefUtil.normalize(servicePolicies.getServiceDef());
+ this.zoneName = zoneName;
+
this.appId = appId;
+ this.options = new RangerPolicyEngineOptions(options);
if (StringUtils.isEmpty(zoneName)) {
this.policies = Collections.unmodifiableList(servicePolicies.getPolicies());
@@ -144,6 +301,8 @@ class RangerPolicyRepository {
init(options);
+ this.contextEnrichers = Collections.unmodifiableList(buildContextEnrichers(options));
+
if(options.disableTrieLookupPrefilter) {
policyResourceTrie = null;
dataMaskResourceTrie = null;
@@ -162,10 +321,13 @@ class RangerPolicyRepository {
this.serviceName = tagPolicies.getServiceName();
this.componentServiceName = componentServiceName;
+ this.zoneName = null;
+
this.serviceDef = normalizeAccessTypeDefs(ServiceDefUtil.normalize(tagPolicies.getServiceDef()), componentServiceDef.getName());
this.componentServiceDef = componentServiceDef;
this.appId = appId;
+ this.options = options;
this.policies = Collections.unmodifiableList(normalizeAndPrunePolicies(tagPolicies.getPolicies(), componentServiceDef.getName()));
this.policyVersion = tagPolicies.getPolicyVersion() != null ? tagPolicies.getPolicyVersion() : -1;
@@ -188,6 +350,8 @@ class RangerPolicyRepository {
init(options);
+ this.contextEnrichers = Collections.unmodifiableList(buildContextEnrichers(options));
+
if(options.disableTrieLookupPrefilter) {
policyResourceTrie = null;
dataMaskResourceTrie = null;
@@ -208,6 +372,36 @@ class RangerPolicyRepository {
return sb.toString();
}
+ boolean preCleanup() {
+ if (CollectionUtils.isNotEmpty(this.contextEnrichers) && !isContextEnrichersShared) {
+ for (RangerContextEnricher enricher : this.contextEnrichers) {
+ enricher.preCleanup();
+ }
+ return true;
+ }
+ return false;
+ }
+
+ void cleanup() {
+ preCleanup();
+
+ if (CollectionUtils.isNotEmpty(this.contextEnrichers) && !isContextEnrichersShared) {
+ for (RangerContextEnricher enricher : this.contextEnrichers) {
+ enricher.cleanup();
+ }
+ }
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ cleanup();
+ }
+ finally {
+ super.finalize();
+ }
+ }
+
void reorderPolicyEvaluators() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> reorderEvaluators()");
@@ -232,6 +426,8 @@ class RangerPolicyRepository {
String getServiceName() { return serviceName; }
+ String getZoneName() { return zoneName; }
+
RangerServiceDef getServiceDef() {
return serviceDef;
}
@@ -293,6 +489,10 @@ class RangerPolicyRepository {
return rowFilterPolicyEvaluators;
}
+ String getAppId() { return appId; }
+
+ RangerPolicyEngineOptions getOptions() { return options; }
+
List<PolicyEvaluatorForTag> getLikelyMatchPolicyEvaluators(Set<RangerTagForEval> tags, int policyType, Date accessTime) {
List<PolicyEvaluatorForTag> ret = Collections.EMPTY_LIST;
@@ -364,6 +564,11 @@ class RangerPolicyRepository {
}
}
+
+ Map<Long, RangerPolicyEvaluator> getPolicyEvaluatorsMap() { return policyEvaluatorsMap; }
+
+ RangerPolicyEvaluator getPolicyEvaluator(Long id) { return policyEvaluatorsMap.get(id); }
+
private List<RangerPolicyEvaluator> getLikelyMatchAccessPolicyEvaluators(RangerAccessResource resource) {
String resourceStr = resource == null ? null : resource.getAsString();
@@ -524,20 +729,7 @@ class RangerPolicyRepository {
List<RangerPolicy> policiesToPrune = null;
for (RangerPolicy policy : rangerPolicies) {
- normalizeAndPrunePolicyItems(policy.getPolicyItems(), componentType);
- normalizeAndPrunePolicyItems(policy.getDenyPolicyItems(), componentType);
- normalizeAndPrunePolicyItems(policy.getAllowExceptions(), componentType);
- normalizeAndPrunePolicyItems(policy.getDenyExceptions(), componentType);
- normalizeAndPrunePolicyItems(policy.getDataMaskPolicyItems(), componentType);
- normalizeAndPrunePolicyItems(policy.getRowFilterPolicyItems(), componentType);
-
- if (!policy.getIsAuditEnabled() &&
- CollectionUtils.isEmpty(policy.getPolicyItems()) &&
- CollectionUtils.isEmpty(policy.getDenyPolicyItems()) &&
- CollectionUtils.isEmpty(policy.getAllowExceptions()) &&
- CollectionUtils.isEmpty(policy.getDenyExceptions()) &&
- CollectionUtils.isEmpty(policy.getDataMaskPolicyItems()) &&
- CollectionUtils.isEmpty(policy.getRowFilterPolicyItems())) {
+ if (isPolicyNeedsPruning(policy, componentType)) {
if(policiesToPrune == null) {
policiesToPrune = new ArrayList<>();
@@ -555,6 +747,28 @@ class RangerPolicyRepository {
return rangerPolicies;
}
+ private boolean isPolicyNeedsPruning(RangerPolicy policy, final String componentType) {
+
+ normalizeAndPrunePolicyItems(policy.getPolicyItems(), componentType);
+ normalizeAndPrunePolicyItems(policy.getDenyPolicyItems(), componentType);
+ normalizeAndPrunePolicyItems(policy.getAllowExceptions(), componentType);
+ normalizeAndPrunePolicyItems(policy.getDenyExceptions(), componentType);
+ normalizeAndPrunePolicyItems(policy.getDataMaskPolicyItems(), componentType);
+ normalizeAndPrunePolicyItems(policy.getRowFilterPolicyItems(), componentType);
+
+ if (!policy.getIsAuditEnabled() &&
+ CollectionUtils.isEmpty(policy.getPolicyItems()) &&
+ CollectionUtils.isEmpty(policy.getDenyPolicyItems()) &&
+ CollectionUtils.isEmpty(policy.getAllowExceptions()) &&
+ CollectionUtils.isEmpty(policy.getDenyExceptions()) &&
+ CollectionUtils.isEmpty(policy.getDataMaskPolicyItems()) &&
+ CollectionUtils.isEmpty(policy.getRowFilterPolicyItems())) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
private List<? extends RangerPolicy.RangerPolicyItem> normalizeAndPrunePolicyItems(List<? extends RangerPolicy.RangerPolicyItem> policyItems, final String componentType) {
if(CollectionUtils.isNotEmpty(policyItems)) {
final String prefix = componentType + AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR;
@@ -695,32 +909,7 @@ class RangerPolicyRepository {
Collections.sort(rowFilterPolicyEvaluators, comparator);
this.rowFilterPolicyEvaluators = Collections.unmodifiableList(rowFilterPolicyEvaluators);
- List<RangerContextEnricher> contextEnrichers = new ArrayList<RangerContextEnricher>();
- if (CollectionUtils.isNotEmpty(this.policyEvaluators) || CollectionUtils.isNotEmpty(this.dataMaskPolicyEvaluators)
- || CollectionUtils.isNotEmpty(this.rowFilterPolicyEvaluators)) {
- if (CollectionUtils.isNotEmpty(serviceDef.getContextEnrichers())) {
- for (RangerServiceDef.RangerContextEnricherDef enricherDef : serviceDef.getContextEnrichers()) {
- if (enricherDef == null) {
- continue;
- }
- if (!options.disableContextEnrichers || options.enableTagEnricherWithLocalRefresher && StringUtils.equals(enricherDef.getEnricher(), RangerTagEnricher.class.getName())) {
- // This will be true only if the engine is initialized within ranger-admin
- RangerServiceDef.RangerContextEnricherDef contextEnricherDef = enricherDef;
-
- if (options.enableTagEnricherWithLocalRefresher && StringUtils.equals(enricherDef.getEnricher(), RangerTagEnricher.class.getName())) {
- contextEnricherDef = new RangerServiceDef.RangerContextEnricherDef(enricherDef.getItemId(), enricherDef.getName(), "org.apache.ranger.common.RangerAdminTagEnricher", null);
- }
-
- RangerContextEnricher contextEnricher = buildContextEnricher(contextEnricherDef);
-
- if (contextEnricher != null) {
- contextEnrichers.add(contextEnricher);
- }
- }
- }
- }
- }
- this.contextEnrichers = Collections.unmodifiableList(contextEnrichers);
+ this.policyEvaluatorsMap = createPolicyEvaluatorsMap();
if(LOG.isDebugEnabled()) {
LOG.debug("policy evaluation order: " + this.policyEvaluators.size() + " policies");
@@ -750,6 +939,33 @@ class RangerPolicyRepository {
}
}
+ private List<RangerContextEnricher> buildContextEnrichers(RangerPolicyEngineOptions options) {
+ List<RangerContextEnricher> contextEnrichers = new ArrayList<RangerContextEnricher>();
+
+ if (StringUtils.isEmpty(zoneName) && CollectionUtils.isNotEmpty(serviceDef.getContextEnrichers())) {
+ for (RangerServiceDef.RangerContextEnricherDef enricherDef : serviceDef.getContextEnrichers()) {
+ if (enricherDef == null) {
+ continue;
+ }
+ if (!options.disableContextEnrichers || options.enableTagEnricherWithLocalRefresher && StringUtils.equals(enricherDef.getEnricher(), RangerTagEnricher.class.getName())) {
+ // This will be true only if the engine is initialized within ranger-admin
+ RangerServiceDef.RangerContextEnricherDef contextEnricherDef = enricherDef;
+
+ if (options.enableTagEnricherWithLocalRefresher && StringUtils.equals(enricherDef.getEnricher(), RangerTagEnricher.class.getName())) {
+ contextEnricherDef = new RangerServiceDef.RangerContextEnricherDef(enricherDef.getItemId(), enricherDef.getName(), "org.apache.ranger.common.RangerAdminTagEnricher", null);
+ }
+
+ RangerContextEnricher contextEnricher = buildContextEnricher(contextEnricherDef);
+
+ if (contextEnricher != null) {
+ contextEnrichers.add(contextEnricher);
+ }
+ }
+ }
+ }
+ return contextEnrichers;
+ }
+
private RangerContextEnricher buildContextEnricher(RangerServiceDef.RangerContextEnricherDef enricherDef) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyRepository.buildContextEnricher(" + enricherDef + ")");
@@ -893,7 +1109,7 @@ class RangerPolicyRepository {
private Map<String, RangerResourceTrie> createResourceTrieMap(List<RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval) {
final Map<String, RangerResourceTrie> ret;
- if (CollectionUtils.isNotEmpty(evaluators) && serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) {
+ if (serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) {
ret = new HashMap<>();
for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
@@ -906,11 +1122,223 @@ class RangerPolicyRepository {
return ret;
}
+ private void updateTrie(Map<String, RangerResourceTrie> currentMap, Integer policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator newEvaluator) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerPolicyRepository.updateTrie(policyDeltaType=" + policyDeltaType + "): ");
+ }
+ for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
+
+ String resourceDefName = resourceDef.getName();
+
+ RangerResourceTrie trie = currentMap.get(resourceDefName);
+
+ if (policyDeltaType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE) {
+ if (newEvaluator != null) {
+ RangerPolicy.RangerPolicyResource resource = newEvaluator.getPolicyResource().get(resourceDefName);
+ if (resource != null) {
+ trie.add(resource, newEvaluator);
+ }
+ }
+ } else if (policyDeltaType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
+ if (oldEvaluator != null) {
+ RangerPolicy.RangerPolicyResource resource = oldEvaluator.getPolicyResource().get(resourceDefName);
+ if (resource != null) {
+ trie.delete(resource, oldEvaluator);
+ }
+ }
+ } else if (policyDeltaType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE) {
+ if (oldEvaluator != null) {
+ RangerPolicy.RangerPolicyResource oldResource = oldEvaluator.getPolicyResource().get(resourceDefName);
+ if (oldResource != null) {
+ trie.delete(oldResource, oldEvaluator);
+ }
+ }
+ if (newEvaluator != null) {
+ RangerPolicy.RangerPolicyResource newResource = newEvaluator.getPolicyResource().get(resourceDefName);
+
+ if (newResource != null) {
+ trie.add(newResource, newEvaluator);
+ }
+ }
+ } else {
+ LOG.error("policyDeltaType:" + policyDeltaType + " is currently not handled, policy-id:[" + oldEvaluator.getPolicy().getId() +"]");
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerPolicyRepository.updateTrie(policyDeltaType=" + policyDeltaType + "): ");
+ }
+ }
+
+ private Map<Long, RangerPolicyEvaluator> createPolicyEvaluatorsMap() {
+ Map<Long, RangerPolicyEvaluator> tmpPolicyEvaluatorMap = new HashMap<>();
+
+ for (RangerPolicyEvaluator evaluator : getPolicyEvaluators()) {
+ tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+ }
+ for (RangerPolicyEvaluator evaluator : getDataMaskPolicyEvaluators()) {
+ tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+ }
+ for (RangerPolicyEvaluator evaluator : getRowFilterPolicyEvaluators()) {
+ tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+ }
+
+ return Collections.unmodifiableMap(tmpPolicyEvaluatorMap);
+ }
+
+
+ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerPolicyRepository.addPolicy(" + policy +")");
+ }
+ RangerPolicyEvaluator ret = null;
+
+ if (StringUtils.equals(this.serviceDef.getName(), this.componentServiceDef.getName()) || !isPolicyNeedsPruning(policy, this.componentServiceDef.getName())) {
+ policies.add(policy);
+
+ if (!skipBuildingPolicyEvaluator(policy, options)) {
+
+ ret = buildPolicyEvaluator(policy, serviceDef, options);
+
+ if (ret != null) {
+ if (policy.getPolicyType() == null || policy.getPolicyType() == RangerPolicy.POLICY_TYPE_ACCESS) {
+ policyEvaluators.add(ret);
+ } else if (policy.getPolicyType() == RangerPolicy.POLICY_TYPE_DATAMASK) {
+ dataMaskPolicyEvaluators.add(ret);
+ } else if (policy.getPolicyType() == RangerPolicy.POLICY_TYPE_ROWFILTER) {
+ rowFilterPolicyEvaluators.add(ret);
+ } else {
+ LOG.warn("RangerPolicyEngine: ignoring policy id=" + policy.getId() + " - invalid policyType '" + policy.getPolicyType() + "'");
+ }
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerPolicyRepository.addPolicy(" + policy +"): " + ret);
+ }
+ return ret;
+ }
+
+ private void removePolicy(Long id) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerPolicyRepository.removePolicy(" + id +")");
+ }
+ Iterator<RangerPolicy> iterator = policies.iterator();
+ while (iterator.hasNext()) {
+ if (id.equals(iterator.next().getId())) {
+ iterator.remove();
+ break;
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerPolicyRepository.removePolicy(" + id +")");
+ }
+ }
+
+ private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerPolicyRepository.deletePolicyEvaluator(" + evaluator.getPolicy() + ")");
+ }
+ int policyType = evaluator.getPolicy().getPolicyType();
+
+ List<RangerPolicyEvaluator> evaluators = null;
+
+ if (policyType == RangerPolicy.POLICY_TYPE_ACCESS) {
+ evaluators = this.policyEvaluators;
+ } else if (policyType == RangerPolicy.POLICY_TYPE_DATAMASK) {
+ evaluators = this.dataMaskPolicyEvaluators;
+ } else if (policyType == RangerPolicy.POLICY_TYPE_ROWFILTER) {
+ evaluators = this.rowFilterPolicyEvaluators;
+ } else {
+ LOG.error("Unknown policyType:[" + policyType +"]");
+ }
+ if (evaluators != null) {
+ evaluators.remove(evaluator);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerPolicyRepository.deletePolicyEvaluator(" + evaluator.getPolicy() + ")");
+ }
+ }
+
+ private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")");
+ }
+ Integer changeType = delta.getChangeType();
+ Integer policyType = delta.getPolicyType();
+ Long policyId = delta.getPolicyId();
+
+ RangerPolicy policy = delta.getPolicy();
+
+ RangerPolicyEvaluator newEvaluator = null;
+
+ switch (changeType) {
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE:
+ if (policy != null) {
+ newEvaluator = addPolicy(policy);
+ }
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: {
+ removePolicy(policyId);
+ if (policy != null) {
+ newEvaluator = addPolicy(policy);
+ }
+ }
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: {
+ if (currentEvaluator != null) {
+ removePolicy(policyId);
+ }
+ }
+ break;
+ }
+
+ Map<String, RangerResourceTrie> trie = getTrie(policyType);
+
+ if (trie != null) {
+ updateTrie(trie, changeType, currentEvaluator, newEvaluator);
+ }
+
+ if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
+ if (currentEvaluator != null) {
+ deletePolicyEvaluator(currentEvaluator);
+ }
+ }
+
+ RangerPolicyEvaluator ret = changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE ? currentEvaluator : newEvaluator;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")");
+ }
+
+ return ret;
+ }
+
+ private Map<String, RangerResourceTrie> getTrie(final int policyType) {
+ final Map<String, RangerResourceTrie> ret;
+ switch (policyType) {
+ case RangerPolicy.POLICY_TYPE_ACCESS:
+ ret = policyResourceTrie;
+ break;
+ case RangerPolicy.POLICY_TYPE_DATAMASK:
+ ret = dataMaskResourceTrie;
+ break;
+ case RangerPolicy.POLICY_TYPE_ROWFILTER:
+ ret = rowFilterResourceTrie;
+ break;
+ default:
+ ret = null;
+ }
+ return ret;
+ }
+
private StringBuilder toString(StringBuilder sb) {
sb.append("RangerPolicyRepository={");
sb.append("serviceName={").append(serviceName).append("} ");
+ sb.append("zoneName={").append(zoneName).append("} ");
sb.append("serviceDef={").append(serviceDef).append("} ");
sb.append("appId={").append(appId).append("} ");
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
index 289ec9b..3e7c34c 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
@@ -173,6 +173,10 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
RangerPerfTracer.log(perf);
+ if (useAclSummaryForEvaluation && (policy.getPolicyType() == null || policy.getPolicyType() == RangerPolicy.POLICY_TYPE_ACCESS)) {
+ LOG.info("PolicyEvaluator for policy:[" + policy.getId() + "] is set up to use ACL Summary to evaluate access");
+ }
+
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerDefaultPolicyEvaluator.init()");
}
@@ -777,14 +781,18 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
}
if (useAclSummaryForEvaluation && (getPolicy().getPolicyType() == null || getPolicy().getPolicyType() == RangerPolicy.POLICY_TYPE_ACCESS)) {
- LOG.info("Using ACL Summary for checking if access is allowed. PolicyId=[" + getId() +"]");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using ACL Summary for checking if access is allowed. PolicyId=[" + getId() +"]");
+ }
Integer accessResult = lookupPolicyACLSummary(user, userGroups, accessType);
if (accessResult != null && accessResult.equals(RangerPolicyEvaluator.ACCESS_ALLOWED)) {
ret = true;
}
} else {
- LOG.info("Using policyItemEvaluators for checking if access is allowed. PolicyId=[" + getId() +"]");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using policyItemEvaluators for checking if access is allowed. PolicyId=[" + getId() +"]");
+ }
RangerPolicyItemEvaluator item = this.getDeterminingPolicyItem(user, userGroups, accessType);
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
index 8d35319..12a1c1c 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
@@ -632,6 +632,9 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM
}
public static boolean isHierarchyValidForResources(List<RangerResourceDef> hierarchy, Map<String, ?> resources) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> isHierarchyValidForResources(" + StringUtils.join(hierarchy, ",") + ")");
+ }
boolean ret = true;
if (hierarchy != null) {
@@ -655,7 +658,9 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM
} else {
ret = false;
}
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== isHierarchyValidForResources(" + StringUtils.join(hierarchy, ",") + ") : " + ret);
+ }
return ret;
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerAuthContext.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerAuthContext.java
index 9ae3348..8b00144 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerAuthContext.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerAuthContext.java
@@ -35,6 +35,7 @@ import org.apache.ranger.plugin.policyengine.RangerResourceACLs;
import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo;
import org.apache.ranger.plugin.util.GrantRevokeRequest;
import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
+import org.apache.ranger.plugin.util.ServicePolicies;
import java.util.Collection;
import java.util.HashMap;
@@ -254,4 +255,9 @@ public class RangerAuthContext implements RangerPolicyEngine {
return null;
}
+ @Override
+ public RangerPolicyEngine cloneWithDelta(ServicePolicies servicePolicies) {
+ return policyEngine.cloneWithDelta(servicePolicies);
+ }
+
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index 1c870f7..96ca317 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -20,13 +20,16 @@
package org.apache.ranger.plugin.service;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Hashtable;
+import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -252,27 +255,95 @@ public class RangerBasePlugin {
// guard against catastrophic failure during policy engine Initialization or
try {
RangerPolicyEngine oldPolicyEngine = this.policyEngine;
+ ServicePolicies servicePolicies = null;
+ boolean isValid = true;
+ boolean usePolicyDeltas = false;
if (policies == null) {
policies = getDefaultSvcPolicies();
- }
- if (policies == null) {
- this.policyEngine = null;
- readOnlyAuthContext = null;
+ if (policies == null) {
+ LOG.error("Could not get default Service Policies");
+ isValid = false;
+ }
} else {
- currentAuthContext = new RangerAuthContext();
- RangerPolicyEngine policyEngine = new RangerPolicyEngineImpl(appId, policies, policyEngineOptions);
- policyEngine.setUseForwardedIPAddress(useForwardedIPAddress);
- policyEngine.setTrustedProxyAddresses(trustedProxyAddresses);
- this.policyEngine = policyEngine;
- currentAuthContext.setPolicyEngine(this.policyEngine);
- readOnlyAuthContext = new RangerAuthContext(currentAuthContext);
+ if ((policies.getPolicies() == null && policies.getPolicyDeltas() == null) || (policies.getPolicies() != null && policies.getPolicyDeltas() != null)) {
+ LOG.error("Invalid servicePolicies: Both policies and policy-deltas cannot be null OR both of them cannot be non-null");
+ isValid = false;
+ } else if (policies.getPolicies() != null) {
+ usePolicyDeltas = false;
+ } else if (policies.getPolicyDeltas() != null) {
+ // Rebuild policies from deltas
+ RangerPolicyEngineImpl policyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine;
+ List<RangerPolicy> oldResourcePolicies = policyEngineImpl.getResourcePolicies();
+ List<RangerPolicy> oldTagPolicies = policyEngineImpl.getTagPolicies();
+ servicePolicies = ServicePolicies.applyDelta(policies, oldResourcePolicies, oldTagPolicies);
+ if (servicePolicies != null) {
+ usePolicyDeltas = true;
+ } else {
+ isValid = false;
+ LOG.error("Could not apply deltas=" + Arrays.toString(policies.getPolicyDeltas().toArray()));
+ }
+ } else {
+ LOG.error("Should not get here!!");
+ isValid = false;
+ }
}
- contextChanged();
- if (oldPolicyEngine != null && !oldPolicyEngine.preCleanup()) {
- LOG.error("preCleanup() failed on the previous policy engine instance !!");
+ if (isValid) {
+ RangerPolicyEngine newPolicyEngine = null;
+
+ if (!usePolicyDeltas) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("policies are not null. Creating engine from policies");
+ }
+ newPolicyEngine = new RangerPolicyEngineImpl(appId, policies, policyEngineOptions);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("policy-deltas are not null");
+ }
+ if (CollectionUtils.isNotEmpty(policies.getPolicyDeltas())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Non empty policy-deltas found. Cloning engine using policy-deltas");
+ }
+ newPolicyEngine = oldPolicyEngine.cloneWithDelta(policies);
+ if (newPolicyEngine != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Applied policyDeltas=" + Arrays.toString(policies.getPolicyDeltas().toArray()) + ")");
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to apply policyDeltas=" + Arrays.toString(policies.getPolicyDeltas().toArray()) + "), Creating engine from policies");
+ LOG.debug("Creating new engine from servicePolicies:[" + servicePolicies + "]");
+ }
+ newPolicyEngine = new RangerPolicyEngineImpl(appId, servicePolicies, policyEngineOptions);
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Empty policy-deltas. No need to change policy engine");
+ }
+ }
+ }
+
+ if (newPolicyEngine != null) {
+ currentAuthContext = new RangerAuthContext();
+
+ newPolicyEngine.setUseForwardedIPAddress(useForwardedIPAddress);
+ newPolicyEngine.setTrustedProxyAddresses(trustedProxyAddresses);
+ this.policyEngine = newPolicyEngine;
+ currentAuthContext.setPolicyEngine(this.policyEngine);
+ readOnlyAuthContext = new RangerAuthContext(currentAuthContext);
+
+ contextChanged();
+
+ if (oldPolicyEngine != null && !oldPolicyEngine.preCleanup()) {
+ LOG.error("preCleanup() failed on the previous policy engine instance !!");
+ }
+ this.refresher.saveToCache(usePolicyDeltas ? servicePolicies : policies);
+ }
+ } else {
+ LOG.error("Returning without saving policies to cache. Leaving current policy engine as-is");
}
+
} catch (Exception e) {
LOG.error("setPolicies: policy engine initialization failed! Leaving current policy engine as-is. Exception : ", e);
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
index 01ce9b2..9e37cd5 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
@@ -89,11 +89,13 @@ public interface ServiceStore {
PList<RangerPolicy> getPaginatedServicePolicies(String serviceName, SearchFilter filter) throws Exception;
- ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception;
+ ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion, boolean needsBackwardCompatibility) throws Exception;
Long getServicePolicyVersion(String serviceName);
- ServicePolicies getServicePolicies(String serviceName) throws Exception;
+ ServicePolicies getServicePolicyDeltasOrPolicies(String serviceName, Long lastKnownVersion) throws Exception;
+
+ ServicePolicies getOnlyServicePolicyDeltas(String serviceName, Long lastKnownVersion) throws Exception;
RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId);
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
index b5b4f16..e85612a 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
@@ -80,7 +80,7 @@ public class PolicyRefresher extends Thread {
Gson gson = null;
try {
- gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create();
+ gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create();
} catch(Throwable excp) {
LOG.fatal("PolicyRefresher(): failed to create GsonBuilder object", excp);
}
@@ -206,12 +206,8 @@ public class PolicyRefresher extends Thread {
if (!policiesSetInPlugin) {
svcPolicies = loadFromCache();
}
- } else {
- saveToCache(svcPolicies);
}
- RangerPerfTracer.log(perf);
-
if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) {
long freeMemory = Runtime.getRuntime().freeMemory();
long totalMemory = Runtime.getRuntime().totalMemory();
@@ -241,6 +237,8 @@ public class PolicyRefresher extends Thread {
LOG.error("Encountered unexpected exception, ignoring..", excp);
}
+ RangerPerfTracer.log(perf);
+
if(LOG.isDebugEnabled()) {
LOG.debug("<== PolicyRefresher(serviceName=" + serviceName + ").loadPolicy()");
}
@@ -356,7 +354,7 @@ public class PolicyRefresher extends Thread {
return policies;
}
- private void saveToCache(ServicePolicies policies) {
+ public void saveToCache(ServicePolicies policies) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").saveToCache()");
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
new file mode 100644
index 0000000..9c50f8a
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.util;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
+import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+public class RangerPolicyDeltaUtil {
+
+ private static final Log LOG = LogFactory.getLog(RangerPolicyDeltaUtil.class);
+
+ private static final Log PERF_POLICY_DELTA_LOG = RangerPerfTracer.getPerfLogger("policy.delta");
+
+ public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<RangerPolicyDelta> deltas, String serviceType) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")");
+ }
+
+ List<RangerPolicy> ret;
+
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_DELTA_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_DELTA_LOG, "RangerPolicyDelta.applyDeltas()");
+ }
+
+ if (CollectionUtils.isNotEmpty(deltas)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("applyDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", serviceType=" + serviceType +")");
+ }
+ ret = new ArrayList<>(policies);
+
+ for (RangerPolicyDelta delta : deltas) {
+ int changeType = delta.getChangeType();
+ if (!serviceType.equals(delta.getServiceType())) {
+ if (!delta.getServiceType().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) {
+ LOG.error("Found unexpected serviceType in policyDelta:[" + delta + "]. Was expecting serviceType:[" + serviceType + "]. Should NOT have come here!! Ignoring delta and continuing");
+ }
+ continue;
+ }
+ if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
+ if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE) {
+ if (delta.getPolicy() != null) {
+ ret.add(delta.getPolicy());
+ }
+ } else {
+ // Either UPDATE or DELETE
+ Long policyId = delta.getPolicyId();
+
+ Iterator<RangerPolicy> iter = ret.iterator();
+ while (iter.hasNext()) {
+ if (policyId.equals(iter.next().getId())) {
+ iter.remove();
+ break;
+ }
+ }
+ if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE) {
+ if (delta.getPolicy() != null) {
+ ret.add(delta.getPolicy());
+ }
+ }
+ }
+ } else {
+ LOG.warn("Found unexpected changeType in policyDelta:[" + delta +"]. Ignoring delta");
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("applyDeltas(deltas=null, serviceType=" + serviceType +")");
+ }
+ ret = policies;
+ }
+
+ if (CollectionUtils.isNotEmpty(deltas) && CollectionUtils.isNotEmpty(ret)) {
+ ret.sort(RangerPolicy.POLICY_ID_COMPARATOR);
+ }
+
+ RangerPerfTracer.log(perf);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== applyDeltas(serviceType=" + serviceType + "): " + ret);
+ }
+ return ret;
+ }
+
+ public static boolean isValidDeltas(List<RangerPolicyDelta> deltas, String componentServiceType) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> isValidDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", componentServiceType=" + componentServiceType +")");
+ }
+ boolean isValid = true;
+
+ for (RangerPolicyDelta delta : deltas) {
+ final Integer changeType = delta.getChangeType();
+ final Long policyId = delta.getPolicyId();
+
+ if (changeType == null) {
+ isValid = false;
+ break;
+ }
+
+ if (changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE
+ && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE
+ && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
+ isValid = false;
+ } else if (policyId == null) {
+ isValid = false;
+ } else {
+ final String serviceType = delta.getServiceType();
+ final Integer policyType = delta.getPolicyType();
+
+ if (serviceType == null || (!serviceType.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME) &&
+ !serviceType.equals(componentServiceType))) {
+ isValid = false;
+ } else if (policyType == null || (policyType != RangerPolicy.POLICY_TYPE_ACCESS
+ && policyType != RangerPolicy.POLICY_TYPE_DATAMASK
+ && policyType != RangerPolicy.POLICY_TYPE_ROWFILTER)) {
+ isValid = false;
+ }
+ }
+
+ if (!isValid) {
+ break;
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== isValidDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", componentServiceType=" + componentServiceType +"): " + isValid);
+ }
+ return isValid;
+ }
+}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java
index f592ed4..2a4b9c9 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTClient.java
@@ -228,7 +228,7 @@ public class RangerRESTClient {
private void init() {
try {
- gsonBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").setPrettyPrinting().create();
+ gsonBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").create();
} catch(Throwable excp) {
LOG.fatal("RangerRESTClient.init(): failed to create GsonBuilder object", excp);
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java
index f9ef1d3..65f5c01 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java
@@ -62,6 +62,7 @@ public class RangerRESTUtils {
private static final int MAX_PLUGIN_ID_LEN = 255;
public static final String REST_PARAM_CLUSTER_NAME = "clusterName";
+ public static final String REST_PARAM_SUPPORTS_POLICY_DELTAS = "supportsPolicyDeltas";
public static final String REST_PARAM_ZONE_NAME = "zoneName";
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index a2d52a0..605a74a 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -32,6 +32,7 @@ import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher;
import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
@@ -42,19 +43,20 @@ import java.util.concurrent.LinkedBlockingQueue;
public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class);
+ private static final Log TRACE_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.trace");
private static final Log PERF_TRIE_INIT_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.init");
private static final Log PERF_TRIE_OP_LOG = RangerPerfTracer.getPerfLogger("resourcetrie.op");
private static final String DEFAULT_WILDCARD_CHARS = "*?";
private static final String TRIE_BUILDER_THREAD_COUNT = "ranger.policyengine.trie.builder.thread.count";
- private final String resourceName;
- private final boolean optIgnoreCase;
- private final boolean optWildcard;
- private final String wildcardChars;
- private final TrieNode<T> root;
+ private final String resourceName;
+ private final boolean optIgnoreCase;
+ private final boolean optWildcard;
+ private final String wildcardChars;
+ private final TrieNode<T> root;
private final Comparator<T> comparator;
- private final boolean isOptimizedForRetrieval;
+ private final boolean isOptimizedForRetrieval;
public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<T> evaluators) {
this(resourceDef, evaluators, null, true);
@@ -77,8 +79,9 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
builderThreadCount = 1;
}
- LOG.info("builderThreadCount is set to ["+ builderThreadCount +"]");
- PERF_TRIE_INIT_LOG.info("builderThreadCount is set to ["+ builderThreadCount +"]");
+ if (TRACE_LOG.isTraceEnabled()) {
+ TRACE_LOG.trace("builderThreadCount is set to [" + builderThreadCount + "]");
+ }
Map<String, String> matcherOptions = resourceDef.getMatcherOptions();
@@ -117,10 +120,10 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
PERF_TRIE_INIT_LOG.debug(toString());
}
- if (PERF_TRIE_INIT_LOG.isTraceEnabled()) {
+ if (TRACE_LOG.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
root.toString("", sb);
- PERF_TRIE_INIT_LOG.trace("Trie Dump:\n{" + sb.toString() + "}");
+ TRACE_LOG.trace("Trie Dump from RangerResourceTrie.init(name=" + resourceName + "):\n{" + sb.toString() + "}");
}
if(LOG.isDebugEnabled()) {
@@ -149,6 +152,173 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
return null;
}
+ public void add(RangerPolicyResource resource, T evaluator) {
+
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.add(name=" + resource + ")");
+ }
+
+ if (resource.getIsExcludes()) {
+ root.addWildcardEvaluator(evaluator);
+ } else {
+ if (CollectionUtils.isNotEmpty(resource.getValues())) {
+ for (String value : resource.getValues()) {
+ insert(root, value, resource.getIsRecursive(), evaluator);
+ }
+ }
+ }
+
+ RangerPerfTracer.logAlways(perf);
+ if (TRACE_LOG.isTraceEnabled()) {
+ StringBuilder sb = new StringBuilder();
+ root.toString("", sb);
+ TRACE_LOG.trace("Trie Dump from RangerResourceTrie.add(name=" + resource + "):\n{" + sb.toString() + "}");
+ }
+ }
+
+ public void delete(RangerPolicyResource resource, T evaluator) {
+
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.delete(name=" + resource + ")");
+ }
+
+ boolean isRemoved = false;
+ if (resource.getIsExcludes()) {
+ if (CollectionUtils.isNotEmpty(root.wildcardEvaluators)) {
+ isRemoved = root.wildcardEvaluators.remove(evaluator);
+ if (isRemoved && CollectionUtils.isEmpty(root.wildcardEvaluators)) {
+ root.wildcardEvaluators = null;
+ }
+ }
+ }
+ if (!isRemoved) {
+ for (String value : resource.getValues()) {
+ TrieNode<T> node = getNodeForResource(value);
+ if (node != null) {
+ node.removeEvaluatorFromSubtree(evaluator);
+ }
+ }
+ }
+
+ RangerPerfTracer.logAlways(perf);
+ if (TRACE_LOG.isTraceEnabled()) {
+ StringBuilder sb = new StringBuilder();
+ root.toString("", sb);
+ TRACE_LOG.trace("Trie Dump from RangerResourceTrie.delete(name=" + resource + "):\n{" + sb.toString() + "}");
+ }
+ }
+
+ public void wrapUpUpdate() {
+ if (this.isOptimizedForRetrieval) {
+ root.postSetup(null, comparator);
+ } else {
+ root.setup(null, comparator);
+ }
+ }
+
+ private TrieNode<T> copyTrieSubtree(TrieNode<T> source, List<T> parentWildcardEvaluators) {
+ if (TRACE_LOG.isTraceEnabled()) {
+ StringBuilder sb = new StringBuilder();
+ source.toString(sb);
+ TRACE_LOG.trace("==> copyTrieSubtree(" + sb + ", parentWildCardEvaluators=" + (parentWildcardEvaluators != null ? Arrays.toString(parentWildcardEvaluators.toArray()) : "[]") + ")");
+ }
+ TrieNode<T> dest = new TrieNode<>(source.str);
+ boolean setUpCompleted = source.isSetup;
+ if (!setUpCompleted) {
+ synchronized (source) {
+ setUpCompleted = source.isSetup;
+ if (!setUpCompleted) {
+ if (source.wildcardEvaluators != null) {
+ dest.wildcardEvaluators = new ArrayList<>(source.wildcardEvaluators);
+ } else {
+ dest.wildcardEvaluators = null;
+ }
+ if (source.evaluators != null) {
+ if (source.evaluators == source.wildcardEvaluators) {
+ dest.evaluators = null;
+ } else {
+ dest.evaluators = new ArrayList<>(source.evaluators);
+ }
+ } else {
+ dest.evaluators = null;
+ }
+ }
+ }
+ }
+ if (setUpCompleted) {
+ if (source.isSharingParentWildcardEvaluators) {
+ dest.wildcardEvaluators = null;
+ } else {
+ if (source.wildcardEvaluators != null) {
+ dest.wildcardEvaluators = new ArrayList<>(source.wildcardEvaluators);
+ if (parentWildcardEvaluators != null) {
+ dest.wildcardEvaluators.removeAll(parentWildcardEvaluators);
+ }
+ } else {
+ dest.wildcardEvaluators = null;
+ }
+ }
+ if (source.evaluators != null) {
+ if (source.evaluators == source.wildcardEvaluators) {
+ dest.evaluators = null;
+ } else {
+ dest.evaluators = new ArrayList<>(source.evaluators);
+ if (source.wildcardEvaluators != null) {
+ dest.evaluators.removeAll(source.wildcardEvaluators);
+ }
+ }
+ } else {
+ dest.evaluators = null;
+ }
+ }
+
+ Map<Character, TrieNode<T>> children = source.getChildren();
+ for (Map.Entry<Character, TrieNode<T>> entry : children.entrySet()) {
+ TrieNode<T> copy = copyTrieSubtree(entry.getValue(), source.wildcardEvaluators);
+ dest.addChild(copy);
+ }
+
+ if (TRACE_LOG.isTraceEnabled()) {
+ StringBuilder sourceAsString = new StringBuilder(), destAsString = new StringBuilder();
+ source.toString(sourceAsString);
+ dest.toString(destAsString);
+
+ TRACE_LOG.trace("<== copyTrieSubtree(" + sourceAsString + ", parentWildCardEvaluators=" + (parentWildcardEvaluators != null ? Arrays.toString(parentWildcardEvaluators.toArray()) : "[]") + ") : " + destAsString);
+ }
+ return dest;
+ }
+
+ public RangerResourceTrie(RangerResourceTrie<T> other) {
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG, "RangerResourceTrie.copyTrie(name=" + other.resourceName + ")");
+ }
+
+ this.resourceName = other.resourceName;
+ this.optIgnoreCase = other.optIgnoreCase;
+ this.optWildcard = other.optWildcard;
+ this.wildcardChars = other.wildcardChars;
+ this.comparator = other.comparator;
+ this.isOptimizedForRetrieval = other.isOptimizedForRetrieval;
+ this.root = copyTrieSubtree(other.root, null);
+
+ RangerPerfTracer.logAlways(perf);
+
+ if (PERF_TRIE_INIT_LOG.isDebugEnabled()) {
+ PERF_TRIE_INIT_LOG.debug(toString());
+ }
+ if (TRACE_LOG.isTraceEnabled()) {
+ StringBuilder sb = new StringBuilder();
+ root.toString("", sb);
+ TRACE_LOG.trace("Trie Dump from RangerResourceTrie.copyTrie(name=" + other.resourceName + "):\n{" + sb.toString() + "}");
+ }
+ }
+
private TrieNode<T> buildTrie(RangerServiceDef.RangerResourceDef resourceDef, List<T> evaluators, Comparator<T> comparator, int builderThreadCount) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> buildTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isMultiThreaded=" + (builderThreadCount > 1) + ")");
@@ -418,6 +588,48 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
return ret;
}
+ private TrieNode<T> getNodeForResource(String resource) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerResourceTrie.getNodeForResource(" + resource + ")");
+ }
+
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerResourceTrie.getNodeForResource(resource=" + resource + ")");
+ }
+
+ TrieNode<T> curr = root;
+ final int len = resource.length();
+ int i = 0;
+
+ while (i < len) {
+
+ final TrieNode<T> child = curr.getChild(getLookupChar(resource, i));
+
+ if (child == null) {
+ break;
+ }
+
+ final String childStr = child.getStr();
+
+ if (!resource.regionMatches(optIgnoreCase, i, childStr, 0, childStr.length())) {
+ break;
+ }
+
+ curr = child;
+ i += childStr.length();
+ }
+
+ RangerPerfTracer.logAlways(perf);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerResourceTrie.getNodeForResource(" + resource + ")");
+ }
+
+ return curr;
+ }
+
private List<T> getEvaluatorsForResources(Collection<String> resources) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerResourceTrie.getEvaluatorsForResources(" + resources + ")");
@@ -776,6 +988,11 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
if (setupNeeded) {
setup(parent.getWildcardEvaluators(), comparator);
isSetup = true;
+ if (TRACE_LOG.isTraceEnabled()) {
+ StringBuilder sb = new StringBuilder();
+ this.toString(sb);
+ TRACE_LOG.trace("Set up is completed for this TriNode as a part of access evaluation : [" + sb + "]");
+ }
}
}
}
@@ -816,14 +1033,32 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
}
}
- public void toString(String prefix, StringBuilder sb) {
- String nodeValue = prefix;
-
- if (str != null) {
- nodeValue += str;
+ private void removeEvaluatorFromSubtree(T evaluator) {
+ if (CollectionUtils.isNotEmpty(wildcardEvaluators)) {
+ if (wildcardEvaluators.remove(evaluator)) {
+ if (CollectionUtils.isEmpty(wildcardEvaluators)) {
+ wildcardEvaluators = null;
+ }
+ for (Map.Entry<Character, TrieNode<U>> entry : children.entrySet()) {
+ entry.getValue().removeEvaluatorFromSubtree(evaluator);
+ }
+ }
}
+ if (CollectionUtils.isNotEmpty(evaluators)) {
+ if (evaluators.remove(evaluator)) {
+ if (CollectionUtils.isEmpty(evaluators)) {
+ evaluators = null;
+ }
+ }
+ }
+ }
+
+ public void toString(StringBuilder sb) {
+ String nodeValue = this.str;
sb.append("nodeValue=").append(nodeValue);
+ sb.append("; isSetup=").append(isSetup);
+ sb.append("; isSharingParentWildcardEvaluators=").append(isSharingParentWildcardEvaluators);
sb.append("; childCount=").append(children == null ? 0 : children.size());
sb.append("; evaluators=[ ");
if (evaluators != null) {
@@ -839,6 +1074,13 @@ public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> {
sb.append(evaluator.getId()).append(" ");
}
}
+ }
+
+ public void toString(String prefix, StringBuilder sb) {
+ String nodeValue = prefix + (str != null ? str : "");
+
+ sb.append(prefix);
+ toString(sb);
sb.append("]\n");
if (children != null) {
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java
index 664523e..7eb2bb3 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServicePolicies.java
@@ -20,6 +20,8 @@
package org.apache.ranger.plugin.util;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -30,6 +32,7 @@ import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
import org.codehaus.jackson.annotate.JsonAutoDetect;
@@ -54,6 +57,7 @@ public class ServicePolicies implements java.io.Serializable {
private String auditMode = RangerPolicyEngine.AUDIT_DEFAULT;
private TagPolicies tagPolicies;
private Map<String, SecurityZoneInfo> securityZones;
+ private List<RangerPolicyDelta> policyDeltas;
/**
* @return the serviceName
@@ -161,12 +165,16 @@ public class ServicePolicies implements java.io.Serializable {
+ "policyVersion=" + policyVersion + ", "
+ "policyUpdateTime=" + policyUpdateTime + ", "
+ "policies=" + policies + ", "
+ + "tagPolicies=" + tagPolicies + ", "
+ + "policyDeltas=" + policyDeltas + ", "
+ "serviceDef=" + serviceDef + ", "
+ "auditMode=" + auditMode + ", "
- + "tagPolicies=" + tagPolicies + ", "
+ "securityZones=" + securityZones
;
}
+ public List<RangerPolicyDelta> getPolicyDeltas() { return this.policyDeltas; }
+
+ public void setPolicyDeltas(List<RangerPolicyDelta> policyDeltas) { this.policyDeltas = policyDeltas; }
@JsonAutoDetect(fieldVisibility=Visibility.ANY)
@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
@@ -287,6 +295,7 @@ public class ServicePolicies implements java.io.Serializable {
private String zoneName;
private List<HashMap<String, List<String>>> resources;
private List<RangerPolicy> policies;
+ private List<RangerPolicyDelta> policyDeltas;
public String getZoneName() {
return zoneName;
@@ -300,6 +309,8 @@ public class ServicePolicies implements java.io.Serializable {
return policies;
}
+ public List<RangerPolicyDelta> getPolicyDeltas() { return policyDeltas; }
+
public void setZoneName(String zoneName) {
this.zoneName = zoneName;
}
@@ -312,12 +323,68 @@ public class ServicePolicies implements java.io.Serializable {
this.policies = policies;
}
+ public void setPolicyDeltas(List<RangerPolicyDelta> policyDeltas) { this.policyDeltas = policyDeltas; }
+
@Override
public String toString() {
return "zoneName=" + zoneName + ", "
+ "resources=" + resources + ", "
- + "policies=" + policies
+ + "policies=" + policies + ", "
+ + "policyDeltas=" + policyDeltas
;
}
}
+ public static ServicePolicies copyHeader(ServicePolicies source) {
+ ServicePolicies ret = new ServicePolicies();
+
+ ret.setServiceName(source.getServiceName());
+ ret.setServiceId(source.getServiceId());
+ ret.setPolicyVersion(source.getPolicyVersion());
+ ret.setAuditMode(source.getAuditMode());
+ ret.setServiceDef(source.getServiceDef());
+ ret.setPolicyUpdateTime(source.getPolicyUpdateTime());
+ ret.setPolicyDeltas(Collections.emptyList());
+ ret.setPolicies(Collections.emptyList());
+ if (source.getTagPolicies() != null) {
+ TagPolicies tagPolicies = copyHeader(source.getTagPolicies());
+ ret.setTagPolicies(tagPolicies);
+ }
+
+ return ret;
+ }
+
+ public static TagPolicies copyHeader(TagPolicies source) {
+ TagPolicies ret = new TagPolicies();
+
+ ret.setServiceName(source.getServiceName());
+ ret.setServiceId(source.getServiceId());
+ ret.setPolicyVersion(source.getPolicyVersion());
+ ret.setAuditMode(source.getAuditMode());
+ ret.setServiceDef(source.getServiceDef());
+ ret.setPolicyUpdateTime(source.getPolicyUpdateTime());
+ ret.setPolicies(Collections.emptyList());
+
+ return ret;
+ }
+
+ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, final List<RangerPolicy> oldResourcePolicies, final List<RangerPolicy> oldTagPolicies) {
+ ServicePolicies ret = copyHeader(servicePolicies);
+
+ List<RangerPolicy> newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName());
+
+ final List<RangerPolicy> newTagPolicies;
+ if (servicePolicies.getTagPolicies() != null) {
+ final List<RangerPolicy> policies = oldTagPolicies == null ? new ArrayList<>() : oldTagPolicies;
+ newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(policies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName());
+ } else {
+ newTagPolicies = null;
+ }
+
+ ret.setPolicies(newResourcePolicies);
+
+ if (ret.getTagPolicies() != null) {
+ ret.getTagPolicies().setPolicies(newTagPolicies);
+ }
+ return ret;
+ }
}
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
index 9d9be6c..9bd5e24 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
@@ -35,6 +35,7 @@ import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler;
import org.apache.ranger.plugin.contextenricher.RangerTagForEval;
import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.RangerValiditySchedule;
import org.apache.ranger.plugin.model.validation.RangerValidityScheduleValidator;
@@ -67,6 +68,7 @@ import static org.junit.Assert.*;
public class TestPolicyEngine {
static Gson gsonBuilder;
+ long requestCount = 0L;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -218,6 +220,27 @@ public class TestPolicyEngine {
}
@Test
+ public void testPolicyEngine_hive_incremental_add() {
+ String[] hiveTestResourceFiles = {"/policyengine/test_policyengine_hive_incremental_add.json"};
+
+ runTestsFromResourceFiles(hiveTestResourceFiles);
+ }
+
+ @Test
+ public void testPolicyEngine_hive_incremental_delete() {
+ String[] hiveTestResourceFiles = {"/policyengine/test_policyengine_hive_incremental_delete.json"};
+
+ runTestsFromResourceFiles(hiveTestResourceFiles);
+ }
+
+ @Test
+ public void testPolicyEngine_hive_incremental_update() {
+ String[] hiveTestResourceFiles = {"/policyengine/test_policyengine_hive_incremental_update.json"};
+
+ runTestsFromResourceFiles(hiveTestResourceFiles);
+ }
+
+ @Test
public void testPolicyEngine_hiveForTag() {
String[] hiveTestResourceFiles = { "/policyengine/test_policyengine_tag_hive.json" };
@@ -395,11 +418,23 @@ public class TestPolicyEngine {
policyEngineForResourceAccessInfo.setUseForwardedIPAddress(useForwardedIPAddress);
policyEngineForResourceAccessInfo.setTrustedProxyAddresses(trustedProxyAddresses);
- long requestCount = 0L;
+ runTestCaseTests(policyEngine, policyEngineForResourceAccessInfo, testCase.serviceDef, testName, testCase.tests);
- RangerAccessRequest request = null;
+ if (CollectionUtils.isNotEmpty(testCase.updatedPolicies)) {
+ servicePolicies.setPolicyDeltas(testCase.updatedPolicies);
+ servicePolicies.setPolicyVersion(-1L);
+ RangerPolicyEngine updatedPolicyEngine = policyEngine.cloneWithDelta(servicePolicies);
+ RangerPolicyEngine updatedPolicyEngineForResourceAccessInfo = policyEngineForResourceAccessInfo.cloneWithDelta(servicePolicies);
+ runTestCaseTests(updatedPolicyEngine, updatedPolicyEngineForResourceAccessInfo, testCase.serviceDef, testName, testCase.updatedTests);
- for(TestData test : testCase.tests) {
+ }
+ }
+
+ private void runTestCaseTests(RangerPolicyEngine policyEngine, RangerPolicyEngine policyEngineForResourceAccessInfo, RangerServiceDef serviceDef, String testName, List<TestData> tests) {
+
+ RangerAccessRequest request = null;
+
+ for(TestData test : tests) {
request = test.request;
if ((requestCount++ % 10) == 1) {
policyEngine.reorderPolicyEvaluators();
@@ -464,7 +499,7 @@ public class TestPolicyEngine {
// Safe cast
RangerAccessResourceImpl accessResource = (RangerAccessResourceImpl) request.getResource();
- accessResource.setServiceDef(testCase.serviceDef);
+ accessResource.setServiceDef(serviceDef);
request = newRequest;
@@ -517,6 +552,7 @@ public class TestPolicyEngine {
assertEquals("deniedGroups mismatched! - " + test.name, expected.getDeniedGroups(), result.getDeniedGroups());
}
}
+
}
static class PolicyEngineTestCase {
@@ -526,6 +562,9 @@ public class TestPolicyEngine {
public TagPolicyInfo tagPolicyInfo;
public String auditMode;
public List<TestData> tests;
+
+ public List<RangerPolicyDelta> updatedPolicies;
+ public List<TestData> updatedTests;
class TestData {
public String name;
diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hive_incremental_add.json b/agents-common/src/test/resources/policyengine/test_policyengine_hive_incremental_add.json
new file mode 100644
index 0000000..fce1572
--- /dev/null
+++ b/agents-common/src/test/resources/policyengine/test_policyengine_hive_incremental_add.json
@@ -0,0 +1,345 @@
+{
+ "serviceName":"hivedev",
+
+ "serviceDef":{
+ "name":"hive",
+ "id":3,
+ "resources":[
+ {"name":"database","level":1,"mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Database","description":"Hive Database"},
+ {"name":"table","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Table","description":"Hive Table"},
+ {"name":"udf","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive UDF","description":"Hive UDF"},
+ {"name":"column","level":3,"parent":"table","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Column","description":"Hive Column"}
+ ],
+ "accessTypes":[
+ {"name":"select","label":"Select"},
+ {"name":"update","label":"Update"},
+ {"name":"create","label":"Create"},
+ {"name":"drop","label":"Drop"},
+ {"name":"alter","label":"Alter"},
+ {"name":"index","label":"Index"},
+ {"name":"lock","label":"Lock"},
+ {"name":"all","label":"All"}
+ ]
+ },
+
+ "policies":[
+ {"id":1,"name":"db=default: audit-all-access","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["default"]},"table":{"values":["*"]},"column":{"values":["*"]}},
+ "policyItems":[
+ {"accesses":[],"users":[],"groups":["public"],"delegateAdmin":false}
+ ]
+ }
+ ,
+ {"id":2,"name":"db=default; table=test*; column=*","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["default"]},"table":{"values":["test*"]},"column":{"values":["*"]}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ,
+ {"accesses":[{"type":"create","isAllowed":true},{"type":"drop","isAllowed":true}],"users":["admin"],"groups":["admin"],"delegateAdmin":true}
+ ]
+ }
+ ,
+ {"id":3,"name":"db=db1; table=tbl*; column=*","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["db1"]},"table":{"values":["tbl*"]},"column":{"values":["*"]}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ]
+ }
+ ],
+
+ "tests":[
+ {"name":"DENY 'select tmp_1 from db1.tmp ;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"tmp_1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select tmp_1 from db1.tmp for user1"
+ ,"remoteIPAddress":"1.1.1.1","forwardedAddresses":["127.0.0.1","10.10.10.10"]
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'select abc_1 from db1.tmp ;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"abc_1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select abc_1 from db1.tmp for user1"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'use default;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'use default;' for user2",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user2","userGroups":["users"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'use default;' to user3",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'use default;' to group1",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users", "group1"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'use default;' to group2",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users", "group2"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'use default;' to user3/group3",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users", "group3"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'use finance;' to user3/group3",
+ "request":{
+ "resource":{"elements":{"database":"finance"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"use finance"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to user2",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user2","userGroups":["users"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.testtable;' to user3",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to group1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users","group1"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to group2",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users","group2"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.testtable;' to user3/group3",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users","group3"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.table1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1","column":"col1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select col1 from default.table1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'create table default.testtable1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"user1","userGroups":["users"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'create table default.testtable1;' to user1/group1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"user1","userGroups":["users","group1"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'create table default.testtable1;' to admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"admin","userGroups":["users"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'create table default.testtable1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"user1","userGroups":["users","admin"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'drop table default.testtable1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"user1","userGroups":["users"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'drop table default.testtable1;' to user1/group1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"user1","userGroups":["users","group1"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'drop table default.testtable1;' to admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"admin","userGroups":["users"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'drop table default.testtable1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"user1","userGroups":["users","admin"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'create table default.table1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"create","user":"user1","userGroups":["users"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'create table default.table1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"create","user":"user1","userGroups":["users","admin"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'drop table default.table1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"drop","user":"user1","userGroups":["users"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'drop table default.table1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"drop","user":"user1","userGroups":["users","admin"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.table1;' to user3",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users"],"requestData":"select col1 from default.table1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY '_any access to db1/table1' for user1: table-level mismatch",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"table1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"show columns in table1 from db1;"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY '_any access to db1/_/col1' for user1: table not specified but column was specified",
+ "request":{
+ "resource":{"elements":{"database":"db1", "column":"col1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"fictional use case when request specified a lower level resource by skipping intermediate resource"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW '_any access to db1' for user1: match when request has less levels than policy",
+ "request":{
+ "resource":{"elements":{"database":"db1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"use db1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":3}
+ }
+ ,
+ {"name":"ALLOW '_any access to db1/table1' for user1: match when request has same levels as policy",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tbl1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"describe db1.tbl1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":3}
+ }
+ ,
+ {"name":"ALLOW '_any access to db1/tbl1/col1' for user1: match when request has more specific levels than policy",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tbl1", "column":"col1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"fictional case: request for any match today happens only at a higher levels"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":3}
+ }
+ ],
+ "updatedPolicies": [
+ {"changeType": 0,
+ "policy": {
+ "id":4,"version":1,"name":"db=db1; table=tmp; column=tmp*","isEnabled":true,"isAuditEnabled":true, "policyType":0,"serviceType":"hive",
+ "resources":{"database":{"values":["db1"]},"table":{"values":["tmp"]},"column":{"values":["tmp*"], "isExcludes":true}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ]
+ }
+ }
+ ],
+ "updatedTests": [
+ {"name":"ALLOW 'select abc_1 from db1.tmp ;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"abc_1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select abc_1 from db1.tmp for user1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":4}
+ }
+ ]
+}
+
diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hive_incremental_delete.json b/agents-common/src/test/resources/policyengine/test_policyengine_hive_incremental_delete.json
new file mode 100644
index 0000000..e60bfe9
--- /dev/null
+++ b/agents-common/src/test/resources/policyengine/test_policyengine_hive_incremental_delete.json
@@ -0,0 +1,352 @@
+{
+ "serviceName":"hivedev",
+
+ "serviceDef":{
+ "name":"hive",
+ "id":3,
+ "resources":[
+ {"name":"database","level":1,"mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Database","description":"Hive Database"},
+ {"name":"table","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Table","description":"Hive Table"},
+ {"name":"udf","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive UDF","description":"Hive UDF"},
+ {"name":"column","level":3,"parent":"table","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Column","description":"Hive Column"}
+ ],
+ "accessTypes":[
+ {"name":"select","label":"Select"},
+ {"name":"update","label":"Update"},
+ {"name":"create","label":"Create"},
+ {"name":"drop","label":"Drop"},
+ {"name":"alter","label":"Alter"},
+ {"name":"index","label":"Index"},
+ {"name":"lock","label":"Lock"},
+ {"name":"all","label":"All"}
+ ]
+ },
+
+ "policies":[
+ {"id":1,"name":"db=default: audit-all-access","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["default"]},"table":{"values":["*"]},"column":{"values":["*"]}},
+ "policyItems":[
+ {"accesses":[],"users":[],"groups":["public"],"delegateAdmin":false}
+ ]
+ }
+ ,
+ {"id":2,"name":"db=default; table=test*; column=*","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["default"]},"table":{"values":["test*"]},"column":{"values":["*"]}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ,
+ {"accesses":[{"type":"create","isAllowed":true},{"type":"drop","isAllowed":true}],"users":["admin"],"groups":["admin"],"delegateAdmin":true}
+ ]
+ }
+ ,
+ {"id":3,"name":"db=db1; table=tbl*; column=*","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["db1"]},"table":{"values":["tbl*"]},"column":{"values":["*"]}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ]
+ }
+ ,
+ {"id":4,"name":"db=db1; table=tmp; column=tmp*","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["db1"]},"table":{"values":["tmp"]},"column":{"values":["tmp*"], "isExcludes":true}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ],
+ "policyType":0
+ }
+ ],
+
+ "tests":[
+ {"name":"DENY 'select tmp_1 from db1.tmp ;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"tmp_1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select tmp_1 from db1.tmp for user1"
+ ,"remoteIPAddress":"1.1.1.1","forwardedAddresses":["127.0.0.1","10.10.10.10"]
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'select abc_1 from db1.tmp ;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"abc_1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select abc_1 from db1.tmp for user1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":4}
+ }
+ ,
+ {"name":"ALLOW 'use default;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'use default;' for user2",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user2","userGroups":["users"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'use default;' to user3",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'use default;' to group1",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users", "group1"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'use default;' to group2",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users", "group2"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'use default;' to user3/group3",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users", "group3"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'use finance;' to user3/group3",
+ "request":{
+ "resource":{"elements":{"database":"finance"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"use finance"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to user2",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user2","userGroups":["users"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.testtable;' to user3",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to group1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users","group1"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to group2",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users","group2"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.testtable;' to user3/group3",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users","group3"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.table1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1","column":"col1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select col1 from default.table1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'create table default.testtable1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"user1","userGroups":["users"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'create table default.testtable1;' to user1/group1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"user1","userGroups":["users","group1"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'create table default.testtable1;' to admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"admin","userGroups":["users"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'create table default.testtable1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"user1","userGroups":["users","admin"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'drop table default.testtable1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"user1","userGroups":["users"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'drop table default.testtable1;' to user1/group1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"user1","userGroups":["users","group1"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'drop table default.testtable1;' to admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"admin","userGroups":["users"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'drop table default.testtable1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"user1","userGroups":["users","admin"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'create table default.table1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"create","user":"user1","userGroups":["users"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'create table default.table1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"create","user":"user1","userGroups":["users","admin"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'drop table default.table1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"drop","user":"user1","userGroups":["users"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'drop table default.table1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"drop","user":"user1","userGroups":["users","admin"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.table1;' to user3",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users"],"requestData":"select col1 from default.table1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY '_any access to db1/table1' for user1: table-level mismatch",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"table1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"show columns in table1 from db1;"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY '_any access to db1/_/col1' for user1: table not specified but column was specified",
+ "request":{
+ "resource":{"elements":{"database":"db1", "column":"col1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"fictional use case when request specified a lower level resource by skipping intermediate resource"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW '_any access to db1' for user1: match when request has less levels than policy",
+ "request":{
+ "resource":{"elements":{"database":"db1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"use db1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":3}
+ }
+ ,
+ {"name":"ALLOW '_any access to db1/table1' for user1: match when request has same levels as policy",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tbl1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"describe db1.tbl1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":3}
+ }
+ ,
+ {"name":"ALLOW '_any access to db1/tbl1/col1' for user1: match when request has more specific levels than policy",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tbl1", "column":"col1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"fictional case: request for any match today happens only at a higher levels"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":3}
+ }
+ ],
+ "updatedPolicies": [
+ {"changeType": 2,
+ "policy": {
+ "id": 4,
+ "version": 1,
+ "policyType": 0,
+ "serviceType": "hive"
+ }
+ }
+ ],
+ "updatedTests": [
+ {"name":"DENY 'select abc_1 from db1.tmp ;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"abc_1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select abc_1 from db1.tmp for user1"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ]
+}
+
diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hive_incremental_update.json b/agents-common/src/test/resources/policyengine/test_policyengine_hive_incremental_update.json
new file mode 100644
index 0000000..dea1c2b
--- /dev/null
+++ b/agents-common/src/test/resources/policyengine/test_policyengine_hive_incremental_update.json
@@ -0,0 +1,351 @@
+{
+ "serviceName":"hivedev",
+
+ "serviceDef":{
+ "name":"hive",
+ "id":3,
+ "resources":[
+ {"name":"database","level":1,"mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Database","description":"Hive Database"},
+ {"name":"table","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Table","description":"Hive Table"},
+ {"name":"udf","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive UDF","description":"Hive UDF"},
+ {"name":"column","level":3,"parent":"table","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Column","description":"Hive Column"}
+ ],
+ "accessTypes":[
+ {"name":"select","label":"Select"},
+ {"name":"update","label":"Update"},
+ {"name":"create","label":"Create"},
+ {"name":"drop","label":"Drop"},
+ {"name":"alter","label":"Alter"},
+ {"name":"index","label":"Index"},
+ {"name":"lock","label":"Lock"},
+ {"name":"all","label":"All"}
+ ]
+ },
+
+ "policies":[
+ {"id":1,"name":"db=default: audit-all-access","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["default"]},"table":{"values":["*"]},"column":{"values":["*"]}},
+ "policyItems":[
+ {"accesses":[],"users":[],"groups":["public"],"delegateAdmin":false}
+ ]
+ }
+ ,
+ {"id":2,"name":"db=default; table=test*; column=*","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["default"]},"table":{"values":["test*"]},"column":{"values":["*"]}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ,
+ {"accesses":[{"type":"create","isAllowed":true},{"type":"drop","isAllowed":true}],"users":["admin"],"groups":["admin"],"delegateAdmin":true}
+ ]
+ }
+ ,
+ {"id":3,"name":"db=db1; table=tbl*; column=*","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["db1"]},"table":{"values":["tbl*"]},"column":{"values":["*"]}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ]
+ }
+ ,
+ {"id":4,"name":"db=db1; table=tmp; column=tmp*","isEnabled":true,"isAuditEnabled":true,
+ "resources":{"database":{"values":["db1"]},"table":{"values":["tmp"]},"column":{"values":["tmp*"], "isExcludes":true}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ],
+ "policyType": 0
+ }
+ ],
+
+ "tests":[
+ {"name":"DENY 'select tmp_1 from db1.tmp ;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"tmp_1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select tmp_1 from db1.tmp for user1"
+ ,"remoteIPAddress":"1.1.1.1","forwardedAddresses":["127.0.0.1","10.10.10.10"]
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'select abc_1 from db1.tmp ;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"abc_1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select abc_1 from db1.tmp for user1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":4}
+ }
+ ,
+ {"name":"ALLOW 'use default;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'use default;' for user2",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user2","userGroups":["users"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'use default;' to user3",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'use default;' to group1",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users", "group1"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'use default;' to group2",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users", "group2"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'use default;' to user3/group3",
+ "request":{
+ "resource":{"elements":{"database":"default"}},
+ "accessType":"","user":"user3","userGroups":["users", "group3"],"requestData":"use default"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'use finance;' to user3/group3",
+ "request":{
+ "resource":{"elements":{"database":"finance"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"use finance"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to user2",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user2","userGroups":["users"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.testtable;' to user3",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to group1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users","group1"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'select col1 from default.testtable;' to group2",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users","group2"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.testtable;' to user3/group3",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users","group3"],"requestData":"select col1 from default.testtable"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.table1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1","column":"col1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select col1 from default.table1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'create table default.testtable1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"user1","userGroups":["users"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'create table default.testtable1;' to user1/group1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"user1","userGroups":["users","group1"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'create table default.testtable1;' to admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"admin","userGroups":["users"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'create table default.testtable1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"create","user":"user1","userGroups":["users","admin"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'drop table default.testtable1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"user1","userGroups":["users"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'drop table default.testtable1;' to user1/group1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"user1","userGroups":["users","group1"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW 'drop table default.testtable1;' to admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"admin","userGroups":["users"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"ALLOW 'drop table default.testtable1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"testtable1"}},
+ "accessType":"drop","user":"user1","userGroups":["users","admin"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":2}
+ }
+ ,
+ {"name":"DENY 'create table default.table1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"create","user":"user1","userGroups":["users"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'create table default.table1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"create","user":"user1","userGroups":["users","admin"],"requestData":"create table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'drop table default.table1;' to user1",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"drop","user":"user1","userGroups":["users"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'drop table default.table1;' to user1/admin",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1"}},
+ "accessType":"drop","user":"user1","userGroups":["users","admin"],"requestData":"drop table default.testtable1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY 'select col1 from default.table1;' to user3",
+ "request":{
+ "resource":{"elements":{"database":"default","table":"table1","column":"col1"}},
+ "accessType":"select","user":"user3","userGroups":["users"],"requestData":"select col1 from default.table1"
+ },
+ "result":{"isAudited":true,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY '_any access to db1/table1' for user1: table-level mismatch",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"table1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"show columns in table1 from db1;"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"DENY '_any access to db1/_/col1' for user1: table not specified but column was specified",
+ "request":{
+ "resource":{"elements":{"database":"db1", "column":"col1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"fictional use case when request specified a lower level resource by skipping intermediate resource"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ,
+ {"name":"ALLOW '_any access to db1' for user1: match when request has less levels than policy",
+ "request":{
+ "resource":{"elements":{"database":"db1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"use db1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":3}
+ }
+ ,
+ {"name":"ALLOW '_any access to db1/table1' for user1: match when request has same levels as policy",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tbl1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"describe db1.tbl1"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":3}
+ }
+ ,
+ {"name":"ALLOW '_any access to db1/tbl1/col1' for user1: match when request has more specific levels than policy",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tbl1", "column":"col1"}},
+ "accessType":"","user":"user1","userGroups":["users"],"requestData":"fictional case: request for any match today happens only at a higher levels"
+ },
+ "result":{"isAudited":true,"isAllowed":true,"policyId":3}
+ }
+ ],
+ "updatedPolicies": [
+ {"changeType": 1,
+ "policy": {"id":4,"version":1,"name":"db=db1; table=tmp; column=tmp*","isEnabled":true,"isAuditEnabled":true,"serviceType":"hive","policyType":0,
+ "resources":{"database":{"values":["db1"]},"table":{"values":["tmp"]},"column":{"values":["tmp*"]}},
+ "policyItems":[
+ {"accesses":[{"type":"select","isAllowed":true}],"users":["user1","user2"],"groups":["group1","group2"],"delegateAdmin":false}
+ ]
+ }
+ }
+ ],
+ "updatedTests": [
+ {"name":"DENY 'select abc_1 from db1.tmp ;' for user1",
+ "request":{
+ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"abc_1"}},
+ "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select abc_1 from db1.tmp for user1"
+ },
+ "result":{"isAudited":false,"isAllowed":false,"policyId":-1}
+ }
+ ]
+}
\ No newline at end of file
diff --git a/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java b/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java
index d856f89..b04de1d 100644
--- a/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java
+++ b/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java
@@ -63,6 +63,7 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient {
String _sslConfigFileName = null;
String _serviceName = null;
String _clusterName = null;
+ String _supportsPolicyDeltas = null;
String _pluginId = null;
int _restClientConnTimeOutMs;
int _restClientReadTimeOutMs;
@@ -81,8 +82,12 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient {
_restClientConnTimeOutMs = RangerConfiguration.getInstance().getInt(configPropertyPrefix + ".policy.rest.client.connection.timeoutMs", 120 * 1000);
_restClientReadTimeOutMs = RangerConfiguration.getInstance().getInt(configPropertyPrefix + ".policy.rest.client.read.timeoutMs", 30 * 1000);
_clusterName = RangerConfiguration.getInstance().get(configPropertyPrefix + ".ambari.cluster.name", "");
+ _supportsPolicyDeltas = RangerConfiguration.getInstance().get(configPropertyPrefix + ".policy.rest.supports.policy.deltas", "false");
+ if (!"true".equalsIgnoreCase(_supportsPolicyDeltas)) {
+ _supportsPolicyDeltas = "false";
+ }
- LOG.info("Init params: " + String.format("Base URL[%s], SSL Congig filename[%s], ServiceName=[%s]", _baseUrl, _sslConfigFileName, _serviceName));
+ LOG.info("Init params: " + String.format("Base URL[%s], SSL Congig filename[%s], ServiceName=[%s], SupportsPolicyDeltas=[%s]", _baseUrl, _sslConfigFileName, _serviceName, _supportsPolicyDeltas));
_client = getClient();
_client.property(ClientProperties.CONNECT_TIMEOUT, _restClientConnTimeOutMs);
@@ -119,6 +124,7 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient {
.queryParam(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis))
.queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, _pluginId)
.queryParam(RangerRESTUtils.REST_PARAM_CLUSTER_NAME, _clusterName)
+ .queryParam(RangerRESTUtils.REST_PARAM_SUPPORTS_POLICY_DELTAS, _supportsPolicyDeltas)
.request(MediaType.APPLICATION_JSON_TYPE)
.get();
}
@@ -134,6 +140,7 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient {
.queryParam(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis))
.queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, _pluginId)
.queryParam(RangerRESTUtils.REST_PARAM_CLUSTER_NAME, _clusterName)
+ .queryParam(RangerRESTUtils.REST_PARAM_SUPPORTS_POLICY_DELTAS, _supportsPolicyDeltas)
.request(MediaType.APPLICATION_JSON_TYPE)
.get();
}
diff --git a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
index b46a481..59ed477 100644
--- a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
+++ b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql
@@ -15,6 +15,7 @@
DROP VIEW IF EXISTS `vx_trx_log`;
DROP TABLE IF EXISTS `x_security_zone_ref_resource`;
+DROP TABLE IF EXISTS `x_policy_change_log`;
DROP TABLE IF EXISTS `x_policy_ref_group`;
DROP TABLE IF EXISTS `x_policy_ref_user`;
DROP TABLE IF EXISTS `x_policy_ref_datamask_type`;
@@ -1415,8 +1416,23 @@ CREATE TABLE IF NOT EXISTS `x_security_zone_ref_resource`(
CONSTRAINT `x_sz_ref_resource_FK_upd_by_id` FOREIGN KEY (`upd_by_id`) REFERENCES `x_portal_user` (`id`),
CONSTRAINT `x_sz_ref_resource_FK_zone_id` FOREIGN KEY (`zone_id`) REFERENCES `x_security_zone` (`id`),
CONSTRAINT `x_sz_ref_resource_FK_resource_def_id` FOREIGN KEY (`resource_def_id`) REFERENCES `x_resource_def` (`id`)
-)ROW_FORMAT=DYNAMIC;
+) ROW_FORMAT=DYNAMIC;
+
+CREATE TABLE IF NOT EXISTS `x_policy_change_log` (
+`id` bigint(20) NOT NULL AUTO_INCREMENT,
+`create_time` datetime NULL DEFAULT NULL,
+`service_id` bigint(20) NOT NULL,
+`change_type` int(11) NOT NULL,
+`policy_version` bigint(20) NOT NULL DEFAULT '0',
+`service_type` varchar(256) NULL DEFAULT NULL,
+`policy_type` int(11) NULL DEFAULT NULL,
+`zone_name` varchar(256) NULL DEFAULT NULL,
+`policy_id` bigint(20) NULL DEFAULT NULL,
+primary key (`id`)
+) ROW_FORMAT=DYNAMIC;
+CREATE INDEX x_policy_change_log_IDX_service_id ON x_policy_change_log(service_id);
+CREATE INDEX x_policy_change_log_IDX_policy_version ON x_policy_change_log(policy_version);
CREATE INDEX x_service_config_def_IDX_def_id ON x_service_config_def(def_id);
CREATE INDEX x_resource_def_IDX_def_id ON x_resource_def(def_id);
CREATE INDEX x_access_type_def_IDX_def_id ON x_access_type_def(def_id);
@@ -1520,6 +1536,7 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('035',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('036',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('037',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y');
+INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('038',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',UTC_TIMESTAMP(),'Ranger 1.0.0',UTC_TIMESTAMP(),'localhost','Y');
INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed)
diff --git a/security-admin/db/mysql/patches/038-add-policy-change-log-table.sql b/security-admin/db/mysql/patches/038-add-policy-change-log-table.sql
new file mode 100644
index 0000000..758b38a
--- /dev/null
+++ b/security-admin/db/mysql/patches/038-add-policy-change-log-table.sql
@@ -0,0 +1,33 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS `x_policy_change_log`;
+
+CREATE TABLE IF NOT EXISTS `x_policy_change_log` (
+`id` bigint(20) NOT NULL AUTO_INCREMENT,
+`create_time` datetime NULL DEFAULT NULL,
+`service_id` bigint(20) NOT NULL,
+`change_type` int(11) NOT NULL,
+`policy_version` bigint(20) NOT NULL DEFAULT '0',
+`service_type` varchar(256) NULL DEFAULT NULL,
+`policy_type` int(11) NULL DEFAULT NULL,
+`zone_name` varchar(256) NULL DEFAULT NULL,
+`policy_id` bigint(20) NULL DEFAULT NULL,
+primary key (`id`)
+) ROW_FORMAT=DYNAMIC;
+
+CREATE INDEX x_policy_change_log_IDX_service_id ON x_policy_change_log(service_id);
+CREATE INDEX x_policy_change_log_IDX_policy_version ON x_policy_change_log(policy_version);
+
diff --git a/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql b/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql
index 35c70c7..e1dc7e0 100644
--- a/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql
+++ b/security-admin/db/oracle/optimized/current/ranger_core_db_oracle.sql
@@ -84,13 +84,13 @@ call spdropsequence('X_PLUGIN_INFO_SEQ');
call spdropsequence('X_POLICY_LABEL_MAP_SEQ');
call spdropsequence('X_POLICY_LABEL_SEQ');
call spdropsequence('X_UGSYNC_AUDIT_INFO_SEQ');
-call spdropsequence('X_SEC_ZONE_REF_GROUP_SEQ');
-call spdropsequence('X_SEC_ZONE_REF_USER_SEQ');
-call spdropsequence('X_SEC_ZONE_REF_RESOURCE_SEQ');
-call spdropsequence('X_SEC_ZONE_REF_SERVICE_SEQ');
+call spdropsequence('X_SZONE_REF_ADMIN_GROUP_SEQ');
+call spdropsequence('X_SZONE_REF_ADMIN_USER_SEQ');
+call spdropsequence('X_SZONE_REF_RESOURCE_SEQ');
+call spdropsequence('X_SZONE_REF_SERVICE_SEQ');
call spdropsequence('X_RANGER_GLOBAL_STATE_SEQ');
call spdropsequence('X_SECURITY_ZONE_SEQ');
-
+call spdropsequence('X_POLICY_CHANGE_LOG_SEQ');
CREATE SEQUENCE SEQ_GEN_IDENTITY START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
CREATE SEQUENCE X_ACCESS_AUDIT_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
CREATE SEQUENCE X_ASSET_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
@@ -153,10 +153,11 @@ CREATE SEQUENCE X_POLICY_LABEL_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
CREATE SEQUENCE X_UGSYNC_AUDIT_INFO_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
CREATE SEQUENCE X_SECURITY_ZONE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
CREATE SEQUENCE X_RANGER_GLOBAL_STATE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
-CREATE SEQUENCE X_SEC_ZONE_REF_SERVICE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
-CREATE SEQUENCE X_SEC_ZONE_REF_RESOURCE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
-CREATE SEQUENCE X_SEC_ZONE_REF_USER_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
-CREATE SEQUENCE X_SEC_ZONE_REF_GROUP_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
+CREATE SEQUENCE X_SZONE_REF_SERVICE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
+CREATE SEQUENCE X_SZONE_REF_RESOURCE_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
+CREATE SEQUENCE X_SZONE_REF_ADMIN_USER_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
+CREATE SEQUENCE X_SZONE_REF_ADMIN_GROUP_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
+CREATE SEQUENCE X_POLICY_CHANGE_LOG_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
call spdropsequence('X_DB_VERSION_H_SEQ');
CREATE SEQUENCE X_DB_VERSION_H_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
commit;
@@ -186,6 +187,7 @@ END;/
call spdropview('vx_trx_log');
call spdroptable('x_security_zone_ref_resource');
+call spdroptable('x_policy_change_log');
call spdroptable('x_policy_ref_group');
call spdroptable('x_policy_ref_user');
call spdroptable('x_policy_ref_datamask_type');
@@ -1438,6 +1440,38 @@ CONSTRAINT x_p_ref_grp_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_por
CONSTRAINT x_p_ref_grp_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id)
);
commit;
+CREATE TABLE x_policy_change_log(
+id NUMBER(20) NOT NULL,
+create_time DATE DEFAULT NULL NULL,
+service_id NUMBER(20) NOT NULL,
+change_type NUMBER(11) NOT NULL,
+policy_version NUMBER(20) DEFAULT '0' NOT NULL,
+service_type VARCHAR(256) DEFAULT NULL NULL,
+policy_type NUMBER(11) DEFAULT NULL NULL,
+zone_name VARCHAR(256) DEFAULT NULL NULL,
+policy_id NUMBER(20) DEFAULT NULL NULL,
+ PRIMARY KEY (id)
+);
+CREATE INDEX x_plcy_chng_log_IDX_service_id ON x_policy_change_log(service_id);
+CREATE INDEX x_plcy_chng_log_IDX_policy_ver ON x_policy_change_log(policy_version);
+COMMIT;
+
+CREATE TABLE x_security_zone_ref_resource (
+id NUMBER(20) NOT NULL,
+create_time DATE DEFAULT NULL NULL,
+update_time DATE DEFAULT NULL NULL,
+added_by_id NUMBER(20) DEFAULT NULL NULL,
+upd_by_id NUMBER(20) DEFAULT NULL NULL,
+zone_id NUMBER(20) DEFAULT NULL NULL,
+resource_def_id NUMBER(20) DEFAULT NULL NULL,
+resource_name VARCHAR(255) DEFAULT NULL NULL,
+primary key (id),
+CONSTRAINT x_sz_ref_res_FK_added_by_id FOREIGN KEY (added_by_id) REFERENCES x_portal_user (id),
+CONSTRAINT x_sz_ref_res_FK_upd_by_id FOREIGN KEY (upd_by_id) REFERENCES x_portal_user (id),
+CONSTRAINT x_sz_ref_res_FK_zone_id FOREIGN KEY (zone_id) REFERENCES x_security_zone (id),
+CONSTRAINT x_sz_ref_res_FK_res_def_id FOREIGN KEY (resource_def_id) REFERENCES x_resource_def (id)
+);
+commit;
CREATE TABLE x_security_zone_ref_resource (
id NUMBER(20) NOT NULL,
@@ -1686,6 +1720,7 @@ INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,act
INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval, '035',sys_extract_utc(systimestamp),'Ranger 1.0.0',sys_extract_utc(systimestamp),'localhost','Y');
INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval, '036',sys_extract_utc(systimestamp),'Ranger 1.0.0',sys_extract_utc(systimestamp),'localhost','Y');
INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval, '037',sys_extract_utc(systimestamp),'Ranger 1.0.0',sys_extract_utc(systimestamp),'localhost','Y');
+INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval, '038',sys_extract_utc(systimestamp),'Ranger 1.0.0',sys_extract_utc(systimestamp),'localhost','Y');
INSERT INTO x_db_version_h (id,version,inst_at,inst_by,updated_at,updated_by,active) VALUES (X_DB_VERSION_H_SEQ.nextval, 'DB_PATCHES',sys_extract_utc(systimestamp),'Ranger 1.0.0',sys_extract_utc(systimestamp),'localhost','Y');
INSERT INTO x_user_module_perm (id,user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (X_USER_MODULE_PERM_SEQ.nextval,getXportalUIdByLoginId('admin'),getModulesIdByName('Reports'),sys_extract_utc(systimestamp),sys_extract_utc(systimestamp),getXportalUIdByLoginId('admin'),getXportalUIdByLoginId('admin'),1);
INSERT INTO x_user_module_perm (id,user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (X_USER_MODULE_PERM_SEQ.nextval,getXportalUIdByLoginId('admin'),getModulesIdByName('Resource Based Policies'),sys_extract_utc(systimestamp),sys_extract_utc(systimestamp),getXportalUIdByLoginId('admin'),getXportalUIdByLoginId('admin'),1);
diff --git a/security-admin/db/oracle/patches/038-add-policy-change-log-table.sql b/security-admin/db/oracle/patches/038-add-policy-change-log-table.sql
new file mode 100644
index 0000000..10ce644
--- /dev/null
+++ b/security-admin/db/oracle/patches/038-add-policy-change-log-table.sql
@@ -0,0 +1,57 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE OR REPLACE PROCEDURE spdropsequence(ObjName IN varchar2)
+IS
+v_counter integer;
+BEGIN
+ select count(*) into v_counter from user_sequences where sequence_name = upper(ObjName);
+ if (v_counter > 0) then
+ execute immediate 'DROP SEQUENCE ' || ObjName;
+ end if;
+END;/
+/
+
+call spdropsequence('X_POLICY_CHANGE_LOG_SEQ');
+
+CREATE OR REPLACE PROCEDURE spdroptable(ObjName IN varchar2)
+IS
+v_counter integer;
+BEGIN
+ select count(*) into v_counter from user_tables where table_name = upper(ObjName);
+ if (v_counter > 0) then
+ execute immediate 'drop table ' || ObjName || ' cascade constraints';
+ end if;
+END;/
+/
+
+call spdroptable('X_POLICY_CHANGE_LOG');
+
+CREATE SEQUENCE X_POLICY_CHANGE_LOG_SEQ START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE;
+CREATE TABLE x_policy_change_log(
+id NUMBER(20) NOT NULL,
+create_time DATE DEFAULT NULL NULL,
+service_id NUMBER(20) NOT NULL,
+change_type NUMBER(11) NOT NULL,
+policy_version NUMBER(20) DEFAULT '0' NOT NULL,
+service_type VARCHAR(256) DEFAULT NULL NULL,
+policy_type NUMBER(11) DEFAULT NULL NULL,
+zone_name VARCHAR(256) DEFAULT NULL NULL,
+policy_id NUMBER(20) DEFAULT NULL NULL,
+ PRIMARY KEY (id)
+);
+CREATE INDEX x_plcy_chng_log_IDX_service_id ON x_policy_change_log(service_id);
+CREATE INDEX x_plcy_chng_log_IDX_policy_ver ON x_policy_change_log(policy_version);
+COMMIT;
\ No newline at end of file
diff --git a/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql b/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql
index dfa8c82..dae8086 100644
--- a/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql
+++ b/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql
@@ -14,6 +14,7 @@
-- limitations under the License.
DROP TABLE IF EXISTS x_security_zone_ref_resource CASCADE;
+DROP TABLE IF EXISTS x_policy_change_log;
DROP TABLE IF EXISTS x_policy_ref_group CASCADE;
DROP TABLE IF EXISTS x_policy_ref_user CASCADE;
DROP TABLE IF EXISTS x_policy_ref_datamask_type CASCADE;
@@ -82,12 +83,13 @@ DROP TABLE IF EXISTS x_portal_user_role CASCADE;
DROP TABLE IF EXISTS x_portal_user CASCADE;
DROP TABLE IF EXISTS x_db_version_h CASCADE;
-DROP SEQUENCE IF EXISTS x_sec_zone_ref_group_seq;
-DROP SEQUENCE IF EXISTS x_sec_zone_ref_user_seq;
-DROP SEQUENCE IF EXISTS x_sec_zone_ref_resource_seq;
-DROP SEQUENCE IF EXISTS x_sec_zone_ref_service_seq;
+DROP SEQUENCE IF EXISTS x_security_zone_ref_group_seq;
+DROP SEQUENCE IF EXISTS x_security_zone_ref_user_seq;
+DROP SEQUENCE IF EXISTS x_security_zone_ref_resource_seq;
+DROP SEQUENCE IF EXISTS x_security_zone_ref_service_seq;
DROP SEQUENCE IF EXISTS x_ranger_global_state_seq;
DROP SEQUENCE IF EXISTS x_security_zone_seq;
+DROP SEQUENCE IF EXISTS x_policy_change_log_seq;
DROP SEQUENCE IF EXISTS x_policy_ref_group_seq;
DROP SEQUENCE IF EXISTS x_policy_ref_user_seq;
DROP SEQUENCE IF EXISTS x_policy_ref_datamask_type_seq;
@@ -1406,6 +1408,23 @@ CONSTRAINT x_sz_ref_group_FK_zone_id FOREIGN KEY (zone_id) REFERENCES x_security
CONSTRAINT x_sz_ref_group_FK_group_id FOREIGN KEY (group_id) REFERENCES x_group (id)
);
commit;
+CREATE SEQUENCE x_policy_change_log_seq;
+CREATE TABLE x_policy_change_log (
+id BIGINT DEFAULT nextval('x_policy_change_log_seq'::regclass),
+create_time TIMESTAMP DEFAULT NULL NULL,
+service_id bigint NOT NULL,
+change_type int NOT NULL,
+policy_version bigint DEFAULT '0' NOT NULL,
+service_type varchar(256) DEFAULT NULL NULL,
+policy_type int DEFAULT NULL NULL,
+zone_name varchar(256) DEFAULT NULL NULL,
+policy_id bigint DEFAULT NULL NULL,
+primary key (id)
+);
+commit;
+CREATE INDEX x_policy_change_log_IDX_service_id ON x_policy_change_log(service_id);
+CREATE INDEX x_policy_change_log_IDX_policy_version ON x_policy_change_log(policy_version);
+commit;
CREATE INDEX xa_access_audit_added_by_id ON xa_access_audit(added_by_id);
CREATE INDEX xa_access_audit_upd_by_id ON xa_access_audit(upd_by_id);
@@ -1610,6 +1629,7 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('035',current_timestamp,'Ranger 1.0.0',current_timestamp,'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('036',current_timestamp,'Ranger 1.0.0',current_timestamp,'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('037',current_timestamp,'Ranger 1.0.0',current_timestamp,'localhost','Y');
+INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('038',current_timestamp,'Ranger 1.0.0',current_timestamp,'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',current_timestamp,'Ranger 1.0.0',current_timestamp,'localhost','Y');
INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES
diff --git a/security-admin/db/postgres/patches/038-add-policy-change-log-table.sql b/security-admin/db/postgres/patches/038-add-policy-change-log-table.sql
new file mode 100644
index 0000000..a10cb55
--- /dev/null
+++ b/security-admin/db/postgres/patches/038-add-policy-change-log-table.sql
@@ -0,0 +1,35 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS x_policy_change_log;
+DROP SEQUENCE IF EXISTS x_policy_change_log_seq;
+
+CREATE SEQUENCE x_policy_change_log_seq;
+
+CREATE TABLE x_policy_change_log (
+id BIGINT DEFAULT nextval('x_policy_change_log_seq'::regclass),
+create_time TIMESTAMP DEFAULT NULL NULL,
+service_id bigint NOT NULL,
+change_type int NOT NULL,
+policy_version bigint DEFAULT '0' NOT NULL,
+zone_name varchar(256) DEFAULT NULL NULL,
+policy_type int DEFAULT NULL NULL,
+
+primary key (id)
+);
+commit;
+CREATE INDEX x_policy_change_log_IDX_service_id ON x_policy_change_log(service_id);
+CREATE INDEX x_policy_change_log_IDX_policy_version ON x_policy_change_log(policy_version);
+commit;
diff --git a/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql b/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql
index 81c6172..1a5db0b 100644
--- a/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql
+++ b/security-admin/db/sqlanywhere/optimized/current/ranger_core_db_sqlanywhere.sql
@@ -36,6 +36,7 @@ BEGIN
END
GO
call dbo.removeForeignKeysAndTable('x_security_zone_ref_resource')
+call dbo.removeForeignKeysAndTable('x_policy_change_log')
GO
call dbo.removeForeignKeysAndTable('x_policy_ref_group')
GO
@@ -1176,6 +1177,20 @@ CREATE TABLE dbo.x_security_zone_ref_group(
CONSTRAINT x_sz_ref_agroup_PK_id PRIMARY KEY CLUSTERED(id)
)
GO
+CREATE TABLE dbo.x_policy_change_log(
+ id bigint IDENTITY NOT NULL,
+ create_time datetime DEFAULT NULL NULL,
+ service_id bigint NOT NULL,
+ change_type int NOT NULL,
+ policy_version bigint DEFAULT 0 NOT NULL,
+ service_type varchar(256) DEFAULT NULL NULL,
+ policy_type int DEFAULT NULL NULL,
+ zone_name varchar(256) DEFAULT NULL NULL,
+ policy_id bigint DEFAULT NULL NULL,
+ CONSTRAINT x_policy_change_log_PK_id PRIMARY KEY CLUSTERED(id)
+)
+GO
+
ALTER TABLE dbo.x_asset ADD CONSTRAINT x_asset_FK_added_by_id FOREIGN KEY(added_by_id) REFERENCES dbo.x_portal_user(id)
GO
ALTER TABLE dbo.x_asset ADD CONSTRAINT x_asset_FK_upd_by_id FOREIGN KEY(upd_by_id) REFERENCES dbo.x_portal_user (id)
@@ -1863,6 +1878,10 @@ BEGIN
RETURN (myid);
END;
+CREATE NONCLUSTERED INDEX x_policy_change_log_IDX_service_id ON dbo.x_policy_change_log(service_id ASC)
+GO
+CREATE NONCLUSTERED INDEX x_policy_change_log_IDX_policy_version ON dbo.x_policy_change_log(policy_version ASC)
+GO
insert into x_portal_user (create_time,update_time,first_name,last_name,pub_scr_name,login_id,password,email,status) values (GETDATE(),GETDATE(),'Admin','','Admin','admin','ceb4f32325eda6142bd65215f4c0f371','',1)
GO
insert into x_portal_user_role (create_time,update_time,user_id,user_role,status) values (GETDATE(),GETDATE(),dbo.getXportalUIdByLoginId('admin'),'ROLE_SYS_ADMIN',1)
@@ -1947,6 +1966,8 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active
GO
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('037',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y');
GO
+INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('038',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y');
+GO
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y');
GO
INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (dbo.getXportalUIdByLoginId('admin'),dbo.getModulesIdByName('Reports'),CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,dbo.getXportalUIdByLoginId('admin'),dbo.getXportalUIdByLoginId('admin'),1);
diff --git a/security-admin/db/sqlanywhere/patches/038-add-policy-change-log-table.sql b/security-admin/db/sqlanywhere/patches/038-add-policy-change-log-table.sql
new file mode 100644
index 0000000..a95cec7
--- /dev/null
+++ b/security-admin/db/sqlanywhere/patches/038-add-policy-change-log-table.sql
@@ -0,0 +1,64 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE OR REPLACE PROCEDURE dbo.removeForeignKeysAndTable (IN table_name varchar(100))
+AS
+BEGIN
+ DECLARE @stmt VARCHAR(300)
+ DECLARE @tblname VARCHAR(300)
+ DECLARE @drpstmt VARCHAR(1000)
+ DECLARE cur CURSOR FOR select 'alter table dbo.' + table_name + ' drop constraint ' + role from SYS.SYSFOREIGNKEYS where foreign_creator ='dbo' and foreign_tname = table_name
+ OPEN cur WITH HOLD
+ fetch cur into @stmt
+ WHILE (@@sqlstatus = 0)
+ BEGIN
+ execute(@stmt)
+ fetch cur into @stmt
+ END
+ close cur
+ DEALLOCATE CURSOR cur
+ SET @tblname ='dbo.' + table_name;
+ SET @drpstmt = 'DROP TABLE IF EXISTS ' + @tblname;
+ execute(@drpstmt)
+END
+GO
+call dbo.removeForeignKeysAndTable('x_policy_change_log')
+GO
+
+CREATE TABLE dbo.x_policy_change_log(
+ id bigint IDENTITY NOT NULL,
+ create_time datetime DEFAULT NULL NULL,
+ service_id bigint NOT NULL,
+ change_type int NOT NULL,
+ policy_version bigint DEFAULT 0 NOT NULL,
+ service_type varchar(256) DEFAULT NULL NULL,
+ zone_name varchar(256) DEFAULT NULL NULL,
+ policy_type int DEFAULT NULL NULL,
+ policy_id bigint DEFAULT NULL NULL,
+ CONSTRAINT x_policy_change_log_PK_id PRIMARY KEY CLUSTERED(id)
+)
+GO
+CREATE NONCLUSTERED INDEX x_policy_change_log_IDX_service_id ON dbo.x_policy_change_log(service_id ASC)
+GO
+CREATE NONCLUSTERED INDEX x_policy_change_log_IDX_policy_version ON dbo.x_policy_change_log(policy_version ASC)
+GO
+exit
+
+
+
+
+
+
+
diff --git a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql
index 845e089..bf33ae1 100644
--- a/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql
+++ b/security-admin/db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql
@@ -601,6 +601,10 @@ IF (OBJECT_ID('vx_trx_log') IS NOT NULL)
BEGIN
DROP VIEW [dbo].[vx_trx_log]
END
+IF (OBJECT_ID('x_policy_change_log') IS NOT NULL)
+BEGIN
+ DROP TABLE [dbo].[x_policy_change_log]
+END
IF (OBJECT_ID('x_policy_ref_group') IS NOT NULL)
BEGIN
DROP TABLE [dbo].[x_policy_ref_group]
@@ -2318,6 +2322,36 @@ GO
SET ANSI_NULLS ON
SET QUOTED_IDENTIFIER ON
SET ANSI_PADDING ON
+CREATE TABLE [dbo].[x_policy_change_log](
+ [id] [bigint] IDENTITY(1,1) NOT NULL,
+ [create_time] [datetime2] DEFAULT NULL NULL,
+ [service_id] [bigint] NOT NULL,
+ [change_type] [int] NOT NULL,
+ [policy_version] [bigint] DEFAULT 0 NOT NULL,
+ [service_type] [varchar](256) DEFAULT NULL NULL,
+ [policy_type] [int] DEFAULT NULL NULL,
+ [zone_name] [varchar](256) DEFAULT NULL NULL,
+ [policy_id] [bigint] DEFAULT NULL NULL,
+ PRIMARY KEY CLUSTERED
+(
+ [id] ASC
+)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+) ON [PRIMARY]
+CREATE NONCLUSTERED INDEX [x_policy_change_log_IDX_service_id] ON [x_policy_change_log]
+(
+ [service_id] ASC
+)
+WITH (SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, IGNORE_DUP_KEY = OFF, ONLINE = OFF) ON [PRIMARY]
+CREATE NONCLUSTERED INDEX [x_policy_change_log_IDX_policy_version] ON [x_policy_change_log]
+(
+ [policy_version] ASC
+)
+WITH (SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, IGNORE_DUP_KEY = OFF, ONLINE = OFF) ON [PRIMARY]
+SET ANSI_NULLS ON
+SET QUOTED_IDENTIFIER ON
+SET ANSI_PADDING ON
+
+
ALTER TABLE [dbo].[x_asset] WITH CHECK ADD CONSTRAINT [x_asset_FK_added_by_id] FOREIGN KEY([added_by_id])
REFERENCES [dbo].[x_portal_user] ([id])
ALTER TABLE [dbo].[x_asset] CHECK CONSTRAINT [x_asset_FK_added_by_id]
@@ -3594,6 +3628,7 @@ INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('035',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('036',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('037',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y');
+INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('038',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y');
INSERT INTO x_db_version_h (version,inst_at,inst_by,updated_at,updated_by,active) VALUES ('DB_PATCHES',CURRENT_TIMESTAMP,'Ranger 1.0.0',CURRENT_TIMESTAMP,'localhost','Y');
INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (dbo.getXportalUIdByLoginId('admin'),dbo.getModulesIdByName('Reports'),CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,dbo.getXportalUIdByLoginId('admin'),dbo.getXportalUIdByLoginId('admin'),1);
INSERT INTO x_user_module_perm (user_id,module_id,create_time,update_time,added_by_id,upd_by_id,is_allowed) VALUES (dbo.getXportalUIdByLoginId('admin'),dbo.getModulesIdByName('Resource Based Policies'),CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,dbo.getXportalUIdByLoginId('admin'),dbo.getXportalUIdByLoginId('admin'),1);
diff --git a/security-admin/db/sqlserver/patches/038-add-policy-change-log-table.sql b/security-admin/db/sqlserver/patches/038-add-policy-change-log-table.sql
new file mode 100644
index 0000000..1762194
--- /dev/null
+++ b/security-admin/db/sqlserver/patches/038-add-policy-change-log-table.sql
@@ -0,0 +1,56 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+GO
+IF (OBJECT_ID('x_policy_change_log') IS NOT NULL)
+BEGIN
+ DROP TABLE [dbo].[x_policy_change_log]
+END
+GO
+SET ANSI_NULLS ON
+GO
+SET QUOTED_IDENTIFIER ON
+GO
+SET ANSI_PADDING ON
+GO
+CREATE TABLE [dbo].[x_policy_change_log](
+ [id] [bigint] IDENTITY(1,1) NOT NULL,
+ [create_time] [datetime2] DEFAULT NULL NULL,
+ [service_id] [bigint] NOT NULL,
+ [change_type] [int] NOT NULL,
+ [policy_version] [bigint] DEFAULT 0 NOT NULL,
+ [service_type] [varchar](256) DEFAULT NULL NULL,
+ [policy_type] [int] DEFAULT NULL NULL,
+ [zone_name] [varchar](256) DEFAULT NULL NULL,
+ [policy_id] [bigint] DEFAULT NULL NULL,
+ PRIMARY KEY CLUSTERED
+(
+ [id] ASC
+)WITH (PAD_INDEX = OFF,STATISTICS_NORECOMPUTE = OFF,IGNORE_DUP_KEY = OFF,ALLOW_ROW_LOCKS = ON,ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
+) ON [PRIMARY]
+GO
+CREATE NONCLUSTERED INDEX [x_policy_change_log_IDX_service_id] ON [x_policy_change_log]
+(
+ [service_id] ASC
+)
+WITH (SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, IGNORE_DUP_KEY = OFF, ONLINE = OFF) ON [PRIMARY]
+GO
+CREATE NONCLUSTERED INDEX [x_policy_change_log_IDX_policy_version] ON [x_policy_change_log]
+(
+ [policy_version] ASC
+)
+WITH (SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, IGNORE_DUP_KEY = OFF, ONLINE = OFF) ON [PRIMARY]
+GO
+exit
\ No newline at end of file
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index e1b244d..4b1c0c4 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -70,6 +70,7 @@ import org.apache.ranger.plugin.model.RangerSecurityZone;
import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator;
import org.apache.ranger.plugin.model.validation.RangerValidator;
import org.apache.ranger.plugin.model.validation.ValidationFailureDetails;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
@@ -126,6 +127,7 @@ import org.apache.ranger.plugin.store.AbstractServiceStore;
import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
import org.apache.ranger.plugin.store.PList;
import org.apache.ranger.plugin.store.ServicePredicateUtil;
+import org.apache.ranger.plugin.util.RangerPolicyDeltaUtil;
import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.plugin.util.ServicePolicies;
import org.apache.ranger.rest.ServiceREST;
@@ -199,7 +201,9 @@ public class ServiceDBStore extends AbstractServiceStore {
public static final String ENCRYPT_KEY = PropertiesUtil.getProperty("ranger.password.encryption.key", PasswordUtils.DEFAULT_ENCRYPT_KEY);
public static final String SALT = PropertiesUtil.getProperty("ranger.password.salt", PasswordUtils.DEFAULT_SALT);
public static final Integer ITERATION_COUNT = PropertiesUtil.getIntProperty("ranger.password.iteration.count", PasswordUtils.DEFAULT_ITERATION_COUNT);
-
+ public static final boolean SUPPORTS_POLICY_DELTAS = RangerConfiguration.getInstance().getBoolean("ranger.admin.supports.policy.deltas", false);
+ public static final Integer RETENTION_PERIOD_IN_DAYS = RangerConfiguration.getInstance().getInt("ranger.admin.delta.retention.time.in.days", 7);
+
static {
try {
LOCAL_HOSTNAME = java.net.InetAddress.getLocalHost().getCanonicalHostName();
@@ -330,6 +334,7 @@ public class ServiceDBStore extends AbstractServiceStore {
EmbeddedServiceDefsUtil.instance().init(dbStore);
getServiceUpgraded();
createGenericUsers();
+ resetPolicyUpdateLog(RETENTION_PERIOD_IN_DAYS, false);
return null;
}
});
@@ -1577,7 +1582,7 @@ public class ServiceDBStore extends AbstractServiceStore {
service = svcService.update(service);
if (hasTagServiceValueChanged || hasIsEnabledChanged) {
- updatePolicyVersion(service);
+ updatePolicyVersion(service, RangerPolicyDelta.CHANGE_TYPE_SERVICE_CHANGE, null);
}
}
@@ -1868,8 +1873,10 @@ public class ServiceDBStore extends AbstractServiceStore {
XXPolicy xCreatedPolicy = daoMgr.getXXPolicy().getById(policy.getId());
policyRefUpdater.createNewPolMappingForRefTable(policy, xCreatedPolicy, xServiceDef);
createNewLabelsForPolicy(xCreatedPolicy, policyLabels);
- handlePolicyUpdate(service);
+
RangerPolicy createdPolicy = policyService.getPopulatedViewObject(xCreatedPolicy);
+
+ handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE, createdPolicy);
dataHistService.createObjectDataHistory(createdPolicy, RangerDataHistService.ACTION_CREATE);
List<XXTrxLog> trxLogList = policyService.getTransactionLog(createdPolicy, RangerPolicyService.OPERATION_CREATE_CONTEXT);
@@ -1963,10 +1970,12 @@ public class ServiceDBStore extends AbstractServiceStore {
policyRefUpdater.cleanupRefTables(policy);
deleteExistingPolicyLabel(policy);
+
policyRefUpdater.createNewPolMappingForRefTable(policy, newUpdPolicy, xServiceDef);
createNewLabelsForPolicy(newUpdPolicy, policyLabels);
- handlePolicyUpdate(service);
+
RangerPolicy updPolicy = policyService.getPopulatedViewObject(newUpdPolicy);
+ handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE, updPolicy);
dataHistService.createObjectDataHistory(updPolicy, RangerDataHistService.ACTION_UPDATE);
bizUtil.createTrxLog(trxLogList);
@@ -2008,8 +2017,9 @@ public class ServiceDBStore extends AbstractServiceStore {
policyRefUpdater.cleanupRefTables(policy);
deleteExistingPolicyLabel(policy);
policyService.delete(policy);
- handlePolicyUpdate(service);
-
+
+ handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE, policy);
+
dataHistService.createObjectDataHistory(policy, RangerDataHistService.ACTION_DELETE);
bizUtil.createTrxLog(trxLogList);
@@ -2197,8 +2207,8 @@ public class ServiceDBStore extends AbstractServiceStore {
List<RangerPolicy> ret = null;
- ServicePolicies servicePolicies = RangerServicePoliciesCache.getInstance().getServicePolicies(service.getName(), service.getId(), this);
- List<RangerPolicy> policies = servicePolicies != null ? servicePolicies.getPolicies() : null;
+ ServicePolicies servicePolicies = RangerServicePoliciesCache.getInstance().getServicePolicies(service.getName(), service.getId(), -1L, true, this);
+ final List<RangerPolicy> policies = servicePolicies != null ? servicePolicies.getPolicies() : null;
if(policies != null && filter != null) {
Map<String, String> filterResources = filter.getParamsWithPrefix(SearchFilter.RESOURCE_PREFIX, true);
@@ -2217,7 +2227,7 @@ public class ServiceDBStore extends AbstractServiceStore {
LOG.debug("Using" + (useLegacyResourceSearch ? " old " : " new ") + "way of filtering service-policies");
}
- ret = new ArrayList<RangerPolicy>(policies);
+ ret = new ArrayList<>(policies);
predicateUtil.applyFilter(ret, filter);
if (!useLegacyResourceSearch && CollectionUtils.isNotEmpty(ret)) {
@@ -2402,9 +2412,9 @@ public class ServiceDBStore extends AbstractServiceStore {
}
@Override
- public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception {
+ public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion, boolean needsBackwardCompatibility) throws Exception {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ")");
+ LOG.debug("==> ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + needsBackwardCompatibility + ")");
}
ServicePolicies ret = null;
@@ -2422,7 +2432,7 @@ public class ServiceDBStore extends AbstractServiceStore {
}
if (lastKnownVersion == null || serviceVersionInfoDbObj == null || serviceVersionInfoDbObj.getPolicyVersion() == null || !lastKnownVersion.equals(serviceVersionInfoDbObj.getPolicyVersion())) {
- ret = RangerServicePoliciesCache.getInstance().getServicePolicies(serviceName, serviceDbObj.getId(), this);
+ ret = RangerServicePoliciesCache.getInstance().getServicePolicies(serviceName, serviceDbObj.getId(), lastKnownVersion, needsBackwardCompatibility, this);
}
if (ret != null && lastKnownVersion != null && lastKnownVersion.equals(ret.getPolicyVersion())) {
@@ -2435,7 +2445,7 @@ public class ServiceDBStore extends AbstractServiceStore {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
+ LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + needsBackwardCompatibility + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
}
return ret;
@@ -2450,9 +2460,20 @@ public class ServiceDBStore extends AbstractServiceStore {
}
@Override
- public ServicePolicies getServicePolicies(String serviceName) throws Exception {
+ public ServicePolicies getServicePolicyDeltasOrPolicies(String serviceName, Long lastKnownVersion) throws Exception {
+ boolean getOnlyDeltas = false;
+ return getServicePolicies(serviceName, lastKnownVersion, getOnlyDeltas);
+ }
+
+ @Override
+ public ServicePolicies getOnlyServicePolicyDeltas(String serviceName, Long lastKnownVersion) throws Exception {
+ boolean getOnlyDeltas = true;
+ return getServicePolicies(serviceName, lastKnownVersion, getOnlyDeltas);
+ }
+
+ private ServicePolicies getServicePolicies(String serviceName, Long lastKnownVersion, boolean getOnlyDeltas) throws Exception {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> ServiceDBStore.getServicePolicies(" + serviceName + ")");
+ LOG.debug("==> ServiceDBStore.getServicePolicies(" + serviceName + ", " + lastKnownVersion + ")");
}
ServicePolicies ret = null;
@@ -2474,27 +2495,59 @@ public class ServiceDBStore extends AbstractServiceStore {
if (serviceDef == null) {
throw new Exception("service-def does not exist. id=" + serviceDbObj.getType());
}
- List<RangerPolicy> policies = null;
- ServicePolicies.TagPolicies tagPolicies = null;
+ String serviceType = serviceDef.getName();
- String auditMode = getAuditMode(serviceDef.getName(), serviceName);
+ String auditMode = getAuditMode(serviceType, serviceName);
if (serviceDbObj.getIsenabled()) {
+
+ XXService tagServiceDbObj = null;
+ RangerServiceDef tagServiceDef = null;
+ XXServiceVersionInfo tagServiceVersionInfoDbObj= null;
+
if (serviceDbObj.getTagService() != null) {
- XXService tagServiceDbObj = daoMgr.getXXService().getById(serviceDbObj.getTagService());
+ tagServiceDbObj = daoMgr.getXXService().getById(serviceDbObj.getTagService());
+ if (tagServiceDbObj != null && !tagServiceDbObj.getIsenabled()) {
+ tagServiceDbObj = null;
+ }
+ }
- if (tagServiceDbObj != null && tagServiceDbObj.getIsenabled()) {
- RangerServiceDef tagServiceDef = getServiceDef(tagServiceDbObj.getType());
+ if (tagServiceDbObj != null) {
+ tagServiceDef = getServiceDef(tagServiceDbObj.getType());
- if (tagServiceDef == null) {
- throw new Exception("service-def does not exist. id=" + tagServiceDbObj.getType());
- }
+ if (tagServiceDef == null) {
+ throw new Exception("service-def does not exist. id=" + tagServiceDbObj.getType());
+ }
- XXServiceVersionInfo tagServiceVersionInfoDbObj = daoMgr.getXXServiceVersionInfo().findByServiceId(serviceDbObj.getTagService());
+ tagServiceVersionInfoDbObj = daoMgr.getXXServiceVersionInfo().findByServiceId(serviceDbObj.getTagService());
+
+ if (tagServiceVersionInfoDbObj == null) {
+ LOG.warn("serviceVersionInfo does not exist. name=" + tagServiceDbObj.getName());
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Support for incremental policy updates enabled using \"ranger.admin.supports.policy.deltas\" configuation parameter :[" + SUPPORTS_POLICY_DELTAS +"]");
+ }
+
+ if (SUPPORTS_POLICY_DELTAS) {
+ ret = getServicePoliciesWithDeltas(serviceDef, serviceDbObj, tagServiceDef, tagServiceDbObj, lastKnownVersion);
+ }
+
+ if (ret != null) {
+ ret.setPolicyVersion(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyVersion());
+ ret.setPolicyUpdateTime(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyUpdateTime());
+ ret.setAuditMode(auditMode);
+ if (ret.getTagPolicies() != null) {
+ ret.getTagPolicies().setPolicyVersion(tagServiceVersionInfoDbObj == null ? null : tagServiceVersionInfoDbObj.getPolicyVersion());
+ ret.getTagPolicies().setPolicyUpdateTime(tagServiceVersionInfoDbObj == null ? null : tagServiceVersionInfoDbObj.getPolicyUpdateTime());
+ ret.getTagPolicies().setAuditMode(auditMode);
+ }
+ } else if (!getOnlyDeltas) {
+ ServicePolicies.TagPolicies tagPolicies = null;
+
+ if (tagServiceDbObj != null) {
- if (tagServiceVersionInfoDbObj == null) {
- LOG.warn("serviceVersionInfo does not exist. name=" + tagServiceDbObj.getName());
- }
tagPolicies = new ServicePolicies.TagPolicies();
tagPolicies.setServiceId(tagServiceDbObj.getId());
@@ -2505,29 +2558,235 @@ public class ServiceDBStore extends AbstractServiceStore {
tagPolicies.setServiceDef(tagServiceDef);
tagPolicies.setAuditMode(auditMode);
}
+ List<RangerPolicy> policies = getServicePoliciesFromDb(serviceDbObj);
+
+ ret = new ServicePolicies();
+
+ ret.setServiceId(serviceDbObj.getId());
+ ret.setServiceName(serviceDbObj.getName());
+ ret.setPolicyVersion(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyVersion());
+ ret.setPolicyUpdateTime(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyUpdateTime());
+ ret.setPolicies(policies);
+ ret.setServiceDef(serviceDef);
+ ret.setAuditMode(auditMode);
+ ret.setTagPolicies(tagPolicies);
}
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceDBStore.getServicePolicies(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()) + ", delta-count=" + ((ret == null || ret.getPolicyDeltas() == null) ? 0 : ret.getPolicyDeltas().size()));
+ }
- policies = getServicePoliciesFromDb(serviceDbObj);
+ return ret;
+ }
- } else {
- policies = new ArrayList<RangerPolicy>();
+ private static class RangerPolicyDeltaComparator implements Comparator<RangerPolicyDelta>, java.io.Serializable {
+ @Override
+ public int compare(RangerPolicyDelta me, RangerPolicyDelta other) {
+ return Long.compare(me.getId(), other.getId());
+ }
+ }
+
+ private static final Comparator<RangerPolicyDelta> POLICY_DELTA_ID_COMPARATOR = new RangerPolicyDeltaComparator();
+
+ private static List<RangerPolicyDelta> compressDeltas(List<RangerPolicyDelta> deltas) {
+ List<RangerPolicyDelta> ret = new ArrayList<>();
+
+ final Map<Long, List<RangerPolicyDelta>> policyDeltaMap = new HashMap<>();
+
+ for (RangerPolicyDelta delta : deltas) {
+ Long policyId = delta.getPolicyId();
+ List<RangerPolicyDelta> oldPolicyDeltas = policyDeltaMap.get(policyId);
+
+ if (oldPolicyDeltas == null) {
+ oldPolicyDeltas = new ArrayList<>();
+ policyDeltaMap.put(policyId, oldPolicyDeltas);
+ }
+ oldPolicyDeltas.add(delta);
+ }
+
+ for (Map.Entry<Long, List<RangerPolicyDelta>> entry : policyDeltaMap.entrySet()) {
+ List<RangerPolicyDelta> policyDeltas = entry.getValue();
+
+ if (policyDeltas.size() == 1) {
+ ret.addAll(policyDeltas);
+ } else { // Will always be greater than 1
+ List<RangerPolicyDelta> policyDeltasForPolicy = new ArrayList<>();
+ RangerPolicyDelta first = policyDeltas.get(0);
+
+ policyDeltasForPolicy.add(first);
+
+ int index = 1;
+
+ switch (first.getChangeType()) {
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE:
+ while (index < policyDeltas.size()) {
+ RangerPolicyDelta policyDelta = policyDeltas.get(index);
+ switch (policyDelta.getChangeType()) {
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE:
+ LOG.error("Multiple policy creates!! [" + policyDelta + "]");
+ policyDeltasForPolicy = null;
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE:
+ for (int i = index + 1; i < policyDeltas.size(); i++) {
+ RangerPolicyDelta next = policyDeltas.get(i);
+ if (next.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE) {
+ index = i;
+ } else {
+ break;
+ }
+ }
+ policyDeltasForPolicy.add(policyDeltas.get(index));
+ index++;
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE:
+ if (policyDeltas.size() == index + 1) {
+ // Last one
+ policyDeltasForPolicy.clear();
+ index++;
+ } else {
+ LOG.error("CHANGE_TYPE_POLICY_DELETE should be the last policyDelta, found:[" + policyDeltas.get(index+1) +"]");
+ policyDeltasForPolicy = null;
+ }
+ break;
+ default:
+ break;
+ }
+ if (policyDeltasForPolicy == null) {
+ break;
+ }
+ }
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE:
+ while (index < policyDeltas.size()) {
+ RangerPolicyDelta policyDelta = policyDeltas.get(index);
+
+ switch (policyDelta.getChangeType()) {
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE:
+ LOG.error("Should not get here! policy is created after it is updated!! policy-delta:[" + policyDelta + "]");
+ policyDeltasForPolicy = null;
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE:
+ for (int i = index + 1; i < policyDeltas.size(); i++) {
+ RangerPolicyDelta next = policyDeltas.get(i);
+ if (next.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE) {
+ index = i;
+ } else {
+ break;
+ }
+ }
+ policyDeltasForPolicy.clear();
+ policyDeltasForPolicy.add(policyDeltas.get(index));
+ index++;
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE:
+ if (policyDeltas.size() == index + 1) {
+ // Last one
+ policyDeltasForPolicy.clear();
+ policyDeltasForPolicy.add(policyDeltas.get(index));
+ index++;
+ } else {
+ LOG.error("CHANGE_TYPE_POLICY_DELETE should be the last policyDelta, found:[" + policyDeltas.get(index+1) +"]");
+ policyDeltasForPolicy = null;
+ }
+ break;
+ default:
+ break;
+ }
+ if (policyDeltasForPolicy == null) {
+ break;
+ }
+ }
+ break;
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE:
+ LOG.error("CHANGE_TYPE_POLICY_DELETE should be the last policyDelta, found:[" + policyDeltas.get(index) +"]");
+ policyDeltasForPolicy = null;
+ break;
+ default:
+ LOG.error("Should not get here for valid policy-delta:[" + first + "]");
+ break;
+ }
+ if (policyDeltasForPolicy != null) {
+ ret.addAll(policyDeltasForPolicy);
+ } else {
+ ret = null;
+ break;
+ }
+ }
}
- ret = new ServicePolicies();
+ if (ret != null) {
+ ret.sort(POLICY_DELTA_ID_COMPARATOR);
+ }
- ret.setServiceId(serviceDbObj.getId());
- ret.setServiceName(serviceDbObj.getName());
- ret.setPolicyVersion(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyVersion());
- ret.setPolicyUpdateTime(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyUpdateTime());
- ret.setPolicies(policies);
- ret.setServiceDef(serviceDef);
- ret.setAuditMode(auditMode);
- ret.setTagPolicies(tagPolicies);
+ return ret;
+ }
+
+ ServicePolicies getServicePoliciesWithDeltas(RangerServiceDef serviceDef, XXService service, RangerServiceDef tagServiceDef, XXService tagService, Long lastKnownVersion) {
+ ServicePolicies ret = null;
+
+ // if lastKnownVersion != -1L : try and get deltas. Get delta for serviceName first. Find id of the delta
+ // returned first in the list. and then find all ids greater than that for corresponding tag service.
if (LOG.isDebugEnabled()) {
- LOG.debug("<== ServiceDBStore.getServicePolicies(" + serviceName + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
+ LOG.debug("==> ServiceDBStore.getServicePoliciesWithDeltas(serviceType=" + serviceDef.getName() + ", serviceId=" + service.getId()
+ +", tagServiceId=" + (tagService != null ? tagService.getId() : null) + ", lastKnownVersion=" + lastKnownVersion + ")");
}
+ if (lastKnownVersion != -1L) {
+
+ List<RangerPolicyDelta> resourcePolicyDeltas;
+ List<RangerPolicyDelta> tagPolicyDeltas = null;
+
+ String componentServiceType = serviceDef.getName();
+
+ boolean isValid;
+
+ resourcePolicyDeltas = daoMgr.getXXPolicyChangeLog().findLaterThan(policyService, lastKnownVersion, service.getId());
+ if (CollectionUtils.isNotEmpty(resourcePolicyDeltas)) {
+ isValid = RangerPolicyDeltaUtil.isValidDeltas(resourcePolicyDeltas, componentServiceType);
+
+ if (isValid && tagService != null) {
+ Long id = resourcePolicyDeltas.get(0).getId();
+ tagPolicyDeltas = daoMgr.getXXPolicyChangeLog().findGreaterThan(policyService, id, tagService.getId());
+
+
+ if (CollectionUtils.isNotEmpty(tagPolicyDeltas)) {
+ String tagServiceType = tagServiceDef.getName();
+
+ isValid = RangerPolicyDeltaUtil.isValidDeltas(tagPolicyDeltas, tagServiceType);
+ }
+ }
+ if (isValid) {
+ if (CollectionUtils.isNotEmpty(tagPolicyDeltas)) {
+ resourcePolicyDeltas.addAll(tagPolicyDeltas);
+ }
+ List<RangerPolicyDelta> compressedDeltas = compressDeltas(resourcePolicyDeltas);
+ if (compressedDeltas != null) {
+ ret = new ServicePolicies();
+ ret.setServiceId(service.getId());
+ ret.setServiceName(service.getName());
+ ret.setServiceDef(serviceDef);
+ ret.setPolicies(null);
+ ret.setPolicyDeltas(compressedDeltas);
+
+ if (CollectionUtils.isNotEmpty(tagPolicyDeltas)) {
+ ServicePolicies.TagPolicies tagPolicies = new ServicePolicies.TagPolicies();
+ tagPolicies.setServiceDef(tagServiceDef);
+ tagPolicies.setServiceId(tagService.getId());
+ tagPolicies.setServiceName(tagService.getName());
+ tagPolicies.setPolicies(null);
+ ret.setTagPolicies(tagPolicies);
+ }
+ }
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceDBStore.getServicePoliciesWithDeltas(serviceType=" + serviceDef.getName() + ", serviceId=" + service.getId()
+ +", tagServiceId=" + (tagService != null ? tagService.getId() : null) + ", lastKnownVersion=" + lastKnownVersion + ") : deltasSize=" + (ret != null && CollectionUtils.isNotEmpty(ret.getPolicyDeltas()) ? ret.getPolicyDeltas().size() : 0));
+ }
return ret;
}
@@ -2788,13 +3047,13 @@ public class ServiceDBStore extends AbstractServiceStore {
return validConfigs;
}
- private void handlePolicyUpdate(RangerService service) throws Exception {
- updatePolicyVersion(service);
+ private void handlePolicyUpdate(RangerService service, Integer policyDeltaType, RangerPolicy policy) throws Exception {
+ updatePolicyVersion(service, policyDeltaType, policy);
}
public enum VERSION_TYPE { POLICY_VERSION, TAG_VERSION, POLICY_AND_TAG_VERSION }
- private void updatePolicyVersion(RangerService service) throws Exception {
+ private void updatePolicyVersion(RangerService service, Integer policyDeltaType, RangerPolicy policy) throws Exception {
if(service == null || service.getId() == null) {
return;
}
@@ -2810,10 +3069,6 @@ public class ServiceDBStore extends AbstractServiceStore {
final RangerDaoManager daoManager = daoMgr;
final Long serviceId = serviceDbObj.getId();
- final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION;
-
- Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType);
- transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater);
// if this is a tag service, update all services that refer to this tag service
// so that next policy-download from plugins will get updated tag policies
@@ -2826,32 +3081,41 @@ public class ServiceDBStore extends AbstractServiceStore {
final Long referringServiceId = referringService.getId();
final VERSION_TYPE tagServiceversionType = VERSION_TYPE.POLICY_VERSION;
- Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceversionType);
+ Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceversionType, policy != null ? policy.getZoneName() : null, policyDeltaType, policy);
transactionSynchronizationAdapter.executeOnTransactionCommit(tagServiceVersionUpdater);
}
}
}
+ final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION;
+
+ Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType, policy != null ? policy.getZoneName() : null, policyDeltaType, policy);
+ transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater);
}
- public static void persistVersionChange(RangerDaoManager daoMgr, Long id, VERSION_TYPE versionType) {
+ public static void persistVersionChange(RangerDaoManager daoMgr, Long id, VERSION_TYPE versionType, String zoneName, Integer policyDeltaType, RangerPolicy policy) {
XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo();
XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(id);
+ XXService service = daoMgr.getXXService().getById(id);
+
+ Long nextPolicyVersion = 1L;
+ Date now = new Date();
if(serviceVersionInfoDbObj != null) {
if (versionType == VERSION_TYPE.POLICY_VERSION || versionType == VERSION_TYPE.POLICY_AND_TAG_VERSION) {
- serviceVersionInfoDbObj.setPolicyVersion(getNextVersion(serviceVersionInfoDbObj.getPolicyVersion()));
- serviceVersionInfoDbObj.setPolicyUpdateTime(new Date());
+ nextPolicyVersion = getNextVersion(serviceVersionInfoDbObj.getPolicyVersion());
+
+ serviceVersionInfoDbObj.setPolicyVersion(nextPolicyVersion);
+ serviceVersionInfoDbObj.setPolicyUpdateTime(now);
}
if (versionType == VERSION_TYPE.TAG_VERSION || versionType == VERSION_TYPE.POLICY_AND_TAG_VERSION) {
serviceVersionInfoDbObj.setTagVersion(getNextVersion(serviceVersionInfoDbObj.getTagVersion()));
- serviceVersionInfoDbObj.setTagUpdateTime(new Date());
+ serviceVersionInfoDbObj.setTagUpdateTime(now);
}
serviceVersionInfoDao.update(serviceVersionInfoDbObj);
} else {
- XXService service = daoMgr.getXXService().getById(id);
if (service != null) {
serviceVersionInfoDbObj = new XXServiceVersionInfo();
serviceVersionInfoDbObj.setServiceId(service.getId());
@@ -2863,6 +3127,25 @@ public class ServiceDBStore extends AbstractServiceStore {
serviceVersionInfoDao.create(serviceVersionInfoDbObj);
}
}
+
+ if (service != null && versionType != VERSION_TYPE.TAG_VERSION) {
+ // Build and save PolicyChangeLog
+ XXPolicyChangeLog policyChangeLog = new XXPolicyChangeLog();
+
+ policyChangeLog.setCreateTime(now);
+ policyChangeLog.setServiceId(service.getId());
+ policyChangeLog.setChangeType(policyDeltaType);
+ policyChangeLog.setPolicyVersion(nextPolicyVersion);
+ policyChangeLog.setZoneName(zoneName);
+
+ if (policy != null) {
+ policyChangeLog.setServiceType(policy.getServiceType());
+ policyChangeLog.setPolicyType(policy.getPolicyType());
+ policyChangeLog.setPolicyId(policy.getId());
+ }
+
+ daoMgr.getXXPolicyChangeLog().create(policyChangeLog);
+ }
}
private void createNewLabelsForPolicy(XXPolicy xPolicy, List<String> policyLabels) throws Exception {
@@ -2978,12 +3261,6 @@ public class ServiceDBStore extends AbstractServiceStore {
if(CollectionUtils.isNotEmpty(services)) {
for(XXService service : services) {
- final Long serviceId = service.getId();
- final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION;
-
- Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType);
- transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater);
-
if(isTagServiceDef) {
List<XXService> referringServices = serviceDao.findByTagServiceId(service.getId());
@@ -2993,11 +3270,17 @@ public class ServiceDBStore extends AbstractServiceStore {
final Long referringServiceId = referringService.getId();
final VERSION_TYPE tagServiceVersionType = VERSION_TYPE.POLICY_VERSION;
- Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceVersionType);
+ Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceVersionType, RangerPolicyDelta.CHANGE_TYPE_SERVICE_DEF_CHANGE);
transactionSynchronizationAdapter.executeOnTransactionCommit(tagServiceVersionUpdater);
}
}
}
+
+ final Long serviceId = service.getId();
+ final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION;
+
+ Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType, RangerPolicyDelta.CHANGE_TYPE_SERVICE_DEF_CHANGE);
+ transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater);
}
}
}
@@ -4033,6 +4316,28 @@ public class ServiceDBStore extends AbstractServiceStore {
xUserService.createXUserWithOutLogin(genericUser);
}
+ public void resetPolicyUpdateLog(int retentionInDays, boolean reloadServicePoliciesCache) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> resetPolicyUpdateLog(" + retentionInDays + ", " + reloadServicePoliciesCache + ")");
+ }
+
+ daoMgr.getXXPolicyChangeLog().deleteOlderThan(retentionInDays);
+
+ if (reloadServicePoliciesCache) {
+ List<Long> allServiceIds = daoMgr.getXXService().getAllServiceIds();
+ if (CollectionUtils.isNotEmpty(allServiceIds)) {
+ for (Long serviceId : allServiceIds) {
+ persistVersionChange(daoMgr, serviceId, VERSION_TYPE.POLICY_VERSION, null, RangerPolicyDelta.CHANGE_TYPE_RANGER_ADMIN_START, null);
+ }
+ }
+
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== resetPolicyUpdateLog(" + retentionInDays + ", " + reloadServicePoliciesCache + ")");
+
+ }
+ }
+
public List<String> getPolicyLabels(SearchFilter searchFilter) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ServiceDBStore.getPolicyLabels()");
@@ -4455,7 +4760,7 @@ public class ServiceDBStore extends AbstractServiceStore {
String svcAdminUsers = cfgSvcAdminUsers != null ? cfgSvcAdminUsers.getConfigvalue() : null;
if (svcAdminUsers != null) {
for (String svcAdminUser : svcAdminUsers.split(",")) {
- if (userName.equals(svcAdminUser.trim())) {
+ if (userName.equals(svcAdminUser)) {
ret=true;
break;
}
@@ -4468,15 +4773,29 @@ public class ServiceDBStore extends AbstractServiceStore {
final Long serviceId;
final RangerDaoManager daoManager;
final VERSION_TYPE versionType;
+ final String zoneName;
+ final Integer policyDeltaChange;
+ final RangerPolicy policy;
+
+ public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType) {
+ this(daoManager, serviceId, versionType, null, null, null);
+ }
+
+ public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType, Integer policyDeltaType) {
+ this(daoManager, serviceId, versionType, null, policyDeltaType, null);
+ }
- public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType ) {
+ public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType, String zoneName, Integer policyDeltaType, RangerPolicy policy ) {
this.serviceId = serviceId;
this.daoManager = daoManager;
this.versionType = versionType;
+ this.policyDeltaChange = policyDeltaType;
+ this.zoneName = zoneName;
+ this.policy = policy;
}
@Override
public void run() {
- ServiceDBStore.persistVersionChange(this.daoManager, this.serviceId, this.versionType);
+ ServiceDBStore.persistVersionChange(this.daoManager, this.serviceId, this.versionType, this.zoneName, policyDeltaChange, policy);
}
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
index 0724952..86b3c00 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
@@ -27,8 +27,10 @@ import org.apache.ranger.plugin.store.ServiceStore;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.util.RangerPolicyDeltaUtil;
import org.apache.ranger.plugin.util.ServicePolicies;
+import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -43,10 +45,9 @@ public class RangerServicePoliciesCache {
private static final int MAX_WAIT_TIME_FOR_UPDATE = 10;
public static volatile RangerServicePoliciesCache sInstance = null;
- private final boolean useServicePoliciesCache;
private final int waitTimeInSeconds;
- private final Map<String, ServicePoliciesWrapper> servicePoliciesMap = new HashMap<String, ServicePoliciesWrapper>();
+ private final Map<String, ServicePoliciesWrapper> servicePoliciesMap = new HashMap<>();
public static RangerServicePoliciesCache getInstance() {
if (sInstance == null) {
@@ -60,117 +61,105 @@ public class RangerServicePoliciesCache {
}
private RangerServicePoliciesCache() {
- useServicePoliciesCache = RangerConfiguration.getInstance().getBoolean("ranger.admin.policy.download.usecache", true);
waitTimeInSeconds = RangerConfiguration.getInstance().getInt("ranger.admin.policy.download.cache.max.waittime.for.update", MAX_WAIT_TIME_FOR_UPDATE);
}
public void dump() {
+ if (LOG.isDebugEnabled()) {
- if (useServicePoliciesCache) {
- Set<String> serviceNames = null;
+ final Set<String> serviceNames;
synchronized (this) {
serviceNames = servicePoliciesMap.keySet();
}
if (CollectionUtils.isNotEmpty(serviceNames)) {
- ServicePoliciesWrapper cachedServicePoliciesWrapper = null;
for (String serviceName : serviceNames) {
+ final ServicePoliciesWrapper cachedServicePoliciesWrapper;
+
synchronized (this) {
cachedServicePoliciesWrapper = servicePoliciesMap.get(serviceName);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("serviceName:" + serviceName + ", Cached-MetaData:" + cachedServicePoliciesWrapper);
- }
+ LOG.debug("serviceName:" + serviceName + ", Cached-MetaData:" + cachedServicePoliciesWrapper);
+
}
}
}
}
- public ServicePolicies getServicePolicies(String serviceName, Long serviceId, ServiceStore serviceStore) throws Exception {
+ public ServicePolicies getServicePolicies(String serviceName, Long serviceId, Long lastKnownVersion, boolean needsBackwardCompatibility, ServiceStore serviceStore) throws Exception {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> RangerServicePoliciesCache.getServicePolicies(" + serviceName + ", " + serviceId + ")");
+ LOG.debug("==> RangerServicePoliciesCache.getServicePolicies(" + serviceName + ", " + serviceId + ", " + lastKnownVersion + ", " + needsBackwardCompatibility + ")");
}
ServicePolicies ret = null;
if (StringUtils.isNotBlank(serviceName) && serviceId != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("useServicePoliciesCache=" + useServicePoliciesCache);
- }
-
- ServicePolicies servicePolicies = null;
+ ServicePoliciesWrapper servicePoliciesWrapper;
- if (!useServicePoliciesCache) {
- if (serviceStore != null) {
- try {
- servicePolicies = serviceStore.getServicePolicies(serviceName);
- } catch (Exception exception) {
- LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies from service-store", exception);
- }
- } else {
- LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies as service-store is null!");
- }
- } else {
- ServicePoliciesWrapper servicePoliciesWrapper = null;
-
- synchronized (this) {
- servicePoliciesWrapper = servicePoliciesMap.get(serviceName);
-
- if (servicePoliciesWrapper != null) {
- if (!serviceId.equals(servicePoliciesWrapper.getServiceId())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Service [" + serviceName + "] changed service-id from " + servicePoliciesWrapper.getServiceId()
- + " to " + serviceId);
- LOG.debug("Recreating servicePoliciesWrapper for serviceName [" + serviceName + "]");
- }
- servicePoliciesMap.remove(serviceName);
- servicePoliciesWrapper = null;
+ synchronized (this) {
+ servicePoliciesWrapper = servicePoliciesMap.get(serviceName);
+
+ if (servicePoliciesWrapper != null) {
+ if (!serviceId.equals(servicePoliciesWrapper.getServiceId())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Service [" + serviceName + "] changed service-id from " + servicePoliciesWrapper.getServiceId()
+ + " to " + serviceId);
+ LOG.debug("Recreating servicePoliciesWrapper for serviceName [" + serviceName + "]");
}
- }
-
- if (servicePoliciesWrapper == null) {
- servicePoliciesWrapper = new ServicePoliciesWrapper(serviceId);
- servicePoliciesMap.put(serviceName, servicePoliciesWrapper);
+ servicePoliciesMap.remove(serviceName);
+ servicePoliciesWrapper = null;
}
}
- if (serviceStore != null) {
- boolean refreshed = servicePoliciesWrapper.getLatestOrCached(serviceName, serviceStore);
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("getLatestOrCached returned " + refreshed);
- }
- } else {
- LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies as service-store is null!");
+ if (servicePoliciesWrapper == null) {
+ servicePoliciesWrapper = new ServicePoliciesWrapper(serviceId);
+ servicePoliciesMap.put(serviceName, servicePoliciesWrapper);
}
-
- servicePolicies = servicePoliciesWrapper.getServicePolicies();
}
- ret = servicePolicies;
+ if (serviceStore != null) {
+ ret = servicePoliciesWrapper.getLatestOrCached(serviceName, serviceStore, lastKnownVersion, needsBackwardCompatibility);
+ } else {
+ LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies as service-store is null! Returning cached servicePolicies!");
+ ret = servicePoliciesWrapper.getServicePolicies();
+ }
} else {
LOG.error("getServicePolicies() failed to get policies as serviceName is null or blank and/or serviceId is null!");
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== RangerServicePoliciesCache.getServicePolicies(" + serviceName + ", " + serviceId + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
+ LOG.debug("<== RangerServicePoliciesCache.getServicePolicies(" + serviceName + ", " + serviceId + ", " + lastKnownVersion + ", " + needsBackwardCompatibility + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
}
return ret;
}
private class ServicePoliciesWrapper {
- final Long serviceId;
- ServicePolicies servicePolicies;
- Date updateTime = null;
- long longestDbLoadTimeInMs = -1;
+ final Long serviceId;
+ ServicePolicies servicePolicies;
+ Date updateTime = null;
+ long longestDbLoadTimeInMs = -1;
+ final ReentrantLock lock = new ReentrantLock();
+
+ ServicePolicyDeltasCache deltaCache;
- ReentrantLock lock = new ReentrantLock();
+ class ServicePolicyDeltasCache {
+ final long fromVersion;
+ final ServicePolicies servicePolicyDeltas;
+
+ ServicePolicyDeltasCache(final long fromVersion, ServicePolicies servicePolicyDeltas) {
+ this.fromVersion = fromVersion;
+ this.servicePolicyDeltas = servicePolicyDeltas;
+ }
+ ServicePolicies getServicePolicyDeltasFromVersion(long fromVersion) {
+ return this.fromVersion == fromVersion ? this.servicePolicyDeltas : null;
+ }
+ }
ServicePoliciesWrapper(Long serviceId) {
this.serviceId = serviceId;
@@ -187,51 +176,103 @@ public class RangerServicePoliciesCache {
return updateTime;
}
- long getLongestDbLoadTimeInMs() {
- return longestDbLoadTimeInMs;
- }
-
- boolean getLatestOrCached(String serviceName, ServiceStore serviceStore) throws Exception {
- boolean ret = false;
+ ServicePolicies getLatestOrCached(String serviceName, ServiceStore serviceStore, Long lastKnownVersion, boolean needsBackwardCompatibility) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerServicePoliciesCache.getLatestOrCached(lastKnownVersion=" + lastKnownVersion + ", " + needsBackwardCompatibility + ")");
+ }
+ ServicePolicies ret = null;
+ boolean lockResult = false;
try {
- ret = lock.tryLock(waitTimeInSeconds, TimeUnit.SECONDS);
- if (ret) {
- getLatest(serviceName, serviceStore);
+ final boolean isCacheReloadedByDQEvent;
+
+ lockResult = lock.tryLock(waitTimeInSeconds, TimeUnit.SECONDS);
+
+ if (lockResult) {
+ isCacheReloadedByDQEvent = getLatest(serviceName, serviceStore, lastKnownVersion);
+
+ if (isCacheReloadedByDQEvent) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ServicePolicies cache was completely loaded from database because of a disqualifying event - such as service-definition change!");
+ }
+ }
+
+ if (needsBackwardCompatibility || isCacheReloadedByDQEvent
+ || lastKnownVersion == -1L || lastKnownVersion.equals(servicePolicies.getPolicyVersion())) {
+ // Looking for all policies, or Some disqualifying change encountered
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("All policies were requested, returning cached ServicePolicies");
+ }
+ ret = this.servicePolicies;
+ } else {
+ boolean isDeltaCacheReinitialized = false;
+ ServicePolicies servicePoliciesForDeltas = this.deltaCache != null ? this.deltaCache.getServicePolicyDeltasFromVersion(lastKnownVersion) : null;
+
+ if (servicePoliciesForDeltas == null) {
+ servicePoliciesForDeltas = serviceStore.getOnlyServicePolicyDeltas(serviceName, lastKnownVersion);
+ isDeltaCacheReinitialized = true;
+ }
+ if (servicePoliciesForDeltas != null && servicePoliciesForDeltas.getPolicyDeltas() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deltas were requested. Returning deltas from lastKnownVersion:[" + lastKnownVersion + "]");
+ }
+ if (isDeltaCacheReinitialized) {
+ this.deltaCache = new ServicePolicyDeltasCache(lastKnownVersion, servicePoliciesForDeltas);
+ }
+ ret = servicePoliciesForDeltas;
+ } else {
+ LOG.warn("Deltas were requested, but could not get them!! lastKnownVersion:[" + lastKnownVersion + "]; Returning cached ServicePolicies:[" + (servicePolicies != null ? servicePolicies.getPolicyVersion() : -1L) + "]");
+
+ this.deltaCache = null;
+ ret = this.servicePolicies;
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not get lock in [" + waitTimeInSeconds + "] seconds, returning cached ServicePolicies");
+ }
+ ret = this.servicePolicies;
}
} catch (InterruptedException exception) {
LOG.error("getLatestOrCached:lock got interrupted..", exception);
} finally {
- if (ret) {
+ if (lockResult) {
lock.unlock();
}
}
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("RangerServicePoliciesCache.getLatestOrCached - Returns ServicePolicies:[" + ret +"]");
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerServicePoliciesCache.getLatestOrCached(lastKnownVersion=" + lastKnownVersion + ", " + needsBackwardCompatibility + ") : " + ret);
+ }
return ret;
}
- void getLatest(String serviceName, ServiceStore serviceStore) throws Exception {
-
+ boolean getLatest(String serviceName, ServiceStore serviceStore, Long lastKnownVersion) throws Exception {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> ServicePoliciesWrapper.getLatest(" + serviceName + ")");
+ LOG.debug("==> ServicePoliciesWrapper.getLatest(serviceName=" + serviceName + ", lastKnownVersion=" + lastKnownVersion + ")");
}
+ final Long servicePolicyVersionInDb = serviceStore.getServicePolicyVersion(serviceName);
+ final Long cachedServicePoliciesVersion = servicePolicies != null ? servicePolicies.getPolicyVersion() : -1L;
+
if (LOG.isDebugEnabled()) {
- LOG.debug("Found ServicePolicies in-cache : " + (servicePolicies != null));
+ LOG.debug("ServicePolicies version in cache[" + cachedServicePoliciesVersion + "], ServicePolicies version in database[" + servicePolicyVersionInDb + "]");
}
- Long servicePolicyVersionInDb = serviceStore.getServicePolicyVersion(serviceName);
+ boolean isCacheReloadedByDQEvent = false;
+
+ if (servicePolicyVersionInDb == null || !servicePolicyVersionInDb.equals(cachedServicePoliciesVersion)) {
- if (servicePolicies == null || servicePolicyVersionInDb == null || !servicePolicyVersionInDb.equals(servicePolicies.getPolicyVersion())) {
if (LOG.isDebugEnabled()) {
- LOG.debug("loading servicePolicies from db ... cachedServicePoliciesVersion=" + (servicePolicies != null ? servicePolicies.getPolicyVersion() : null) + ", servicePolicyVersionInDb=" + servicePolicyVersionInDb);
+ LOG.debug("loading servicePolicies from database");
}
- long startTimeMs = System.currentTimeMillis();
-
- ServicePolicies servicePoliciesFromDb = serviceStore.getServicePolicies(serviceName);
-
- long dbLoadTime = System.currentTimeMillis() - startTimeMs;
+ final long startTimeMs = System.currentTimeMillis();
+ final ServicePolicies servicePoliciesFromDb = serviceStore.getServicePolicyDeltasOrPolicies(serviceName, cachedServicePoliciesVersion);
+ final long dbLoadTime = System.currentTimeMillis() - startTimeMs;
if (dbLoadTime > longestDbLoadTimeInMs) {
longestDbLoadTimeInMs = dbLoadTime;
@@ -239,17 +280,71 @@ public class RangerServicePoliciesCache {
updateTime = new Date();
if (servicePoliciesFromDb != null) {
- if (servicePoliciesFromDb.getPolicyVersion() == null) {
- servicePoliciesFromDb.setPolicyVersion(0L);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Successfully loaded ServicePolicies from database: ServicePolicies:[" + servicePoliciesFromDb + "]");
+ }
+ if (servicePolicies == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initializing ServicePolicies cache for the first time");
+ }
+ servicePolicies = servicePoliciesFromDb;
+ pruneUnusedAttributes();
+ } else if (servicePoliciesFromDb.getPolicyDeltas() == null) {
+ // service-policies are loaded because service/service-def changed
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Complete set of policies are loaded from database, because of some disqualifying event");
+ }
+ servicePolicies = servicePoliciesFromDb;
+ pruneUnusedAttributes();
+ isCacheReloadedByDQEvent = true;
+ } else { // Previously cached service policies are still valid - no service/service-def change
+ // Rebuild policies cache from original policies and deltas
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Retrieved policy-deltas from database. These will be applied on top of ServicePolicy version:[" + cachedServicePoliciesVersion +"], policy-deltas:[" + servicePoliciesFromDb.getPolicyDeltas() + "]");
+ }
+ servicePolicies.setPolicyVersion(servicePoliciesFromDb.getPolicyVersion());
+
+ final List<RangerPolicy> policies = servicePolicies.getPolicies() == null ? new ArrayList<>() : servicePolicies.getPolicies();
+ final List<RangerPolicy> newPolicies = RangerPolicyDeltaUtil.applyDeltas(policies, servicePoliciesFromDb.getPolicyDeltas(), servicePolicies.getServiceDef().getName());
+ servicePolicies.setPolicies(newPolicies);
+
+ // Rebuild tag-policies from original tag-policies and deltas
+ if (servicePoliciesFromDb.getTagPolicies() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("This service has associated tag service. Will compute tagPolicies from corresponding policy-deltas");
+ }
+
+ final List<RangerPolicy> tagPolicies = (servicePolicies.getTagPolicies() == null || CollectionUtils.isEmpty(servicePolicies.getTagPolicies().getPolicies())) ? new ArrayList<>() : servicePolicies.getTagPolicies().getPolicies();
+ final List<RangerPolicy> newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPolicies, servicePoliciesFromDb.getPolicyDeltas(), servicePoliciesFromDb.getTagPolicies().getServiceDef().getName());
+ servicePolicies.getTagPolicies().setPolicies(newTagPolicies);
+
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("This service has no associated tag service");
+ }
+ }
}
- servicePolicies = servicePoliciesFromDb;
- pruneUnusedAttributes();
+ this.deltaCache = null;
+ } else {
+ LOG.error("Could not get policies from database, from-version:[" + cachedServicePoliciesVersion + ")");
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ServicePolicies old-version:[" + cachedServicePoliciesVersion + "], new-version:[" + servicePolicies.getPolicyVersion() + "]");
}
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ServicePolicies Cache already has the latest version, version:[" + servicePolicies.getPolicyVersion() + "]");
+ }
+ }
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Latest Cached ServicePolicies:[" + servicePolicies +"]");
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== ServicePoliciesWrapper.getLatest(" + serviceName + ")");
+ LOG.debug("<== ServicePoliciesWrapper.getLatest(serviceName=" + serviceName + ", lastKnownVersion=" + lastKnownVersion + ") : " + isCacheReloadedByDQEvent);
}
+ return isCacheReloadedByDQEvent;
}
private void pruneUnusedAttributes() {
@@ -285,7 +380,8 @@ public class RangerServicePoliciesCache {
sb.append("updateTime=").append(updateTime)
.append(", longestDbLoadTimeInMs=").append(longestDbLoadTimeInMs)
.append(", Service-Version:").append(servicePolicies != null ? servicePolicies.getPolicyVersion() : "null")
- .append(", Number-Of-Policies:").append(servicePolicies != null ? servicePolicies.getPolicies().size() : 0);
+ .append(", Number-Of-Policies:").append(servicePolicies != null && servicePolicies.getPolicies() != null ? servicePolicies.getPolicies().size() : 0)
+ .append(", Number-Of-Policy-Deltas:").append(servicePolicies != null && servicePolicies.getPolicyDeltas() != null ? servicePolicies.getPolicyDeltas().size() : 0);
sb.append("} ");
diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/JPABeanCallbacks.java b/security-admin/src/main/java/org/apache/ranger/common/db/JPABeanCallbacks.java
index 70ad44b..226c060 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/db/JPABeanCallbacks.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/db/JPABeanCallbacks.java
@@ -48,12 +48,14 @@ public class JPABeanCallbacks {
entity.setAddedByUserId(userSession.getUserId());
entity.setUpdatedByUserId(userSession
.getUserId());
+ } else {
+ if (logger.isDebugEnabled()) {
+ logger.debug("User session not found for this request. Identity of originator of this change cannot be recorded");
+ }
}
} else {
if (logger.isDebugEnabled()) {
- logger.debug(
- "Security context not found for this request. obj="
- + o, new Throwable());
+ logger.debug("Security context not found for this request. Identity of originator of this change cannot be recorded");
}
}
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
index be14922..3599cf3 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
@@ -300,5 +300,8 @@ public abstract class RangerDaoManagerBase {
public XXSecurityZoneRefGroupDao getXXSecurityZoneRefGroup() { return new XXSecurityZoneRefGroupDao(this); }
public XXGlobalStateDao getXXGlobalState() { return new XXGlobalStateDao(this); }
+
+ public XXPolicyChangeLogDao getXXPolicyChangeLog() { return new XXPolicyChangeLogDao(this); }
+
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyChangeLogDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyChangeLogDao.java
new file mode 100644
index 0000000..ef436a0
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyChangeLogDao.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.db;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.common.db.BaseDao;
+import org.apache.ranger.entity.XXPolicy;
+import org.apache.ranger.entity.XXPolicyChangeLog;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
+import org.apache.ranger.service.RangerPolicyService;
+import org.springframework.stereotype.Service;
+
+/**
+ */
+@Service
+public class XXPolicyChangeLogDao extends BaseDao<XXPolicyChangeLog> {
+
+ private static final Log LOG = LogFactory.getLog(XXPolicyChangeLogDao.class);
+
+ /**
+ * Default Constructor
+ */
+ public XXPolicyChangeLogDao(RangerDaoManagerBase daoManager) {
+ super(daoManager);
+ }
+
+ public List<RangerPolicyDelta> findLaterThan(RangerPolicyService policyService, Long version, Long serviceId) {
+ final List<RangerPolicyDelta> ret;
+ if (version != null) {
+ List<Object[]> logs = getEntityManager()
+ .createNamedQuery("XXPolicyChangeLog.findSinceVersion", Object[].class)
+ .setParameter("version", version)
+ .setParameter("serviceId", serviceId)
+ .getResultList();
+ // Ensure that the first record has the same version as the base-version from where the records are fetched
+ if (CollectionUtils.isNotEmpty(logs)) {
+ Object[] firstRecord = logs.get(0);
+ Long versionOfFirstRecord = (Long) firstRecord[2];
+ if (version.equals(versionOfFirstRecord)) {
+ logs.remove(0);
+ ret = convert(policyService, logs);
+ } else {
+ ret = null;
+ }
+ } else {
+ ret = null;
+ }
+ } else {
+ ret = null;
+ }
+ return ret;
+ }
+
+ public List<RangerPolicyDelta> findGreaterThan(RangerPolicyService policyService, Long id, Long serviceId) {
+ final List<RangerPolicyDelta> ret;
+ if (id != null) {
+ List<Object[]> logs = getEntityManager()
+ .createNamedQuery("XXPolicyChangeLog.findGreaterThan", Object[].class)
+ .setParameter("id", id)
+ .setParameter("serviceId", serviceId)
+ .getResultList();
+ ret = convert(policyService, logs);
+ } else {
+ ret = null;
+ }
+ return ret;
+ }
+
+ public void deleteOlderThan(int olderThanInDays) {
+
+ Date since = new Date(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(olderThanInDays));
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleting records from x_policy_change_log that are older than " + olderThanInDays + " days, that is, older than " + since);
+ }
+
+ getEntityManager().createNamedQuery("XXPolicyChangeLog.deleteOlderThan").setParameter("olderThan", since).executeUpdate();
+ }
+
+ private List<RangerPolicyDelta> convert(RangerPolicyService policyService, List<Object[]> queryResult) {
+
+ final List<RangerPolicyDelta> ret;
+
+ if (CollectionUtils.isNotEmpty(queryResult)) {
+
+ ret = new ArrayList<>(queryResult.size());
+
+ for (Object[] log : queryResult) {
+
+ RangerPolicy policy;
+ Long logRecordId = (Long) log[0];
+ Integer policyChangeType = (Integer) log[1];
+ String serviceType = (String) log[3];
+ Long policyId = (Long) log[5];
+
+ if (policyId != null) {
+ XXPolicy xxPolicy = daoManager.getXXPolicy().getById(policyId);
+ if (xxPolicy != null) {
+ try {
+ policy = policyService.read(policyId);
+ } catch (Exception e) {
+ LOG.error("Cannot read policy:[" + policyId + "]. Should not have come here!! Offending log-record-id:[" + logRecordId + "] and returning...", e);
+ ret.clear();
+ ret.add(new RangerPolicyDelta(logRecordId, RangerPolicyDelta.CHANGE_TYPE_LOG_ERROR, null));
+ break;
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Policy:[" + policyId + "] not found - log-record - id:[" + logRecordId + "], PolicyChangeType:[" + policyChangeType + "]");
+ }
+
+ // Create a dummy policy as the policy cannot be found - probably already deleted
+ policy = new RangerPolicy();
+ policy.setId(policyId);
+ policy.setVersion((Long) log[2]);
+ policy.setPolicyType((Integer) log[4]);
+ policy.setZoneName((String) log[6]);
+ }
+ policy.setServiceType(serviceType);
+
+ ret.add(new RangerPolicyDelta(logRecordId, policyChangeType, policy));
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("policyId is null! log-record-id:[" + logRecordId + ", service-type:[" + log[3] + "], policy-change-type:[" + log[1] + "]");
+ }
+ ret.clear();
+ ret.add(new RangerPolicyDelta(logRecordId, policyChangeType, null));
+ break;
+ }
+ }
+ } else {
+ ret = null;
+ }
+ return ret;
+
+ }
+
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceDao.java
index a79ba7c..0791f2f 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceDao.java
@@ -110,4 +110,13 @@ public class XXServiceDao extends BaseDao<XXService> {
updateSequence("X_SERVICE_SEQ", maxId + 1);
}
+
+ public List<Long> getAllServiceIds() {
+ try {
+ return getEntityManager().createNamedQuery("XXService.getAllServiceIds", Long.class)
+ .getResultList();
+ } catch (NoResultException e) {
+ return new ArrayList<>();
+ }
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyChangeLog.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyChangeLog.java
new file mode 100644
index 0000000..df87f70
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPolicyChangeLog.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.entity;
+
+import java.util.Date;
+
+import javax.persistence.Cacheable;
+import javax.persistence.Entity;
+import javax.persistence.Column;
+import javax.persistence.EntityListeners;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.SequenceGenerator;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.ranger.common.AppConstants;
+import org.apache.ranger.common.DateUtil;
+
+@EntityListeners( org.apache.ranger.common.db.JPABeanCallbacks.class)
+@Entity
+@Cacheable
+@XmlRootElement
+@Table(name = "x_policy_change_log")
+public class XXPolicyChangeLog implements java.io.Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Id
+ @SequenceGenerator(name = "X_POLICY_CHANGE_LOG_SEQ", sequenceName = "X_POLICY_CHANGE_LOG_SEQ", allocationSize = 1)
+ @GeneratedValue(strategy = GenerationType.AUTO, generator = "X_POLICY_CHANGE_LOG_SEQ")
+ @Column(name = "id")
+ protected Long id;
+
+ @Temporal(TemporalType.TIMESTAMP)
+ @Column(name="create_time" )
+ protected Date createTime = DateUtil.getUTCDate();
+
+ @Column(name = "service_id")
+ protected Long serviceId;
+
+ @Column(name = "change_type")
+ protected Integer changeType;
+
+ @Column(name = "policy_version")
+ protected Long policyVersion;
+
+ @Column(name = "service_type")
+ protected String serviceType;
+
+ @Column(name = "policy_type")
+ protected Integer policyType;
+
+ @Column(name = "zone_name")
+ protected String zoneName;
+
+ @Column(name = "policy_id")
+ protected Long policyId;
+
+ /**
+ * Default constructor. This will set all the attributes to default value.
+ */
+ public XXPolicyChangeLog( ) {
+ }
+
+ public int getMyClassType( ) {
+ return AppConstants.CLASS_TYPE_NONE;
+ }
+
+ public String getMyDisplayValue() {
+ return null;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public Long getId() {
+ return this.id;
+ }
+
+ public void setCreateTime( Date createTime ) {
+ this.createTime = createTime;
+ }
+
+ public Date getCreateTime( ) {
+ return this.createTime;
+ }
+
+ public void setServiceId(Long serviceId) {
+ this.serviceId = serviceId;
+ }
+
+ public Long getServiceId() {
+ return this.serviceId;
+ }
+
+ public void setPolicyVersion(Long policyVersion) {
+ this.policyVersion = policyVersion;
+ }
+
+ public Long getPolicyVersion() {
+ return this.policyVersion;
+ }
+
+ public void setChangeType(Integer changeType) {
+ this.changeType = changeType;
+ }
+
+ public Integer getChangeType() {
+ return this.changeType;
+ }
+
+ public String getServiceType() { return this.serviceType; }
+
+ public void setServiceType(String serviceType) {
+ this.serviceType = serviceType;
+ }
+
+ public Integer getPolicyType() { return this.policyType; }
+
+ public void setPolicyType(Integer policyType) {
+ this.policyType = policyType;
+ }
+
+ public String getZoneName() { return this.zoneName; }
+
+ public void setZoneName(String zoneName) {
+ this.zoneName = zoneName;
+ }
+
+ public Long getPolicyId() { return this.policyId; }
+
+ public void setPolicyId(Long policyId) {
+ this.policyId = policyId;
+ }
+
+ /**
+ * This return the bean content in string format
+ * @return formatedStr
+ */
+ @Override
+ public String toString( ) {
+ String str = "XXPolicyChangeLog={";
+ str += "id={" + id + "} ";
+ str += "createTime={" + createTime + "} ";
+ str += "serviceId={" + serviceId + "} ";
+ str += "changeType={" + changeType + "} ";
+ str += "policyVersion={" + policyVersion + "} ";
+ str += "serviceType={" + serviceType + "} ";
+ str += "policyType={" + policyType + "} ";
+ str += "zoneName={" + zoneName + "} ";
+ str += "policyId={" + policyId + "} ";
+ str += "}";
+ return str;
+ }
+
+ /**
+ * Checks for all attributes except referenced db objects
+ * @return true if all attributes match
+ */
+ @Override
+ public boolean equals( Object obj) {
+ if (obj == null)
+ return false;
+ if (this == obj)
+ return true;
+ if (!super.equals(obj))
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ XXPolicyChangeLog other = (XXPolicyChangeLog) obj;
+ if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) {
+ return false;
+ }
+ if ((this.serviceId == null && other.serviceId != null) || (this.serviceId != null && !this.serviceId.equals(other.serviceId))) {
+ return false;
+ }
+ if ((this.policyVersion == null && other.policyVersion != null) || (this.policyVersion != null && !this.policyVersion.equals(other.policyVersion))) {
+ return false;
+ }
+ if ((this.createTime == null && other.createTime != null) || (this.createTime != null && !this.createTime.equals(other.createTime))) {
+ return false;
+ }
+ if ((this.changeType == null && other.changeType != null) || (this.changeType != null && !this.changeType.equals(other.changeType))) {
+ return false;
+ }
+ if ((this.serviceType == null && other.serviceType != null) || (this.serviceType != null && !this.serviceType.equals(other.serviceType))) {
+ return false;
+ }
+ if ((this.policyType == null && other.policyType != null) || (this.policyType != null && !this.policyType.equals(other.policyType))) {
+ return false;
+ }
+ if ((this.zoneName == null && other.zoneName != null) || (this.zoneName != null && !this.zoneName.equals(other.zoneName))) {
+ return false;
+ }
+ if ((this.policyId == null && other.policyId != null) || (this.policyId != null && !this.policyId.equals(other.policyId))) {
+ return false;
+ }
+ return true;
+ }
+
+ public static boolean equals(Object object1, Object object2) {
+ if (object1 == object2) {
+ return true;
+ }
+ if ((object1 == null) || (object2 == null)) {
+ return false;
+ }
+ return object1.equals(object2);
+ }
+
+}
+
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java
index d708927..ce577e0 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java
@@ -524,7 +524,7 @@ public class AssetREST {
ServicePolicies servicePolicies = null;
try {
- servicePolicies = serviceREST.getServicePoliciesIfUpdated(repository, lastKnowPolicyVersion, 0L, agentId, "","",request);
+ servicePolicies = serviceREST.getServicePoliciesIfUpdated(repository, lastKnowPolicyVersion, 0L, agentId, "", "", false, request);
} catch(Exception excp) {
logger.error("failed to retrieve policies for repository " + repository, excp);
}
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java
index 0281c94..073404e 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java
@@ -460,4 +460,19 @@ public class PublicAPIsv2 {
}
return pluginInfoList.getPluginInfoList();
}
+
+ @DELETE
+ @Path("/api/server/policydeltas")
+ @PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
+ public void deletePolicyDeltas(@DefaultValue("7") @QueryParam("days") Integer olderThan, @DefaultValue("false") @QueryParam("reloadServicePoliciesCache") Boolean reloadServicePoliciesCache, @Context HttpServletRequest request) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("==> PublicAPIsv2.deletePolicyDeltas(" + olderThan + ", " + reloadServicePoliciesCache + ")");
+ }
+
+ serviceREST.deletePolicyDeltas(olderThan, reloadServicePoliciesCache, request);
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("<== PublicAPIsv2.deletePolicyDeltas(" + olderThan + ", " + reloadServicePoliciesCache + ")");
+ }
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
index a43d076..78029e0 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
@@ -90,6 +90,7 @@ import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerSecurityZone;
import org.apache.ranger.plugin.model.RangerService;
import org.apache.ranger.plugin.model.RangerServiceDef;
@@ -102,7 +103,6 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResource;
import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineCacheForEngineOptions;
-import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
import org.apache.ranger.plugin.service.ResourceLookupContext;
import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
@@ -2734,11 +2734,13 @@ public class ServiceREST {
@QueryParam("pluginId") String pluginId,
@DefaultValue("") @QueryParam("clusterName") String clusterName,
@DefaultValue("") @QueryParam("zoneName") String zoneName,
+ @DefaultValue("false") @QueryParam("supportsPolicyDeltas") Boolean supportsPolicyDeltas,
@Context HttpServletRequest request) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ServiceREST.getServicePoliciesIfUpdated("
+ serviceName + ", " + lastKnownVersion + ", "
- + lastActivationTime + ")");
+ + lastActivationTime + ", " + pluginId + ", "
+ + clusterName + ", " + supportsPolicyDeltas + ")");
}
ServicePolicies ret = null;
@@ -2766,7 +2768,7 @@ public class ServiceREST {
if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.getServicePoliciesIfUpdated(serviceName=" + serviceName + ",lastKnownVersion=" + lastKnownVersion + ",lastActivationTime=" + lastActivationTime + ")");
}
- ServicePolicies servicePolicies = svcStore.getServicePoliciesIfUpdated(serviceName, lastKnownVersion);
+ ServicePolicies servicePolicies = svcStore.getServicePoliciesIfUpdated(serviceName, lastKnownVersion, !supportsPolicyDeltas);
if (servicePolicies == null) {
downloadedVersion = lastKnownVersion;
@@ -2776,12 +2778,17 @@ public class ServiceREST {
Map<String, RangerSecurityZone.RangerSecurityZoneService> securityZones = zoneStore.getSecurityZonesForService(serviceName);
ServicePolicies updatedServicePolicies = getUpdatedServicePoliciesForZones(servicePolicies, securityZones);
downloadedVersion = updatedServicePolicies.getPolicyVersion();
- ret = filterServicePolicies(updatedServicePolicies);
+ if (lastKnownVersion == -1L || !supportsPolicyDeltas) {
+ ret = filterServicePolicies(updatedServicePolicies);
+ } else {
+ ret = updatedServicePolicies;
+ }
+
httpCode = HttpServletResponse.SC_OK;
- logMsg = "Returning " + (ret.getPolicies() != null ? ret.getPolicies().size() : 0) + " policies. Policy version=" + ret.getPolicyVersion();
+ logMsg = "Returning " + (ret.getPolicies() != null ? ret.getPolicies().size() : (ret.getPolicyDeltas() != null ? ret.getPolicyDeltas().size() : 0)) + " policies. Policy version=" + ret.getPolicyVersion();
}
} catch (Throwable excp) {
- LOG.error("getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + lastActivationTime + ") failed");
+ LOG.error("getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + lastActivationTime + ") failed", excp);
httpCode = HttpServletResponse.SC_BAD_REQUEST;
logMsg = excp.getMessage();
@@ -2798,7 +2805,7 @@ public class ServiceREST {
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== ServiceREST.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + lastActivationTime + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
+ LOG.debug("<== ServiceREST.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + lastActivationTime + ", " + pluginId + ", " + clusterName + ", " + supportsPolicyDeltas + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
}
return ret;
@@ -2814,10 +2821,13 @@ public class ServiceREST {
@QueryParam("pluginId") String pluginId,
@DefaultValue("") @QueryParam("clusterName") String clusterName,
@DefaultValue("") @QueryParam("zoneName") String zoneName,
+ @DefaultValue("false") @QueryParam("supportsPolicyDeltas") Boolean supportsPolicyDeltas,
@Context HttpServletRequest request) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ServiceREST.getSecureServicePoliciesIfUpdated("
- + serviceName + ", " + lastKnownVersion + ")");
+ + serviceName + ", " + lastKnownVersion + ", "
+ + lastActivationTime + ", " + pluginId + ", "
+ + clusterName + ", " + supportsPolicyDeltas + ")");
}
ServicePolicies ret = null;
int httpCode = HttpServletResponse.SC_OK;
@@ -2876,7 +2886,7 @@ public class ServiceREST {
}
}
if (isAllowed) {
- ServicePolicies servicePolicies = svcStore.getServicePoliciesIfUpdated(serviceName, lastKnownVersion);
+ ServicePolicies servicePolicies = svcStore.getServicePoliciesIfUpdated(serviceName, lastKnownVersion, !supportsPolicyDeltas);
if (servicePolicies == null) {
downloadedVersion = lastKnownVersion;
httpCode = HttpServletResponse.SC_NOT_MODIFIED;
@@ -2885,9 +2895,13 @@ public class ServiceREST {
Map<String, RangerSecurityZone.RangerSecurityZoneService> securityZones = zoneStore.getSecurityZonesForService(serviceName);
ServicePolicies updatedServicePolicies = getUpdatedServicePoliciesForZones(servicePolicies, securityZones);
downloadedVersion = updatedServicePolicies.getPolicyVersion();
- ret = filterServicePolicies(updatedServicePolicies);
+ if (lastKnownVersion == -1L || !supportsPolicyDeltas) {
+ ret = filterServicePolicies(updatedServicePolicies);
+ } else {
+ ret = updatedServicePolicies;
+ }
httpCode = HttpServletResponse.SC_OK;
- logMsg = "Returning " + (ret.getPolicies() != null ? ret.getPolicies().size() : 0) + " policies. Policy version=" + ret.getPolicyVersion();
+ logMsg = "Returning " + (ret.getPolicies() != null ? ret.getPolicies().size() : (ret.getPolicyDeltas() != null ? ret.getPolicyDeltas().size() : 0)) + " policies. Policy version=" + ret.getPolicyVersion();
}
} else {
@@ -2896,7 +2910,7 @@ public class ServiceREST {
logMsg = "User doesn't have permission to download policy";
}
} catch (Throwable excp) {
- LOG.error("getSecureServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + lastActivationTime + ") failed");
+ LOG.error("getSecureServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + lastActivationTime + ") failed", excp);
httpCode = HttpServletResponse.SC_BAD_REQUEST;
logMsg = excp.getMessage();
} finally {
@@ -2911,10 +2925,25 @@ public class ServiceREST {
throw restErrorUtil.createRESTException(httpCode, logMsg, logError);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== ServiceREST.getSecureServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + lastActivationTime + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
+ LOG.debug("<== ServiceREST.getSecureServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ", " + lastActivationTime + ", " + pluginId + ", " + clusterName + ", " + supportsPolicyDeltas + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()));
}
return ret;
- }
+ }
+
+ @DELETE
+ @Path("/server/policydeltas")
+ @PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
+ public void deletePolicyDeltas(@DefaultValue("7") @QueryParam("days") Integer olderThan, @DefaultValue("false") @QueryParam("reloadServicePoliciesCache") Boolean reloadServicePoliciesCache, @Context HttpServletRequest request) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ServiceREST.deletePolicyDeltas(" + olderThan + ", " + reloadServicePoliciesCache + ")");
+ }
+
+ svcStore.resetPolicyUpdateLog(olderThan, reloadServicePoliciesCache);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceREST.deletePolicyDeltas(" + olderThan + ", " + reloadServicePoliciesCache + ")");
+ }
+ }
private void createPolicyDownloadAudit(String serviceName, Long lastKnownVersion, String pluginId, int httpRespCode, String clusterName, String zoneName, HttpServletRequest request) {
try {
@@ -3369,12 +3398,7 @@ public class ServiceREST {
}
private RangerPolicyEngine getPolicyEngine(String serviceName) throws Exception {
-
- ServicePolicies policies = svcStore.getServicePoliciesIfUpdated(serviceName, -1L);
-
- RangerPolicyEngine ret = new RangerPolicyEngineImpl("ranger-admin", policies, defaultAdminOptions);
-
- return ret;
+ return RangerPolicyEngineCacheForEngineOptions.getInstance().getPolicyEngine(serviceName, svcStore, defaultAdminOptions);
}
@GET
@@ -3651,37 +3675,63 @@ public class ServiceREST {
final ServicePolicies ret;
- if (CollectionUtils.isNotEmpty(servicePolicies.getPolicies()) && MapUtils.isNotEmpty(securityZones)) {
+ if (MapUtils.isNotEmpty(securityZones)) {
- List<RangerPolicy> allPolicies = new ArrayList<>(servicePolicies.getPolicies());
+ ret = new ServicePolicies();
+ ret.setServiceDef(servicePolicies.getServiceDef());
+ ret.setServiceId(servicePolicies.getServiceId());
+ ret.setServiceName(servicePolicies.getServiceName());
+ ret.setAuditMode(servicePolicies.getAuditMode());
+ ret.setPolicyVersion(servicePolicies.getPolicyVersion());
+ ret.setPolicyUpdateTime(servicePolicies.getPolicyUpdateTime());
Map<String, ServicePolicies.SecurityZoneInfo> securityZonesInfo = new HashMap<>();
- for (Map.Entry<String, RangerSecurityZone.RangerSecurityZoneService> entry : securityZones.entrySet()) {
+ if (CollectionUtils.isEmpty(servicePolicies.getPolicyDeltas())) {
+ List<RangerPolicy> allPolicies = new ArrayList<>(servicePolicies.getPolicies());
+
- List<RangerPolicy> zonePolicies = extractZonePolicies(allPolicies, entry.getKey());
+ for (Map.Entry<String, RangerSecurityZone.RangerSecurityZoneService> entry : securityZones.entrySet()) {
- if (CollectionUtils.isNotEmpty(zonePolicies)) {
- allPolicies.removeAll(zonePolicies);
+ List<RangerPolicy> zonePolicies = extractZonePolicies(allPolicies, entry.getKey());
+
+ if (CollectionUtils.isNotEmpty(zonePolicies)) {
+ allPolicies.removeAll(zonePolicies);
+ }
+
+ ServicePolicies.SecurityZoneInfo securityZoneInfo = new ServicePolicies.SecurityZoneInfo();
+ securityZoneInfo.setZoneName(entry.getKey());
+ securityZoneInfo.setPolicies(zonePolicies);
+ securityZoneInfo.setResources(entry.getValue().getResources());
+
+ securityZonesInfo.put(entry.getKey(), securityZoneInfo);
}
- ServicePolicies.SecurityZoneInfo securityZoneInfo = new ServicePolicies.SecurityZoneInfo();
- securityZoneInfo.setPolicies(zonePolicies);
- securityZoneInfo.setResources(entry.getValue().getResources());
+ ret.setPolicies(allPolicies);
+ ret.setTagPolicies(servicePolicies.getTagPolicies());
+ ret.setSecurityZones(securityZonesInfo);
+ } else {
+ List<RangerPolicyDelta> allPolicyDeltas = new ArrayList<>(servicePolicies.getPolicyDeltas());
+
+ for (Map.Entry<String, RangerSecurityZone.RangerSecurityZoneService> entry : securityZones.entrySet()) {
+
+ List<RangerPolicyDelta> zonePolicyDeltas = extractZonePolicyDeltas(allPolicyDeltas, entry.getKey());
+
+ if (CollectionUtils.isNotEmpty(zonePolicyDeltas)) {
+ allPolicyDeltas.removeAll(zonePolicyDeltas);
+ }
+
+ ServicePolicies.SecurityZoneInfo securityZoneInfo = new ServicePolicies.SecurityZoneInfo();
+ securityZoneInfo.setZoneName(entry.getKey());
+ securityZoneInfo.setPolicyDeltas(zonePolicyDeltas);
+ securityZoneInfo.setResources(entry.getValue().getResources());
+
+ securityZonesInfo.put(entry.getKey(), securityZoneInfo);
+ }
+ ret.setPolicyDeltas(allPolicyDeltas);
- securityZonesInfo.put(entry.getKey(), securityZoneInfo);
}
- ret = new ServicePolicies();
- ret.setServiceDef(servicePolicies.getServiceDef());
- ret.setServiceId(servicePolicies.getServiceId());
- ret.setServiceName(servicePolicies.getServiceName());
- ret.setPolicies(allPolicies);
- ret.setTagPolicies(servicePolicies.getTagPolicies());
- ret.setAuditMode(servicePolicies.getAuditMode());
- ret.setPolicyVersion(servicePolicies.getPolicyVersion());
- ret.setPolicyUpdateTime(servicePolicies.getPolicyUpdateTime());
ret.setSecurityZones(securityZonesInfo);
-
} else {
ret = servicePolicies;
}
@@ -3700,6 +3750,19 @@ public class ServiceREST {
return ret;
}
+
+ private static List<RangerPolicyDelta> extractZonePolicyDeltas(final List<RangerPolicyDelta> allPolicyDeltas, final String zoneName) {
+
+ final List<RangerPolicyDelta> ret = new ArrayList<>();
+
+ for (RangerPolicyDelta delta : allPolicyDeltas) {
+ if (StringUtils.equals(delta.getZoneName(), zoneName)) {
+ ret.add(delta);
+ }
+ }
+
+ return ret;
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneService.java
index ab89319..04003f4 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneService.java
@@ -39,6 +39,7 @@ import org.apache.ranger.entity.XXSecurityZone;
import org.apache.ranger.entity.XXServiceVersionInfo;
import org.apache.ranger.entity.XXTrxLog;
import org.apache.ranger.entity.XXUser;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerSecurityZone;
import org.apache.ranger.util.RangerEnumUtil;
import org.springframework.beans.factory.annotation.Autowired;
@@ -202,7 +203,7 @@ public class RangerSecurityZoneService extends RangerSecurityZoneServiceBase<XXS
final Long finalServiceId = serviceVersionInfo.getServiceId();
final ServiceDBStore.VERSION_TYPE versionType = ServiceDBStore.VERSION_TYPE.POLICY_VERSION;
- Runnable serviceVersionUpdater = new ServiceDBStore.ServiceVersionUpdater(finaldaoManager, finalServiceId, versionType);
+ Runnable serviceVersionUpdater = new ServiceDBStore.ServiceVersionUpdater(finaldaoManager, finalServiceId, versionType, null, RangerPolicyDelta.CHANGE_TYPE_SERVICE_CHANGE, null);
daoMgr.getRangerTransactionSynchronizationAdapter().executeOnTransactionCommit(serviceVersionUpdater);
}
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index a660601..eaa4e08 100644
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -450,7 +450,9 @@
and exists (select tagService from XXService tagService where obj.tagService = tagService.id and tagService.isEnabled = TRUE)
</query>
</named-query>
-
+ <named-query name="XXService.getAllServiceIds">
+ <query>select obj.id from XXService obj</query>
+ </named-query>
<!-- XXServiceVersionInfo -->
<named-query name="XXServiceVersionInfo.findByServiceName">
<query>
@@ -1374,9 +1376,35 @@
</named-query>
<named-query name="XXGlobalState.findByStateName">
- <query>
+ <query>
select obj from XXGlobalState obj where obj.stateName = :stateName
+ </query>
+ </named-query>
+
+ <!-- XXPolicyChangeLog -->
+
+ <named-query name="XXPolicyChangeLog.findByServiceId">
+ <query>
+ select obj from XXPolicyChangeLog obj where obj.serviceId = :serviceId
+ </query>
+ </named-query>
+ <named-query name="XXPolicyChangeLog.findSinceVersion">
+ <query>
+ select obj.id, obj.changeType, obj.policyVersion, obj.serviceType, obj.policyType, obj.policyId, obj.zoneName from
+ XXPolicyChangeLog obj where obj.serviceId = :serviceId and obj.policyVersion >= :version order by
+ obj.policyVersion
+ </query>
+ </named-query>
+
+ <named-query name="XXPolicyChangeLog.findGreaterThan">
+ <query>
+ select obj.id, obj.changeType, obj.policyVersion, obj.serviceType, obj.policyType, obj.policyId, obj.zoneName from
+ XXPolicyChangeLog obj where obj.serviceId = :serviceId and obj.id > :id order by obj.id
</query>
</named-query>
+ <named-query name="XXPolicyChangeLog.deleteOlderThan">
+ <query>delete from XXPolicyChangeLog obj where obj.createTime < :olderThan</query>
+ </named-query>
+
</entity-mappings>
diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
index bf19efd..4d46d0e 100644
--- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
+++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java
@@ -2081,7 +2081,7 @@ public class TestServiceDBStore {
Mockito.when(xServiceVersionInfoDao.findByServiceName(serviceName)).thenReturn(xServiceVersionInfo);
ServicePolicies dbServicePolicies = serviceDBStore
- .getServicePoliciesIfUpdated(serviceName, lastKnownVersion);
+ .getServicePoliciesIfUpdated(serviceName, lastKnownVersion, true);
Assert.assertNull(dbServicePolicies);
}
diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestAssetREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestAssetREST.java
index a1b0e45..ef149d5 100644
--- a/security-admin/src/test/java/org/apache/ranger/rest/TestAssetREST.java
+++ b/security-admin/src/test/java/org/apache/ranger/rest/TestAssetREST.java
@@ -635,7 +635,7 @@ public class TestAssetREST {
// Mockito.when(PropertiesUtil.getBooleanProperty("ranger.service.http.enabled",true)).thenReturn(true);
try {
Mockito.when(serviceREST.getServicePoliciesIfUpdated(Mockito.anyString(), Mockito.anyLong(),
- Mockito.anyLong(), Mockito.anyString(), Mockito.anyString() , Mockito.anyString(), (HttpServletRequest) Mockito.any()))
+ Mockito.anyLong(), Mockito.anyString(), Mockito.anyString() , Mockito.anyString() , Mockito.anyBoolean(), (HttpServletRequest) Mockito.any()))
.thenReturn(servicePolicies);
} catch (Exception e) {
fail("test failed due to: " + e.getMessage());
diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java
index ef4865a..ed2b7e5 100644
--- a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java
+++ b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java
@@ -997,7 +997,7 @@ public class TestServiceREST {
ServicePolicies dbServicePolicies = serviceREST
.getServicePoliciesIfUpdated(serviceName, lastKnownVersion, 0L,
- pluginId, "", "", request);
+ pluginId, "", "", false, request);
Assert.assertNull(dbServicePolicies);
}
@@ -1802,7 +1802,7 @@ public class TestServiceREST {
Mockito.when(restErrorUtil.createRESTException(Mockito.anyInt(), Mockito.anyString(), Mockito.anyBoolean()))
.thenThrow(new WebApplicationException());
thrown.expect(WebApplicationException.class);
- serviceREST.getServicePoliciesIfUpdated(serviceName, lastKnownVersion, 0L, pluginId, "", "", request);
+ serviceREST.getServicePoliciesIfUpdated(serviceName, lastKnownVersion, 0L, pluginId, "", "", false, request);
}
@Test
@@ -1813,10 +1813,10 @@ public class TestServiceREST {
Long lastKnownVersion = 1L;
String pluginId = "1";
Mockito.when(serviceUtil.isValidateHttpsAuthentication(serviceName, request)).thenReturn(true);
- Mockito.when(svcStore.getServicePoliciesIfUpdated(serviceName, lastKnownVersion)).thenReturn(servicePolicies);
+ Mockito.when(svcStore.getServicePoliciesIfUpdated(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(servicePolicies);
Mockito.when(zoneStore.getSecurityZonesForService(serviceName)).thenReturn(null);
ServicePolicies dbServicePolicies = serviceREST.getServicePoliciesIfUpdated(serviceName, lastKnownVersion, 0L,
- pluginId, "", "", request);
+ pluginId, "", "", true, request);
Assert.assertNotNull(dbServicePolicies);
}
@@ -1840,7 +1840,7 @@ public class TestServiceREST {
.thenThrow(new WebApplicationException());
thrown.expect(WebApplicationException.class);
- serviceREST.getSecureServicePoliciesIfUpdated(serviceName, lastKnownVersion, 0L, pluginId, "", "", request);
+ serviceREST.getSecureServicePoliciesIfUpdated(serviceName, lastKnownVersion, 0L, pluginId, "", "", false, request);
}
@Test
@@ -1866,7 +1866,7 @@ public class TestServiceREST {
.thenThrow(new WebApplicationException());
thrown.expect(WebApplicationException.class);
- serviceREST.getSecureServicePoliciesIfUpdated(serviceName, lastKnownVersion, 0L, pluginId, "", "", request);
+ serviceREST.getSecureServicePoliciesIfUpdated(serviceName, lastKnownVersion, 0L, pluginId, "", "", false, request);
}
@Test
@@ -1889,17 +1889,17 @@ public class TestServiceREST {
Mockito.when(xServiceDefDao.getById(xService.getType())).thenReturn(xServiceDef);
Mockito.when(svcStore.getServiceByNameForDP(serviceName)).thenReturn(rs);
Mockito.when(bizUtil.isUserAllowed(rs, ServiceREST.Allowed_User_List_For_Grant_Revoke)).thenReturn(true);
- Mockito.when(svcStore.getServicePoliciesIfUpdated(serviceName, lastKnownVersion)).thenReturn(sp);
+ Mockito.when(svcStore.getServicePoliciesIfUpdated(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(sp);
Mockito.when(zoneStore.getSecurityZonesForService(serviceName)).thenReturn(null);
- ServicePolicies dbServiceSecurePolicies = serviceREST.getSecureServicePoliciesIfUpdated(serviceName,
- lastKnownVersion, 0L, pluginId, "", "", request);
+ ServicePolicies dbServiceSecurePolicies = serviceREST.getSecureServicePoliciesIfUpdated(serviceName,
+ lastKnownVersion, 0L, pluginId, "", "", true, request);
Assert.assertNotNull(dbServiceSecurePolicies);
Mockito.verify(serviceUtil).isValidService(serviceName, request);
Mockito.verify(xServiceDao).findByName(serviceName);
Mockito.verify(xServiceDefDao).getById(xService.getType());
Mockito.verify(svcStore).getServiceByNameForDP(serviceName);
Mockito.verify(bizUtil).isUserAllowed(rs, ServiceREST.Allowed_User_List_For_Grant_Revoke);
- Mockito.verify(svcStore).getServicePoliciesIfUpdated(serviceName, lastKnownVersion);
+ Mockito.verify(svcStore).getServicePoliciesIfUpdated(serviceName, lastKnownVersion, false);
}
@Test