You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2022/10/19 04:46:14 UTC
[ranger] branch ranger-2.4 updated: RANGER-3951: optimized memory used for tags in plugins and server
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.4
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.4 by this push:
new 0bd30593b RANGER-3951: optimized memory used for tags in plugins and server
0bd30593b is described below
commit 0bd30593be0eac8afae020028a9b932004129b22
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Fri Oct 14 18:50:19 2022 -0700
RANGER-3951: optimized memory used for tags in plugins and server
(cherry picked from commit 50cd652b1a484e971359d9123ad36c523fdf5929)
---
.../contextenricher/RangerAdminTagRetriever.java | 16 +++-
.../plugin/contextenricher/RangerTagForEval.java | 8 +-
.../ranger/plugin/model/RangerServiceResource.java | 5 +-
.../org/apache/ranger/plugin/model/RangerTag.java | 45 ++++-------
.../apache/ranger/plugin/model/RangerTagDef.java | 7 +-
.../policyengine/RangerPolicyRepository.java | 2 +-
.../ranger/plugin/util/RangerCommonConstants.java | 2 +-
.../plugin/util/RangerServiceTagsDeltaUtil.java | 94 +++++++++++++++-------
.../org/apache/ranger/plugin/util/ServiceTags.java | 31 ++++---
.../apache/ranger/biz/RangerTagDBRetriever.java | 10 ++-
.../java/org/apache/ranger/biz/TagDBStore.java | 12 ++-
.../ranger/common/RangerServiceTagsCache.java | 2 +-
.../patch/PatchForUpdatingTagsJson_J10020.java | 4 +
.../apache/ranger/rest/ServiceTagsProcessor.java | 7 +-
14 files changed, 157 insertions(+), 88 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
index 1df5587cb..b2b7d5f71 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
@@ -33,7 +33,12 @@ import java.util.Map;
public class RangerAdminTagRetriever extends RangerTagRetriever {
private static final Logger LOG = LoggerFactory.getLogger(RangerAdminTagRetriever.class);
+ private static final String OPTION_DEDUP_TAGS = "deDupTags";
+ private static final Boolean OPTION_DEDUP_TAGS_DEFAULT = true;
+
+
private RangerAdminClient adminClient;
+ private boolean deDupTags;
@Override
public void init(Map<String, String> options) {
@@ -45,10 +50,12 @@ public class RangerAdminTagRetriever extends RangerTagRetriever {
pluginConfig = new RangerPluginConfig(serviceDef.getName(), serviceName, appId, null, null, null);
}
+ String deDupTagsVal = options != null? options.get(OPTION_DEDUP_TAGS) : null;
RangerPluginContext pluginContext = getPluginContext();
RangerAdminClient rangerAdmin = pluginContext.getAdminClient();
- this.adminClient = (rangerAdmin != null) ? rangerAdmin : pluginContext.createAdminClient(pluginConfig);
+ this.deDupTags = StringUtils.isNotBlank(deDupTagsVal) ? Boolean.parseBoolean(deDupTagsVal) : OPTION_DEDUP_TAGS_DEFAULT;
+ this.adminClient = (rangerAdmin != null) ? rangerAdmin : pluginContext.createAdminClient(pluginConfig);
} else {
LOG.error("FATAL: Cannot find service/serviceDef to use for retrieving tags. Will NOT be able to retrieve tags.");
}
@@ -70,6 +77,13 @@ public class RangerAdminTagRetriever extends RangerTagRetriever {
LOG.error("Returning null service tags");
}
}
+
+ if (serviceTags != null && !serviceTags.getIsDelta() && deDupTags) {
+ final int countOfDuplicateTags = serviceTags.dedupTags();
+
+ LOG.info("Number of duplicate tags removed from the received serviceTags:[" + countOfDuplicateTags + "]. Number of tags in the de-duplicated serviceTags :[" + serviceTags.getTags().size() + "].");
+ }
+
return serviceTags;
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java
index 1c634b393..bb329fac4 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java
@@ -44,7 +44,7 @@ import java.util.List;
import java.util.Map;
@JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY)
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
@JsonIgnoreProperties(ignoreUnknown=true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@@ -69,10 +69,10 @@ public class RangerTagForEval implements Serializable {
public RangerTagForEval(RangerTag tag, RangerPolicyResourceMatcher.MatchType matchType) {
this.type = tag.getType();
- this.attributes = tag.getAttributes();
- this.options = tag.getOptions();
+ this.attributes = tag.getAttributes() != null ? tag.getAttributes() : Collections.emptyMap();
+ this.options = tag.getOptions() != null ? tag.getOptions() : Collections.emptyMap();
this.matchType = matchType;
- this.validityPeriods = tag.getValidityPeriods();
+ this.validityPeriods = tag.getValidityPeriods() != null ? tag.getValidityPeriods() : Collections.emptyList();
this.validityPeriodEvaluators = createValidityPeriodEvaluators();
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceResource.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceResource.java
index 47d006048..1c46ea5c2 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceResource.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceResource.java
@@ -26,11 +26,10 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.HashMap;
import java.util.Map;
@JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY)
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
@JsonIgnoreProperties(ignoreUnknown=true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@@ -92,7 +91,7 @@ public class RangerServiceResource extends RangerBaseModelObject {
}
public void setResourceElements(Map<String, RangerPolicy.RangerPolicyResource> resource) {
- this.resourceElements = resource == null ? new HashMap<String, RangerPolicy.RangerPolicyResource>() : resource;
+ this.resourceElements = resource;
}
public void setResourceSignature(String resourceSignature) {
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTag.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTag.java
index 9e620c32c..4154ea32b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTag.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTag.java
@@ -26,13 +26,11 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY)
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
@JsonIgnoreProperties(ignoreUnknown=true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@@ -45,7 +43,7 @@ public class RangerTag extends RangerBaseModelObject {
public static final String OPTION_TAG_VALIDITY_PERIODS = "TAG_VALIDITY_PERIODS";
private String type;
- private Short owner = OWNER_SERVICERESOURCE;
+ private Short owner;
private Map<String, String> attributes;
private Map<String, Object> options;
private List<RangerValiditySchedule> validityPeriods;
@@ -87,7 +85,7 @@ public class RangerTag extends RangerBaseModelObject {
}
public void setAttributes(Map<String, String> attributes) {
- this.attributes = attributes == null ? new HashMap<String, String>() : attributes;
+ this.attributes = attributes;
}
public Short getOwner() {
@@ -101,36 +99,15 @@ public class RangerTag extends RangerBaseModelObject {
public Map<String, Object> getOptions() { return options; }
public void setOptions(Map<String, Object> options) {
- if (this.options == null) {
- this.options = new HashMap<>();
- }
- if (this.options == options) {
- return;
- }
- this.options.clear();
-
- if(options != null) {
- for(Map.Entry<String, Object> e : options.entrySet()) {
- this.options.put(e.getKey(), e.getValue());
- }
- }
+ this.options = options;
}
public List<RangerValiditySchedule> getValidityPeriods() { return validityPeriods; }
public void setValidityPeriods(List<RangerValiditySchedule> validityPeriods) {
- if (this.validityPeriods == null) {
- this.validityPeriods = new ArrayList<>();
- }
- if (this.validityPeriods == validityPeriods) {
- return;
- }
- this.validityPeriods.clear();
-
- if(validityPeriods != null) {
- this.validityPeriods.addAll(validityPeriods);
- }
+ this.validityPeriods = validityPeriods;
}
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -158,6 +135,16 @@ public class RangerTag extends RangerBaseModelObject {
}
sb.append("} ");
+ sb.append("options={");
+ if (options != null) {
+ for (Map.Entry<String, Object> e : options.entrySet()) {
+ sb.append(e.getKey()).append("={");
+ sb.append(e.getValue());
+ sb.append("} ");
+ }
+ }
+ sb.append("} ");
+
if (validityPeriods != null) {
sb.append("validityPeriods={").append(validityPeriods).append("} ");
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java
index f1a83cfaf..6fc6f9553 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java
@@ -26,7 +26,6 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -37,7 +36,7 @@ import java.util.Objects;
*
*/
@JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY)
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
@JsonIgnoreProperties(ignoreUnknown=true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@@ -86,7 +85,7 @@ public class RangerTagDef extends RangerBaseModelObject {
}
public void setAttributeDefs(List<RangerTagAttributeDef> attributeDefs) {
- this.attributeDefs = attributeDefs == null ? new ArrayList<RangerTagAttributeDef>() : attributeDefs;
+ this.attributeDefs = attributeDefs;
}
@Override
@@ -119,7 +118,7 @@ public class RangerTagDef extends RangerBaseModelObject {
*/
@JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY)
- @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+ @JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
@JsonIgnoreProperties(ignoreUnknown=true)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index ff83c7434..85a3afd01 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -300,7 +300,7 @@ public class RangerPolicyRepository {
auditFilterResourceTrie = null;
} else {
policyResourceTrie = createResourceTrieMap(policyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
- dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTagTrieForSpace);
+ dataMaskResourceTrie = createResourceTrieMap(dataMaskPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
rowFilterResourceTrie = createResourceTrieMap(rowFilterPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
auditFilterResourceTrie = createResourceTrieMap(auditPolicyEvaluators, options.optimizeTrieForRetrieval, options.optimizeTrieForSpace);
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
index 86f881f80..73e3dadbf 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
@@ -50,7 +50,7 @@ public class RangerCommonConstants {
public static final boolean RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT = false;
public static final boolean PLUGIN_CONFIG_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT = false;
- public static final boolean RANGER_ADMIN_SUPPORTS_TAGS_DEDUP_DEFAULT = false;
+ public static final boolean RANGER_ADMIN_SUPPORTS_TAGS_DEDUP_DEFAULT = true;
public static final boolean POLICY_REST_CLIENT_SESSION_COOKIE_ENABLED = true;
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
index 0365fc992..f2e68aed9 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
@@ -54,6 +54,7 @@ public class RangerServiceTagsDeltaUtil {
LOG.debug("==> RangerServiceTagsDeltaUtil.applyDelta()");
}
+ ServiceTags ret = serviceTags;
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_TAGS_DELTA_LOG)) {
@@ -61,13 +62,15 @@ public class RangerServiceTagsDeltaUtil {
}
if (serviceTags != null && !serviceTags.getIsDelta() && delta != null && delta.getIsDelta()) {
- serviceTags.setServiceName(delta.getServiceName());
- serviceTags.setTagVersion(delta.getTagVersion());
+ ret = new ServiceTags(serviceTags);
+
+ ret.setServiceName(delta.getServiceName());
+ ret.setTagVersion(delta.getTagVersion());
int tagDefsAdded = 0, tagDefsUpdated = 0, tagDefsRemoved = 0;
int tagsAdded = 0, tagsUpdated = 0, tagsRemoved = 0;
- Map<Long, RangerTagDef> tagDefs = serviceTags.getTagDefinitions();
+ Map<Long, RangerTagDef> tagDefs = ret.getTagDefinitions();
for (Iterator<Map.Entry<Long, RangerTagDef>> deltaTagDefIter = delta.getTagDefinitions().entrySet().iterator(); deltaTagDefIter.hasNext(); ) {
Map.Entry<Long, RangerTagDef> entry = deltaTagDefIter.next();
@@ -91,7 +94,7 @@ public class RangerServiceTagsDeltaUtil {
}
}
- Map<Long, RangerTag> tags = serviceTags.getTags();
+ Map<Long, RangerTag> tags = ret.getTags();
Map<Long, Long> replacedTagIds = new HashMap<>();
for (Iterator<Map.Entry<Long, RangerTag>> deltaTagIter = delta.getTags().entrySet().iterator(); deltaTagIter.hasNext(); ) {
@@ -106,15 +109,15 @@ public class RangerServiceTagsDeltaUtil {
tagsRemoved++;
if (isSupportsTagsDedup()) {
- serviceTags.cachedTags.remove(removedTag);
+ ret.cachedTags.remove(removedTag);
}
}
} else {
if (isSupportsTagsDedup()) {
- Long cachedTagId = serviceTags.cachedTags.get(deltaTag);
+ Long cachedTagId = ret.cachedTags.get(deltaTag);
if (cachedTagId == null) {
- serviceTags.cachedTags.put(deltaTag, deltaTagId);
+ ret.cachedTags.put(deltaTag, deltaTagId);
tags.put(deltaTagId, deltaTag);
} else {
replacedTagIds.put(deltaTagId, cachedTagId);
@@ -132,8 +135,8 @@ public class RangerServiceTagsDeltaUtil {
}
}
- List<RangerServiceResource> serviceResources = serviceTags.getServiceResources();
- Map<Long, List<Long>> resourceToTagIds = serviceTags.getResourceToTagIds();
+ List<RangerServiceResource> serviceResources = ret.getServiceResources();
+ Map<Long, List<Long>> resourceToTagIds = ret.getResourceToTagIds();
Map<Long, RangerServiceResource> idResourceMap = serviceResources.stream().collect(Collectors.toMap(RangerServiceResource::getId, Function.identity()));
Map<Long, RangerServiceResource> resourcesToRemove = new HashMap<>();
Map<Long, RangerServiceResource> resourcesToAdd = new HashMap<>();
@@ -237,7 +240,7 @@ public class RangerServiceTagsDeltaUtil {
RangerPerfTracer.log(perf);
- return serviceTags;
+ return ret;
}
public static void pruneUnusedAttributes(ServiceTags serviceTags) {
@@ -245,33 +248,70 @@ public class RangerServiceTagsDeltaUtil {
serviceTags.setTagUpdateTime(null);
for (Map.Entry<Long, RangerTagDef> entry : serviceTags.getTagDefinitions().entrySet()) {
- RangerTagDef tagDef = entry.getValue();
- tagDef.setCreatedBy(null);
- tagDef.setCreateTime(null);
- tagDef.setUpdatedBy(null);
- tagDef.setUpdateTime(null);
- tagDef.setGuid(null);
+ pruneUnusedAttributes(entry.getValue());
}
for (Map.Entry<Long, RangerTag> entry : serviceTags.getTags().entrySet()) {
- RangerTag tag = entry.getValue();
- tag.setCreatedBy(null);
- tag.setCreateTime(null);
- tag.setUpdatedBy(null);
- tag.setUpdateTime(null);
- tag.setGuid(null);
+ pruneUnusedAttributes(entry.getValue());
}
for (RangerServiceResource serviceResource : serviceTags.getServiceResources()) {
- serviceResource.setCreatedBy(null);
- serviceResource.setCreateTime(null);
- serviceResource.setUpdatedBy(null);
- serviceResource.setUpdateTime(null);
- serviceResource.setGuid(null);
+ pruneUnusedAttributes(serviceResource);
}
}
}
+ public static void pruneUnusedAttributes(RangerTagDef tagDef) {
+ tagDef.setCreatedBy(null);
+ tagDef.setCreateTime(null);
+ tagDef.setUpdatedBy(null);
+ tagDef.setUpdateTime(null);
+ tagDef.setGuid(null);
+ tagDef.setVersion(null);
+
+ if (tagDef.getAttributeDefs() != null && tagDef.getAttributeDefs().isEmpty()) {
+ tagDef.setAttributeDefs(null);
+ }
+ }
+
+ public static void pruneUnusedAttributes(RangerTag tag) {
+ tag.setCreatedBy(null);
+ tag.setCreateTime(null);
+ tag.setUpdatedBy(null);
+ tag.setUpdateTime(null);
+ tag.setGuid(null);
+ tag.setVersion(null);
+
+ if (tag.getOwner() != null && tag.getOwner().shortValue() == RangerTag.OWNER_SERVICERESOURCE) {
+ tag.setOwner(null);
+ }
+
+ if (tag.getAttributes() != null && tag.getAttributes().isEmpty()) {
+ tag.setAttributes(null);
+ }
+
+ if (tag.getOptions() != null && tag.getOptions().isEmpty()) {
+ tag.setOptions(null);
+ }
+
+ if (tag.getValidityPeriods() != null && tag.getValidityPeriods().isEmpty()) {
+ tag.setValidityPeriods(null);
+ }
+ }
+
+ public static void pruneUnusedAttributes(RangerServiceResource serviceResource) {
+ serviceResource.setCreatedBy(null);
+ serviceResource.setCreateTime(null);
+ serviceResource.setUpdatedBy(null);
+ serviceResource.setUpdateTime(null);
+ serviceResource.setGuid(null);
+ serviceResource.setVersion(null);
+
+ if (serviceResource.getAdditionalInfo() != null && serviceResource.getAdditionalInfo().isEmpty()) {
+ serviceResource.setAdditionalInfo(null);
+ }
+ }
+
public static boolean isSupportsTagsDedup() {
if (!SUPPORTS_TAGS_DEDUP_INITIALIZED) {
RangerAdminConfig config = RangerAdminConfig.getInstance();
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
index b4f250038..d2a37297f 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
@@ -91,6 +91,22 @@ public class ServiceTags implements java.io.Serializable {
setIsDelta(isDelta);
setTagsChangeExtent(tagsChangeExtent);
}
+
+ public ServiceTags(ServiceTags other) {
+ setOp(other.getOp());
+ setServiceName(other.getServiceName());
+ setTagVersion(other.getTagVersion());
+ setTagUpdateTime(other.getTagUpdateTime());
+ setTagDefinitions(other.getTagDefinitions() != null ? new HashMap<>(other.getTagDefinitions()) : null);
+ setTags(other.getTags() != null ? new HashMap<>(other.getTags()) : null);
+ setServiceResources(other.getServiceResources() != null ? new ArrayList<>(other.getServiceResources()) : null);
+ setResourceToTagIds(other.getResourceToTagIds() != null ? new HashMap<>(other.getResourceToTagIds()) : null);
+ setIsDelta(other.getIsDelta());
+ setTagsChangeExtent(other.getTagsChangeExtent());
+
+ this.cachedTags = new HashMap<>(other.cachedTags);
+ }
+
/**
* @return the op
*/
@@ -217,14 +233,11 @@ public class ServiceTags implements java.io.Serializable {
}
public int dedupTags() {
- final int ret;
+ final int ret;
+ final Map<Long, Long> replacedIds = new HashMap<>();
+ final int initialTagsCount = tags.size();
- Map<Long, Long> replacedIds = new HashMap<>();
- Iterator<Map.Entry<Long, RangerTag>> iter = tags.entrySet().iterator();
-
- final int initialTagsCount = tags.size();
-
- while (iter.hasNext()) {
+ for (Iterator<Map.Entry<Long, RangerTag>> iter = tags.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry<Long, RangerTag> entry = iter.next();
Long tagId = entry.getKey();
RangerTag tag = entry.getValue();
@@ -241,9 +254,7 @@ public class ServiceTags implements java.io.Serializable {
final int finalTagsCount = tags.size();
for (Map.Entry<Long, List<Long>> resourceEntry : resourceToTagIds.entrySet()) {
- ListIterator<Long> listIter = resourceEntry.getValue().listIterator();
-
- while (iter.hasNext()) {
+ for (ListIterator<Long> listIter = resourceEntry.getValue().listIterator(); listIter.hasNext(); ) {
Long tagId = listIter.next();
Long replacerTagId = replacedIds.get(tagId);
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
index c6b8ab937..321148307 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java
@@ -36,6 +36,7 @@ import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.entity.*;
import org.apache.ranger.plugin.model.*;
import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.apache.ranger.plugin.util.RangerServiceTagsDeltaUtil;
import org.apache.ranger.service.RangerServiceResourceService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -109,6 +110,7 @@ public class RangerTagDBRetriever {
}
}
+
List<RangerServiceResource> getServiceResources() {
return serviceResources;
}
@@ -295,13 +297,19 @@ public class RangerTagDBRetriever {
ret.setResourceElements(serviceResourceElements);
List<RangerTag> tags = gsonBuilder.fromJson(xServiceResource.getTags(), RangerServiceResourceService.duplicatedDataType);
+
+ if (CollectionUtils.isNotEmpty(tags)) {
+ for (RangerTag tag : tags) {
+ RangerServiceTagsDeltaUtil.pruneUnusedAttributes(tag);
+ }
+ }
+
lookupCache.serviceResourceToTags.put(xServiceResource.getId(), tags);
}
}
return ret;
}
-
}
private class TagRetrieverTagDefContext {
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index 1022accce..0ab3258c6 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -744,7 +744,7 @@ public class TagDBStore extends AbstractTagStore {
rangerTagResourceMapService.delete(tagResourceMap);
- if (tag.getOwner() == RangerTag.OWNER_SERVICERESOURCE) {
+ if (tag.getOwner() == null || tag.getOwner() == RangerTag.OWNER_SERVICERESOURCE) {
deleteTag(tagId);
}
// We also need to update tags stored with the resource
@@ -1259,6 +1259,8 @@ public class TagDBStore extends AbstractTagStore {
}
}
+ RangerServiceTagsDeltaUtil.pruneUnusedAttributes(tag);
+
ret.getTags().put(tag.getId(), tag);
}
@@ -1270,6 +1272,8 @@ public class TagDBStore extends AbstractTagStore {
RangerTagDef tagDef = xTagDef != null ? rangerTagDefService.getPopulatedViewObject(xTagDef) : null;
if (tagDef != null) {
+ RangerServiceTagsDeltaUtil.pruneUnusedAttributes(tagDef);
+
ret.getTagDefinitions().put(tagDef.getId(), tagDef);
} else {
if (LOG.isDebugEnabled()) {
@@ -1309,6 +1313,8 @@ public class TagDBStore extends AbstractTagStore {
List<Long> resourceTagIds = new ArrayList<>(tags.size());
for (RangerTag tag : tags) {
+ RangerServiceTagsDeltaUtil.pruneUnusedAttributes(tag);
+
if (!ret.getTags().containsKey(tag.getId())) {
ret.getTags().put(tag.getId(), tag);
}
@@ -1321,13 +1327,13 @@ public class TagDBStore extends AbstractTagStore {
}
}
+ RangerServiceTagsDeltaUtil.pruneUnusedAttributes(serviceResource);
+
ret.getServiceResources().add(serviceResource);
tagsChangeExtent = ServiceTags.TagsChangeExtent.SERVICE_RESOURCE;
}
ret.setTagsChangeExtent(tagsChangeExtent);
-
- RangerServiceTagsDeltaUtil.pruneUnusedAttributes(ret);
}
} else {
if (LOG.isDebugEnabled()) {
diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
index 93c283fbc..7fc4caa21 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java
@@ -314,7 +314,7 @@ public class RangerServiceTagsCache {
if (LOG.isDebugEnabled()) {
LOG.debug("Retrieved tag-deltas from database. These will be applied on top of ServiceTags version:[" + cachedServiceTagsVersion + "], tag-deltas:[" + serviceTagsFromDb.getTagVersion() + "]");
}
- RangerServiceTagsDeltaUtil.applyDelta(serviceTags, serviceTagsFromDb);
+ this.serviceTags = RangerServiceTagsDeltaUtil.applyDelta(serviceTags, serviceTagsFromDb);
this.deltaCache = new ServiceTagsDeltasCache(cachedServiceTagsVersion, serviceTagsFromDb);
}
} else {
diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java
index 301d5c30f..c8516d2b7 100644
--- a/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java
+++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchForUpdatingTagsJson_J10020.java
@@ -828,6 +828,10 @@ public class PatchForUpdatingTagsJson_J10020 extends BaseLoader {
String attributeName = xTagAttribute.getName();
String attributeValue = xTagAttribute.getValue();
+ if (tag.getAttributes() == null) {
+ tag.setAttributes(new HashMap<>());
+ }
+
tag.getAttributes().put(attributeName, attributeValue);
} else if (xTagAttribute.getTagId().compareTo(tag.getId()) > 0) {
if (iterTagAttribute.hasPrevious()) {
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
index b256e2838..79786c220 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
@@ -34,6 +34,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -432,7 +433,7 @@ public class ServiceTagsProcessor {
return ret;
}
private boolean isResourcePrivateTag(RangerTag tag) {
- return tag.getOwner() == RangerTag.OWNER_SERVICERESOURCE;
+ return tag.getOwner() == null || tag.getOwner() == RangerTag.OWNER_SERVICERESOURCE;
}
private RangerTag findMatchingTag(RangerTag incomingTag, List<RangerTag> existingTags) throws Exception {
@@ -468,8 +469,8 @@ public class ServiceTagsProcessor {
if (StringUtils.equals(incomingTag.getType(), existingTag.getType())) {
// Check attribute values
- Map<String, String> incomingTagAttributes = incomingTag.getAttributes();
- Map<String, String> existingTagAttributes = existingTag.getAttributes();
+ Map<String, String> incomingTagAttributes = incomingTag.getAttributes() != null ? incomingTag.getAttributes() : Collections.emptyMap();
+ Map<String, String> existingTagAttributes = existingTag.getAttributes() != null ? existingTag.getAttributes() : Collections.emptyMap();
if (CollectionUtils.isEqualCollection(incomingTagAttributes.keySet(), existingTagAttributes.keySet())) {