You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ab...@apache.org on 2021/12/02 17:37:34 UTC
[ranger] branch master updated: RANGER-3519: Provide an option to optimize space needed by Trie objects - Part 2
This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 5852efd RANGER-3519: Provide an option to optimize space needed by Trie objects - Part 2
5852efd is described below
commit 5852efde1cba728ad580231ad02145ea72861186
Author: Abhay Kulkarni <ab...@apache.org>
AuthorDate: Thu Dec 2 09:23:05 2021 -0800
RANGER-3519: Provide an option to optimize space needed by Trie objects - Part 2
---
.../RangerFileBasedTagRetriever.java | 9 +
.../service/RangerDefaultRequestProcessor.java | 12 +
.../ranger/plugin/util/RangerCommonConstants.java | 4 +
.../plugin/util/RangerServiceTagsDeltaUtil.java | 58 ++-
.../org/apache/ranger/plugin/util/ServiceTags.java | 48 +++
distro/src/main/assembly/ranger-tools.xml | 12 +
ranger-tools/scripts/create_requests.py | 2 +-
ranger-tools/scripts/gen_service_policies.sh | 475 +++++++++++++++++++++
ranger-tools/scripts/gen_service_tags.sh | 30 +-
.../policyengine/RangerPolicyenginePerfTester.java | 1 +
.../src/test/resources/testdata/ranger-config.xml | 4 +
ranger-tools/testdata/ranger-config.xml | 4 +
.../java/org/apache/ranger/biz/TagDBStore.java | 7 +
13 files changed, 649 insertions(+), 17 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
index b858879..ab3b4a7 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
@@ -39,6 +39,7 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever {
private URL serviceTagsFileURL;
private String serviceTagsFileName;
private Gson gsonBuilder;
+ private boolean deDupTags;
@Override
public void init(Map<String, String> options) {
@@ -53,6 +54,7 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever {
String serviceTagsFileNameProperty = "serviceTagsFileName";
String serviceTagsDefaultFileName = "/testdata/test_servicetags_hive.json";
+ String deDupTagsProperty = "deDupTags";
if (StringUtils.isNotBlank(serviceName) && serviceDef != null && StringUtils.isNotBlank(appId)) {
InputStream serviceTagsFileStream = null;
@@ -61,6 +63,8 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever {
// Open specified file from options- it should contain service-tags
serviceTagsFileName = options != null? options.get(serviceTagsFileNameProperty) : null;
+ String deDupTagsVal = options != null? options.get(deDupTagsProperty) : "false";
+ deDupTags = Boolean.parseBoolean(deDupTagsVal);
serviceTagsFileName = serviceTagsFileName == null ? serviceTagsDefaultFileName : serviceTagsFileName;
@@ -137,6 +141,11 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever {
if (serviceTags.getTagVersion() <= lastKnownVersion) {
// No change in serviceTags
serviceTags = null;
+ } else {
+ if (deDupTags) {
+ final int countOfDuplicateTags = serviceTags.dedupTags();
+ LOG.info("Number of duplicate tags removed from the received serviceTags:[" + countOfDuplicateTags + "]. Number of tags in the de-duplicated serviceTags :[" + serviceTags.getTags().size() + "].");
+ }
}
} catch (IOException e) {
LOG.warn("Error processing input file: or no privilege for reading file " + serviceTagsFileName);
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerDefaultRequestProcessor.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerDefaultRequestProcessor.java
index facf05d..c2e8ae9 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerDefaultRequestProcessor.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerDefaultRequestProcessor.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.service;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
import org.apache.ranger.plugin.contextenricher.RangerContextEnricher;
import org.apache.ranger.plugin.policyengine.PolicyEngine;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
@@ -29,12 +30,15 @@ import org.apache.ranger.plugin.policyengine.RangerAccessRequestProcessor;
import org.apache.ranger.plugin.policyengine.RangerAccessResource;
import org.apache.ranger.plugin.policyengine.RangerMutableResource;
import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
+import org.apache.ranger.plugin.util.RangerPerfTracer;
import java.util.List;
import java.util.Set;
public class RangerDefaultRequestProcessor implements RangerAccessRequestProcessor {
+ private static final Log PERF_CONTEXTENRICHER_REQUEST_LOG = RangerPerfTracer.getPerfLogger("contextenricher.request");
+
protected final PolicyEngine policyEngine;
public RangerDefaultRequestProcessor(PolicyEngine policyEngine) {
@@ -89,7 +93,15 @@ public class RangerDefaultRequestProcessor implements RangerAccessRequestProcess
if (!CollectionUtils.isEmpty(enrichers)) {
for(RangerContextEnricher enricher : enrichers) {
+ RangerPerfTracer perf = null;
+
+ if(RangerPerfTracer.isPerfTraceEnabled(PERF_CONTEXTENRICHER_REQUEST_LOG)) {
+ perf = RangerPerfTracer.getPerfTracer(PERF_CONTEXTENRICHER_REQUEST_LOG, "RangerContextEnricher.enrich(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + ", enricherName=" + enricher.getName() + ")");
+ }
+
enricher.enrich(request);
+
+ RangerPerfTracer.log(perf);
}
}
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
index 63bed50..71029dd 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCommonConstants.java
@@ -36,6 +36,8 @@ public class RangerCommonConstants {
public static final String RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES = ".supports.in.place.tag.updates";
public static final String PLUGIN_CONFIG_SUFFIX_IN_PLACE_TAG_UPDATES = ".supports.in.place.tag.updates";
+ public static final String RANGER_ADMIN_SUPPORTS_TAGS_DEDUP = ".supports.tags.dedup";
+
public static final boolean RANGER_ADMIN_SUFFIX_POLICY_DELTA_DEFAULT = false;
public static final boolean PLUGIN_CONFIG_SUFFIX_POLICY_DELTA_DEFAULT = false;
@@ -48,6 +50,8 @@ public class RangerCommonConstants {
public static final boolean RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT = false;
public static final boolean PLUGIN_CONFIG_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT = false;
+ public static final boolean RANGER_ADMIN_SUPPORTS_TAGS_DEDUP_DEFAULT = false;
+
public static final boolean POLICY_REST_CLIENT_SESSION_COOKIE_ENABLED = true;
public static final String SCRIPT_OPTION_ENABLE_JSON_CTX = "enableJsonCtx";
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
index 6b70b22..088b2b8 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerServiceTagsDeltaUtil.java
@@ -22,6 +22,7 @@ package org.apache.ranger.plugin.util;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.authorization.hadoop.config.RangerAdminConfig;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.model.RangerTagDef;
@@ -30,6 +31,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.ListIterator;
import java.util.Map;
public class RangerServiceTagsDeltaUtil {
@@ -38,6 +40,9 @@ public class RangerServiceTagsDeltaUtil {
private static final Log PERF_TAGS_DELTA_LOG = RangerPerfTracer.getPerfLogger("tags.delta");
+ private static boolean SUPPORTS_TAGS_DEDUP_INITIALIZED = false;
+ private static boolean SUPPORTS_TAGS_DEDUP = false;
+
/*
It should be possible to call applyDelta() multiple times with serviceTags and delta resulting from previous call to applyDelta()
The end result should be same if called once or multiple times.
@@ -61,13 +66,33 @@ public class RangerServiceTagsDeltaUtil {
Map<Long, List<Long>> resourceToTagIds = serviceTags.getResourceToTagIds();
boolean isAnyMaterialChange = false;
- for (Map.Entry<Long, RangerTag> tagEntry : delta.getTags().entrySet()) {
- if (StringUtils.isEmpty(tagEntry.getValue().getType())) {
- if (null != tags.remove(tagEntry.getKey())) {
+ Map<Long, Long> replacedIds = new HashMap<>();
+
+ Iterator<Map.Entry<Long, RangerTag>> deltaTagIter = delta.getTags().entrySet().iterator();
+
+ while (deltaTagIter.hasNext()) {
+ Map.Entry<Long, RangerTag> entry = deltaTagIter.next();
+ Long tagId = entry.getKey();
+ RangerTag tag = entry.getValue();
+
+ if (StringUtils.isEmpty(tag.getType())) {
+ if (null != tags.remove(tagId)) {
isAnyMaterialChange = true;
+ if (isSupportsTagsDedup()) {
+ serviceTags.cachedTags.remove(tag);
+ }
}
} else {
- tags.put(tagEntry.getKey(), tagEntry.getValue());
+ if (isSupportsTagsDedup()) {
+ Long cachedTagId = serviceTags.cachedTags.get(tag);
+
+ if (cachedTagId == null) {
+ serviceTags.cachedTags.put(tag, tagId);
+ } else {
+ replacedIds.put(tagId, cachedTagId);
+ deltaTagIter.remove();
+ }
+ }
}
}
@@ -112,6 +137,21 @@ public class RangerServiceTagsDeltaUtil {
resourceToTagIds.remove(deletedServiceResourceId);
}
+ if (isSupportsTagsDedup()) {
+ for (Map.Entry<Long, List<Long>> resourceEntry : delta.getResourceToTagIds().entrySet()) {
+ ListIterator<Long> listIter = resourceEntry.getValue().listIterator();
+
+ while (listIter.hasNext()) {
+ Long tagId = listIter.next();
+ Long replacerTagId = replacedIds.get(tagId);
+
+ if (replacerTagId != null) {
+ listIter.set(replacerTagId);
+ }
+ }
+ }
+ }
+
resourceToTagIds.putAll(delta.getResourceToTagIds());
// Ensure that any modified service-resources are at head of list of service-resources in delta
@@ -174,4 +214,14 @@ public class RangerServiceTagsDeltaUtil {
}
}
}
+
+ public static boolean isSupportsTagsDedup() {
+ if (!SUPPORTS_TAGS_DEDUP_INITIALIZED) {
+ RangerAdminConfig config = RangerAdminConfig.getInstance();
+
+ SUPPORTS_TAGS_DEDUP = config.getBoolean("ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUPPORTS_TAGS_DEDUP, RangerCommonConstants.RANGER_ADMIN_SUPPORTS_TAGS_DEDUP_DEFAULT);
+ SUPPORTS_TAGS_DEDUP_INITIALIZED = true;
+ }
+ return SUPPORTS_TAGS_DEDUP;
+ }
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
index 9e8e0cf..b4f2500 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
@@ -21,7 +21,9 @@ package org.apache.ranger.plugin.util;
import java.util.Date;
+import java.util.Iterator;
import java.util.List;
+import java.util.ListIterator;
import java.util.Map;
import java.util.HashMap;
import java.util.ArrayList;
@@ -34,6 +36,7 @@ import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.model.RangerTagDef;
import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -64,6 +67,9 @@ public class ServiceTags implements java.io.Serializable {
private Boolean isDelta;
private TagsChangeExtent tagsChangeExtent;
+ @JsonIgnore
+ Map<RangerTag, Long> cachedTags = new HashMap<>();
+
public ServiceTags() {
this(OP_ADD_OR_UPDATE, null, 0L, null, null, null, null, null);
}
@@ -209,4 +215,46 @@ public class ServiceTags implements java.io.Serializable {
return sb;
}
+
+ public int dedupTags() {
+ final int ret;
+
+ Map<Long, Long> replacedIds = new HashMap<>();
+ Iterator<Map.Entry<Long, RangerTag>> iter = tags.entrySet().iterator();
+
+ final int initialTagsCount = tags.size();
+
+ while (iter.hasNext()) {
+ Map.Entry<Long, RangerTag> entry = iter.next();
+ Long tagId = entry.getKey();
+ RangerTag tag = entry.getValue();
+ Long cachedTagId = cachedTags.get(tag);
+
+ if (cachedTagId == null) {
+ cachedTags.put(tag, tagId);
+ } else {
+ replacedIds.put(tagId, cachedTagId);
+ iter.remove();
+ }
+ }
+
+ final int finalTagsCount = tags.size();
+
+ for (Map.Entry<Long, List<Long>> resourceEntry : resourceToTagIds.entrySet()) {
+ ListIterator<Long> listIter = resourceEntry.getValue().listIterator();
+
+ while (iter.hasNext()) {
+ Long tagId = listIter.next();
+ Long replacerTagId = replacedIds.get(tagId);
+
+ if (replacerTagId != null) {
+ listIter.set(replacerTagId);
+ }
+ }
+ }
+
+ ret = initialTagsCount - finalTagsCount;
+
+ return ret;
+ }
}
diff --git a/distro/src/main/assembly/ranger-tools.xml b/distro/src/main/assembly/ranger-tools.xml
index 5fdf3a5..1eb9104 100644
--- a/distro/src/main/assembly/ranger-tools.xml
+++ b/distro/src/main/assembly/ranger-tools.xml
@@ -146,5 +146,17 @@
<destName>create_requests.py</destName>
<fileMode>644</fileMode>
</file>
+ <file>
+ <source>${project.parent.basedir}/ranger-tools/scripts/gen_service_tags.sh</source>
+ <outputDirectory></outputDirectory>
+ <destName>gen_service_tags.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
+ <source>${project.parent.basedir}/ranger-tools/scripts/gen_service_policies.sh</source>
+ <outputDirectory></outputDirectory>
+ <destName>gen_service_policies.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
</files>
</assembly>
diff --git a/ranger-tools/scripts/create_requests.py b/ranger-tools/scripts/create_requests.py
index 51efc05..a333551 100755
--- a/ranger-tools/scripts/create_requests.py
+++ b/ranger-tools/scripts/create_requests.py
@@ -26,7 +26,7 @@ for i in data['serviceResources']:
resource_id = i['id']
# dictionary with table, database, or column
resource_elements = i['resourceElements']
- temp = {'name': "request-" + str(resource_id), 'request': {'resource': {'elements': {}}, 'accessType': "select", 'user': "hrt_1", 'userGroups': [], 'requestData': "request-" + str(resource_id)}, 'result': {'isAudited': 'true', 'isAllowed': 'true', 'policyId': 2}}
+ temp = {'name': "request-" + str(resource_id), 'request': {'resource': {'elements': {}}, 'accessType': "select", 'user': "hrt_1", 'userGroups': [], 'requestData': "request-" + str(resource_id)}, 'result': {'isAudited': 'true', 'isAllowed': 'false', 'policyId': resource_id}}
resource_keys = resource_elements.keys()
for resource_key in resource_keys:
diff --git a/ranger-tools/scripts/gen_service_policies.sh b/ranger-tools/scripts/gen_service_policies.sh
new file mode 100755
index 0000000..3d05d9f
--- /dev/null
+++ b/ranger-tools/scripts/gen_service_policies.sh
@@ -0,0 +1,475 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo_stderr ()
+{
+ echo "$@" >&2
+}
+
+if [ $# -ne 2 ]
+then
+ echo_stderr "usage: $0 <service_name> <num_of_resource_policies>"
+fi
+
+service_name=cm_hive
+num_of_resource_policies=1
+
+if [ $# -eq 1 ]
+then
+ service_name=$1
+ echo_stderr "service_name=${service_name}, num_of_resource_policies=${num_of_resource_policies}"
+fi
+if [ $# -eq 2 ]
+ then
+ num_of_resource_policies=$2
+ echo_stderr "service_name=${service_name}, num_of_resource_policies=${num_of_resource_policies}"
+ else
+ echo_stderr "Assuming service_name=${service_name}, num_of_resource_policies=${num_of_resource_policies}"
+fi
+
+echo "{
+ \"serviceName\": \"${service_name}\",
+ \"serviceId\": 2,
+ \"policies\": [
+ "
+for ((i = 1; i <= $num_of_resource_policies; i++)); do
+ if [ $i -ne 1 ]
+ then
+ echo " ,"
+ fi
+
+ echo " {
+ \"name\": \"${service_name}-${i}\",
+ \"id\": ${i},
+ \"isEnabled\": true,
+ \"isAuditEnabled\": true,
+ \"resources\": {
+ \"database\": { \"values\": [ \"finance_${i}\" ], \"isExcludes\": false, \"isRecursive\": false },
+ \"table\": { \"values\": [ \"tax_2020_${i}\" ], \"isExcludes\": false, \"isRecursive\": false },
+ \"column\": { \"values\": [ \"*\" ], \"isExcludes\": false, \"isRecursive\": false }
+ },
+ \"policyItems\": [],
+ \"denyPolicyItems\": [
+ { \"accesses\": [ { \"type\": \"all\", \"isAllowed\": true } ], \"users\": [ \"hrt_1\" ] }
+ ]
+ }"
+done
+echo " ],"
+echo "
+ \"serviceDef\": {
+ \"name\": \"hive\",
+ \"implClass\": \"org.apache.ranger.services.hive.RangerServiceHive\",
+ \"label\": \"Hive Server2\",
+ \"options\": {},
+ \"configs\": [
+ {
+ \"itemId\": 1,
+ \"name\": \"username\",
+ \"type\": \"string\",
+ \"mandatory\": true,
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"\",
+ \"label\": \"Username\"
+ },
+ {
+ \"itemId\": 2,
+ \"name\": \"password\",
+ \"type\": \"password\",
+ \"mandatory\": true,
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"\",
+ \"label\": \"Password\"
+ },
+ {
+ \"itemId\": 3,
+ \"name\": \"jdbc.driverClassName\",
+ \"type\": \"string\",
+ \"mandatory\": true,
+ \"defaultValue\": \"org.apache.hive.jdbc.HiveDriver\",
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"\"
+ },
+ {
+ \"itemId\": 4,
+ \"name\": \"jdbc.url\",
+ \"type\": \"string\",
+ \"mandatory\": true,
+ \"defaultValue\": \"\",
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"\"
+ },
+ {
+ \"itemId\": 5,
+ \"name\": \"commonNameForCertificate\",
+ \"type\": \"string\",
+ \"mandatory\": false,
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"\",
+ \"label\": \"Common Name for Certificate\"
+ }
+ ],
+ \"resources\": [
+ {
+ \"itemId\": 1,
+ \"name\": \"database\",
+ \"type\": \"string\",
+ \"level\": 10,
+ \"mandatory\": true,
+ \"lookupSupported\": true,
+ \"recursiveSupported\": false,
+ \"excludesSupported\": true,
+ \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher\",
+ \"matcherOptions\": {
+ \"wildCard\": \"true\",
+ \"ignoreCase\": \"true\"
+ },
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"\",
+ \"label\": \"Hive Database\"
+ },
+ {
+ \"itemId\": 2,
+ \"name\": \"table\",
+ \"type\": \"string\",
+ \"level\": 20,
+ \"parent\": \"database\",
+ \"mandatory\": true,
+ \"lookupSupported\": true,
+ \"recursiveSupported\": false,
+ \"excludesSupported\": true,
+ \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher\",
+ \"matcherOptions\": {
+ \"wildCard\": \"true\",
+ \"ignoreCase\": \"true\"
+ },
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"\",
+ \"label\": \"Hive Table\"
+ },
+ {
+ \"itemId\": 3,
+ \"name\": \"udf\",
+ \"type\": \"string\",
+ \"level\": 20,
+ \"parent\": \"database\",
+ \"mandatory\": true,
+ \"lookupSupported\": true,
+ \"recursiveSupported\": false,
+ \"excludesSupported\": true,
+ \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher\",
+ \"matcherOptions\": {
+ \"wildCard\": \"true\",
+ \"ignoreCase\": \"true\"
+ },
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"\",
+ \"label\": \"Hive UDF\"
+ },
+ {
+ \"itemId\": 4,
+ \"name\": \"column\",
+ \"type\": \"string\",
+ \"level\": 30,
+ \"parent\": \"table\",
+ \"mandatory\": true,
+ \"lookupSupported\": true,
+ \"recursiveSupported\": false,
+ \"excludesSupported\": true,
+ \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher\",
+ \"matcherOptions\": {
+ \"wildCard\": \"true\",
+ \"ignoreCase\": \"true\"
+ },
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"\",
+ \"label\": \"Hive Column\"
+ }
+ ],
+ \"accessTypes\": [
+ {
+ \"itemId\": 1,
+ \"name\": \"select\",
+ \"label\": \"select\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 2,
+ \"name\": \"update\",
+ \"label\": \"update\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 3,
+ \"name\": \"create\",
+ \"label\": \"Create\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 4,
+ \"name\": \"drop\",
+ \"label\": \"Drop\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 5,
+ \"name\": \"alter\",
+ \"label\": \"Alter\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 6,
+ \"name\": \"index\",
+ \"label\": \"Index\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 7,
+ \"name\": \"lock\",
+ \"label\": \"Lock\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 8,
+ \"name\": \"all\",
+ \"label\": \"All\",
+ \"impliedGrants\": [
+ \"select\",
+ \"update\",
+ \"create\",
+ \"drop\",
+ \"alter\",
+ \"index\",
+ \"lock\"
+ ]
+ }
+ ],
+ \"policyConditions\": [
+ {
+ \"itemId\": 1,
+ \"name\": \"resources-accessed-together\",
+ \"evaluator\": \"org.apache.ranger.plugin.conditionevaluator.RangerHiveResourcesAccessedTogetherCondition\",
+ \"evaluatorOptions\": {},
+ \"label\": \"Hive Resources Accessed Together?\"
+ }
+ ],
+ \"contextEnrichers\": [],
+ \"enums\": [],
+ \"id\": 3,
+ \"isEnabled\": true
+ },
+ \"tagPolicies\": {
+ \"serviceName\": \"tagdev\",
+ \"serviceId\": 3,
+ \"policyVersion\": 1,
+ \"policies\": [
+ {
+ \"service\": \"tagdev\",
+ \"name\": \"tagdev-EXPIRES_ON\",
+ \"isAuditEnabled\": true,
+ \"resources\": {
+ \"tag\": {
+ \"values\": [
+ \"EXPIRES_ON\"
+ ],
+ \"isExcludes\": false,
+ \"isRecursive\": false
+ }
+ },
+ \"policyItems\": [],
+ \"denyPolicyItems\": [
+ {
+ \"accesses\": [
+ {
+ \"type\": \"hive:select\",
+ \"isAllowed\": true
+ },
+ {
+ \"type\": \"hive:update\",
+ \"isAllowed\": true
+ },
+ {
+ \"type\": \"hive:create\",
+ \"isAllowed\": true
+ },
+ {
+ \"type\": \"hive:drop\",
+ \"isAllowed\": true
+ },
+ {
+ \"type\": \"hive:alter\",
+ \"isAllowed\": true
+ },
+ {
+ \"type\": \"hive:index\",
+ \"isAllowed\": true
+ },
+ {
+ \"type\": \"hive:lock\",
+ \"isAllowed\": true
+ },
+ {
+ \"type\": \"hive:all\",
+ \"isAllowed\": true
+ }
+ ],
+ \"users\": [],
+ \"groups\": [
+ \"public\"
+ ],
+ \"conditions\": [
+ {
+ \"type\": \"accessed-after-expiry\",
+ \"values\": [
+ \"yes\"
+ ]
+ }
+ ],
+ \"isEnabled\": true
+ }
+ ],
+ \"allowExceptions\": [],
+ \"denyExceptions\": [],
+ \"id\": 4,
+ \"isEnabled\": true
+ }
+ ],
+ \"serviceDef\": {
+ \"name\": \"tag\",
+ \"implClass\": \"org.apache.ranger.services.tag.RangerServiceTag\",
+ \"label\": \"TAG\",
+ \"options\": {
+ \"ui.pages\": \"tag-based-policies\"
+ },
+ \"configs\": [],
+ \"resources\": [
+ {
+ \"itemId\": 1,
+ \"name\": \"tag\",
+ \"type\": \"string\",
+ \"level\": 1,
+ \"mandatory\": true,
+ \"lookupSupported\": true,
+ \"recursiveSupported\": false,
+ \"excludesSupported\": false,
+ \"matcher\": \"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher\",
+ \"matcherOptions\": {
+ \"wildCard\": \"false\",
+ \"ignoreCase\": \"false\"
+ },
+ \"validationRegEx\": \"\",
+ \"validationMessage\": \"\",
+ \"uiHint\": \"{ \\\"singleValue\\\":true }\",
+ \"label\": \"TAG\"
+ }
+ ],
+ \"accessTypes\": [
+ {
+ \"itemId\": 3004,
+ \"name\": \"hive:select\",
+ \"label\": \"select\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 3005,
+ \"name\": \"hive:update\",
+ \"label\": \"update\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 3006,
+ \"name\": \"hive:create\",
+ \"label\": \"Create\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 3007,
+ \"name\": \"hive:drop\",
+ \"label\": \"Drop\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 3008,
+ \"name\": \"hive:alter\",
+ \"label\": \"Alter\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 3009,
+ \"name\": \"hive:index\",
+ \"label\": \"Index\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 3010,
+ \"name\": \"hive:lock\",
+ \"label\": \"Lock\",
+ \"impliedGrants\": []
+ },
+ {
+ \"itemId\": 3011,
+ \"name\": \"hive:all\",
+ \"label\": \"All\",
+ \"impliedGrants\": [
+ \"hive:select\",
+ \"hive:update\",
+ \"hive:create\",
+ \"hive:drop\",
+ \"hive:alter\",
+ \"hive:index\",
+ \"hive:lock\"
+ ]
+ }
+ ],
+ \"policyConditions\": [
+ {
+ \"itemId\": 1,
+ \"name\": \"accessed-after-expiry\",
+ \"evaluator\": \"org.apache.ranger.plugin.conditionevaluator.RangerScriptTemplateConditionEvaluator\",
+ \"evaluatorOptions\": {
+ \"scriptTemplate\": \"ctx.isAccessedAfter(\u0027expiry_date\u0027);\"
+ },
+ \"uiHint\": \"{ \\\"singleValue\\\":true }\",
+ \"label\": \"Accessed after expiry_date (yes/no)?\"
+ }
+ ],
+ \"contextEnrichers\": [
+ {
+ \"itemId\": 1,
+ \"name\": \"TagEnricher\",
+ \"enricher\": \"org.apache.ranger.plugin.contextenricher.RangerTagEnricher\",
+ \"enricherOptions\": {
+ \"tagRetrieverClassName\": \"org.apache.ranger.plugin.contextenricher.RangerFileBasedTagRetriever\",
+ \"tagRefresherPollingInterval\": \"60000\",
+ \"serviceTagsFileName\":\"/testdata/test_servicetags_hive.json\"
+ }
+ }
+ ],
+ \"enums\": [],
+ \"id\": 100,
+ \"isEnabled\": true
+ }
+ }
+}"
+
diff --git a/ranger-tools/scripts/gen_service_tags.sh b/ranger-tools/scripts/gen_service_tags.sh
index dd3ee88..9a81a0a 100755
--- a/ranger-tools/scripts/gen_service_tags.sh
+++ b/ranger-tools/scripts/gen_service_tags.sh
@@ -64,17 +64,23 @@ echo "{
\"isEnabled\": true
}
},
- \"tags\": {
- \"1\": {
- \"type\": \"EXPIRES_ON\",
- \"attributes\": {
- \"expiry_date\": \"2027/12/31\",
- \"activates_on\": \"2020/01/01\"
- },
- \"id\": 1,
- \"isEnabled\": true
- }
- },"
+ \"tags\": {"
+for ((i = 1; i <= $num_of_service_resources; i++)); do
+ if [ $i -ne 1 ]
+ then
+ echo " ,"
+ fi
+ echo " \"${i}\": {
+ \"type\": \"EXPIRES_ON\",
+ \"attributes\": {
+ \"expiry_date\": \"2027/12/31\",
+ \"activates_on\": \"2020/01/01\"
+ },
+ \"id\": ${i},
+ \"isEnabled\": true
+ }"
+done
+ echo " },"
echo " \"serviceResources\": ["
for ((i = 1; i <= $num_of_service_resources; i++)); do
if [ $i -ne 1 ]
@@ -96,7 +102,7 @@ for ((i = 1; i <= $num_of_service_resources; i++)); do
then
echo " ,"
fi
- echo " \"${i}\": [ 1 ]"
+ echo " \"${i}\": [ ${i} ]"
done
echo " }
}"
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
index f667628..61526f0 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
@@ -71,6 +71,7 @@ public class RangerPolicyenginePerfTester {
RangerConfiguration configuration = new PerfTestConfiguration(configurationFileURL);
policyEngineOptions.optimizeTrieForSpace = configuration.getBoolean("ranger.policyengine.option.optimize.policy.trie.for.space", false);
policyEngineOptions.optimizeTagTrieForSpace = configuration.getBoolean("ranger.policyengine.option.optimize.tag.trie.for.space", false);
+ policyEngineOptions.optimizeTagTrieForRetrieval = configuration.getBoolean("ranger.policyengine.option.optimize.tag.trie.for.retrieval", false);
PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions, configurationFileURL);
if (!perfTestEngine.init()) {
diff --git a/ranger-tools/src/test/resources/testdata/ranger-config.xml b/ranger-tools/src/test/resources/testdata/ranger-config.xml
index 218db07..186a52e 100644
--- a/ranger-tools/src/test/resources/testdata/ranger-config.xml
+++ b/ranger-tools/src/test/resources/testdata/ranger-config.xml
@@ -16,6 +16,10 @@
<value>1</value>
</property>
<property>
+ <name>ranger.policyengine.option.optimize.tag.trie.for.retrieval</name>
+ <value>false</value>
+ </property>
+ <property>
<name>ranger.policyengine.option.optimize.policy.trie.for.space</name>
<value>true</value>
</property>
diff --git a/ranger-tools/testdata/ranger-config.xml b/ranger-tools/testdata/ranger-config.xml
index 08a83b9..35d0568 100644
--- a/ranger-tools/testdata/ranger-config.xml
+++ b/ranger-tools/testdata/ranger-config.xml
@@ -16,6 +16,10 @@
<value>1</value>
</property>
<property>
+ <name>ranger.policyengine.option.optimize.tag.trie.for.retrieval</name>
+ <value>false</value>
+ </property>
+ <property>
<name>ranger.policyengine.option.optimize.policy.trie.for.space</name>
<value>true</value>
</property>
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index 1bdfce0..085d73d 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -1016,6 +1016,13 @@ public class TagDBStore extends AbstractTagStore {
ret.setTags(tagMap);
ret.setServiceResources(resources);
ret.setResourceToTagIds(resourceToTagIds);
+
+ if (RangerServiceTagsDeltaUtil.isSupportsTagsDedup()) {
+ final int countOfDuplicateTags = ret.dedupTags();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Number of duplicate tags removed from the received serviceTags:[" + countOfDuplicateTags + "]. Number of tags in the de-duplicated serviceTags :[" + ret.getTags().size() + "].");
+ }
+ }
}
if (LOG.isDebugEnabled()) {