You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2024/10/18 00:13:57 UTC
(ranger) branch ranger-2.6 updated (5c413ebf0 -> 27fdfeace)
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a change to branch ranger-2.6
in repository https://gitbox.apache.org/repos/asf/ranger.git
from 5c413ebf0 RANGER-3801: update docker download-archives.sh to download all by default
new c16dd8aaa RANGER-4951: Prune docker cache before docker-build (#404)
new 27fdfeace RANGER-4956: fixed dedup tags to address potential infinite loop
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/maven.yml | 3 +
.../org/apache/ranger/plugin/util/ServiceTags.java | 25 ++--
.../apache/ranger/plugin/util/TestServiceTags.java | 144 +++++++++++++++++++++
3 files changed, 163 insertions(+), 9 deletions(-)
create mode 100644 agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceTags.java
(ranger) 02/02: RANGER-4956: fixed dedup tags to address potential infinite loop
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.6
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 27fdfeace4bf79498113c545c92c1fee24d5f7d2
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Wed Oct 16 16:12:16 2024 -0700
RANGER-4956: fixed dedup tags to address potential infinite loop
(cherry picked from commit 16c94a96407b96939c057c098c0633f204697830)
---
.../org/apache/ranger/plugin/util/ServiceTags.java | 25 ++--
.../apache/ranger/plugin/util/TestServiceTags.java | 144 +++++++++++++++++++++
2 files changed, 160 insertions(+), 9 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
index cc2ebe53a..bca352e8b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
@@ -258,8 +258,8 @@ public class ServiceTags implements java.io.Serializable {
MutablePair<Long, Long> cachedTag = cachedTags.get(tag);
if (cachedTag == null) {
- cachedTags.put(tag, new MutablePair<>(tagId, 0L)); // reference count will be incremented later
- } else {
+ cachedTags.put(tag, new MutablePair<>(tagId, 1L));
+ } else if (!tagId.equals(cachedTag.left)) {
if (tagId < cachedTag.left) {
replacedIds.put(cachedTag.left, tagId);
tagIdsToRemove.add(cachedTag.left);
@@ -270,6 +270,7 @@ public class ServiceTags implements java.io.Serializable {
}
}
}
+
for (Long tagIdToRemove : tagIdsToRemove) {
tags.remove(tagIdToRemove);
}
@@ -278,22 +279,28 @@ public class ServiceTags implements java.io.Serializable {
for (Map.Entry<Long, List<Long>> resourceEntry : resourceToTagIds.entrySet()) {
for (ListIterator<Long> listIter = resourceEntry.getValue().listIterator(); listIter.hasNext(); ) {
- Long tagId = listIter.next();
+ final Long tagId = listIter.next();
+ Long mappedTagId = null;
+
+ for (Long replacerTagId = replacedIds.get(tagId); replacerTagId != null; replacerTagId = replacedIds.get(mappedTagId)) {
+ mappedTagId = replacerTagId;
+ }
- for (Long replacerTagId = replacedIds.get(tagId); replacerTagId != null; replacerTagId = replacedIds.get(tagId)) {
- tagId = replacerTagId;
+ if (mappedTagId == null) {
+ continue;
}
- listIter.set(tagId);
+ listIter.set(mappedTagId);
- RangerTag tag = tags.get(tagId);
- if (tag != null) { // This should always be true
+ RangerTag tag = tags.get(mappedTagId);
+
+ if (tag != null) { // This should always be true
MutablePair<Long, Long> cachedTag = cachedTags.get(tag);
+
if (cachedTag != null) { // This should always be true
cachedTag.right++;
}
}
-
}
}
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceTags.java b/agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceTags.java
new file mode 100644
index 000000000..c41d4e196
--- /dev/null
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceTags.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.plugin.util;
+
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
+import org.apache.ranger.plugin.model.RangerServiceResource;
+import org.apache.ranger.plugin.model.RangerTag;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestServiceTags {
+ private static final RangerServiceResource[] RESOURCES = {
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db1"))),
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db2"))),
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db3"))),
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db4"))),
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db5"))),
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db6"))),
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db7"))),
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db8"))),
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db9"))),
+ new RangerServiceResource("dev_hive", Collections.singletonMap("database", new RangerPolicyResource("db10")))
+ };
+
+ @Test
+ public void testDedupEmpty() {
+ ServiceTags svcTags = new ServiceTags();
+
+ assertEquals(0, svcTags.dedupTags());
+ }
+
+ @Test
+ public void testDedup_NoDupTagsNoAttr() {
+ RangerTag[] tags = {
+ new RangerTag("PII", Collections.emptyMap()),
+ new RangerTag("PHI", Collections.emptyMap()),
+ };
+
+ ServiceTags svcTags = createServiceTags(tags, RESOURCES);
+
+ assertEquals(0, svcTags.dedupTags());
+ }
+
+ @Test
+ public void testDedup_NoDupTagsWithAttr() {
+ RangerTag[] tags = {
+ new RangerTag("PII", Collections.emptyMap()),
+ new RangerTag("PII", Collections.singletonMap("type", "email")),
+ };
+
+ ServiceTags svcTags = createServiceTags(tags, RESOURCES);
+
+ assertEquals(0, svcTags.dedupTags());
+ }
+
+ @Test
+ public void testDedup_DupTagsNoAttr() {
+ RangerTag[] tags = {
+ new RangerTag("PII", Collections.emptyMap()),
+ new RangerTag("PII", Collections.emptyMap()),
+ };
+
+ ServiceTags svcTags = createServiceTags(tags, RESOURCES);
+
+ assertEquals(1, svcTags.dedupTags());
+ }
+
+ @Test
+ public void testDedup_DupTagsWithAttr() {
+ RangerTag[] tags = {
+ new RangerTag("PII", Collections.singletonMap("type", "email")),
+ new RangerTag("PII", Collections.singletonMap("type", "email")),
+ };
+
+ ServiceTags svcTags = createServiceTags(tags, RESOURCES);
+
+ assertEquals(1, svcTags.dedupTags());
+ }
+
+ @Test
+ public void testDedup_DupTagsWithAttr_MultipleCalls() {
+ RangerTag[] tags = {
+ new RangerTag("PII", Collections.singletonMap("type", "email")),
+ new RangerTag("PII", Collections.singletonMap("type", "email")),
+ new RangerTag("PCI", Collections.emptyMap()),
+ new RangerTag("PCI", Collections.emptyMap())
+ };
+
+ ServiceTags svcTags = createServiceTags(tags, RESOURCES);
+
+ assertEquals(2, svcTags.dedupTags());
+ assertEquals(0, svcTags.dedupTags());
+ }
+
+
+ private ServiceTags createServiceTags(RangerTag[] tags, RangerServiceResource[] resources) {
+ ServiceTags ret = new ServiceTags();
+
+ // add tags
+ for (int i = 0; i < tags.length; i++) {
+ RangerTag tag = tags[i];
+
+ tag.setId(i + 1L);
+ ret.getTags().put(tag.getId(), tag);
+ }
+
+ // add resources
+ for (int i = 0; i < resources.length; i++) {
+ RangerServiceResource resource = resources[i];
+
+ resource.setId(i + 1L);
+ ret.getServiceResources().add(resource);
+ }
+
+ // set resourceToTagIds
+ ret.getServiceResources().forEach(resource -> {
+ List<Long> tagIds = new ArrayList<>(ret.getTags().keySet()); // add all available tags
+
+ ret.getResourceToTagIds().put(resource.getId(), tagIds);
+ });
+
+ return ret;
+ }
+}
(ranger) 01/02: RANGER-4951: Prune docker cache before docker-build (#404)
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.6
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit c16dd8aaa8bf8cfe52ca3e13bc91980a27452b2d
Author: Abhishek Kumar <ab...@gmail.com>
AuthorDate: Sun Oct 13 23:02:16 2024 -0700
RANGER-4951: Prune docker cache before docker-build (#404)
(cherry picked from commit 8d89c91360b619071cf347745f766c963da58f7d)
---
.github/workflows/maven.yml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index e8d3191e6..7aa7d6e52 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -92,6 +92,9 @@ jobs:
cd dev-support/ranger-docker
./download-archives.sh hadoop hive hbase kafka knox ozone
+ - name: Clean up Docker space
+ run: docker system prune --all --force --volumes
+
- name: Build all ranger-service images
run: |
cd dev-support/ranger-docker