You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by sp...@apache.org on 2017/12/12 20:36:42 UTC
sentry git commit: SENTRY-1951: Old
SentryStore.retrieveFullPathsImage() should be removed (Arjun Mishra,
reviewed by Alexander Kolbasov, Vadim Spector)
Repository: sentry
Updated Branches:
refs/heads/master 129f4ed9a -> 71283bda2
SENTRY-1951: Old SentryStore.retrieveFullPathsImage() should be removed (Arjun Mishra, reviewed by Alexander Kolbasov, Vadim Spector)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/71283bda
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/71283bda
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/71283bda
Branch: refs/heads/master
Commit: 71283bda28927e59ab7d85f3e081776c190db86a
Parents: 129f4ed
Author: Sergio Pena <se...@cloudera.com>
Authored: Tue Dec 12 14:36:04 2017 -0600
Committer: Sergio Pena <se...@cloudera.com>
Committed: Tue Dec 12 14:36:04 2017 -0600
----------------------------------------------------------------------
.../apache/sentry/hdfs/TestImageRetriever.java | 5 +-
.../hdfs/TestSentryHDFSServiceProcessor.java | 11 +-
.../db/service/persistent/SentryStore.java | 61 ----
.../db/service/persistent/TestSentryStore.java | 284 ++++++++++++++++---
4 files changed, 250 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/71283bda/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java
index b355630..d2d5391 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestImageRetriever.java
@@ -20,7 +20,6 @@ package org.apache.sentry.hdfs;
import com.google.common.collect.Sets;
import org.apache.commons.lang.StringUtils;
import org.apache.sentry.hdfs.service.thrift.TPathChanges;
-import org.apache.sentry.provider.db.service.persistent.PathsImage;
import org.apache.sentry.provider.db.service.persistent.SentryStore;
import org.junit.Before;
import org.junit.Ignore;
@@ -54,8 +53,8 @@ public class TestImageRetriever {
fullPathsImage.put("db1", Sets.newHashSet("/user/db1"));
fullPathsImage.put("db1.table1", Sets.newHashSet("/user/db1/table1"));
- Mockito.when(sentryStoreMock.retrieveFullPathsImage())
- .thenReturn(new PathsImage(fullPathsImage, 1, 1));
+ Mockito.when(sentryStoreMock.retrieveFullPathsImageUpdate(root))
+ .thenReturn(new PathsUpdate(1, 1, true));
imageRetriever = new PathImageRetriever(sentryStoreMock, root);
pathsUpdate = imageRetriever.retrieveFullImage();
http://git-wip-us.apache.org/repos/asf/sentry/blob/71283bda/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java
index f09d1b2..578757e 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java
@@ -25,7 +25,6 @@ import org.apache.sentry.hdfs.service.thrift.TAuthzUpdateResponse;
import org.apache.sentry.provider.db.SentryPolicyStorePlugin;
import org.apache.sentry.provider.db.service.model.MSentryPathChange;
import org.apache.sentry.provider.db.service.model.MSentryPermChange;
-import org.apache.sentry.provider.db.service.persistent.PathsImage;
import org.apache.sentry.provider.db.service.persistent.PermissionsImage;
import org.apache.sentry.provider.db.service.persistent.SentryStore;
import org.junit.BeforeClass;
@@ -58,8 +57,9 @@ public class TestSentryHDFSServiceProcessor {
public void testInitialHDFSSyncReturnsAFullImage() throws Exception {
Mockito.when(sentryStoreMock.getLastProcessedImageID())
.thenReturn(1L);
- Mockito.when(sentryStoreMock.retrieveFullPathsImage())
- .thenReturn(new PathsImage(new HashMap<String, Collection<String>>(), 1, 1));
+ String[]prefixes = {"/"};
+ Mockito.when(sentryStoreMock.retrieveFullPathsImageUpdate(prefixes))
+ .thenReturn(new PathsUpdate(1, 1, true));
Mockito.when(sentryStoreMock.getLastProcessedPermChangeID())
.thenReturn(1L);
@@ -84,8 +84,9 @@ public class TestSentryHDFSServiceProcessor {
public void testRequestSyncUpdatesWhenNewImagesArePersistedReturnsANewFullImage() throws Exception {
Mockito.when(sentryStoreMock.getLastProcessedImageID())
.thenReturn(2L);
- Mockito.when(sentryStoreMock.retrieveFullPathsImage())
- .thenReturn(new PathsImage(new HashMap<String, Collection<String>>(), 3, 2));
+ String[]prefixes = {"/"};
+ Mockito.when(sentryStoreMock.retrieveFullPathsImageUpdate(prefixes))
+ .thenReturn(new PathsUpdate(3, 2, true));
Mockito.when(sentryStoreMock.getLastProcessedPermChangeID())
.thenReturn(3L);
http://git-wip-us.apache.org/repos/asf/sentry/blob/71283bda/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index c730a03..6c4631f 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -2670,37 +2670,6 @@ public class SentryStore {
}
/**
- * Retrieves an up-to-date hive paths snapshot. <p>
- * It reads hiveObj to paths mapping from {@link MAuthzPathsMapping} table and
- * gets the changeID of latest delta update, from {@link MSentryPathChange}, that
- * the snapshot corresponds to.
- *
- * NOTE: this method used to be used in the actual code but is now only used for tests
- * which should be refactored to use the new {@link #retrieveFullPathsImageUpdate} functionality.
- *
- * TODO: Remove retrieveFullPathsImage method and reimplement tests.
- *
- * @return an up-to-date hive paths snapshot contains mapping of hiveObj to < Paths >.
- * For empty image return {@link #EMPTY_CHANGE_ID} and a empty map.
- * @throws Exception
- */
- public PathsImage retrieveFullPathsImage() throws Exception {
- return (PathsImage) tm.executeTransaction(
- new TransactionBlock() {
- public Object execute(PersistenceManager pm) throws Exception {
- // curChangeID could be 0 for the first full snapshot fetching
- // from HMS. It does not have corresponding delta update.
- pm.setDetachAllOnCommit(false); // No need to detach objects
- long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPathChange.class);
- long curImageID = getCurrentAuthzPathsSnapshotID(pm);
- Map<String, Collection<String>> pathImage = retrieveFullPathsImageCore(pm, curImageID);
-
- return new PathsImage(pathImage, curChangeID, curImageID);
- }
- });
- }
-
- /**
* Retrieves an up-to-date hive paths snapshot.
* The image only contains PathsDump in it.
* <p>
@@ -2732,36 +2701,6 @@ public class SentryStore {
}
/**
- * Retrieves an up-to-date hive paths snapshot from {@code MAuthzPathsMapping} table.
- * The snapshot is represented by a snapshot ID, and a map from hiveObj to paths.
- *
- * @return a mapping of hiveObj to < Paths >.
- */
- private Map<String, Collection<String>> retrieveFullPathsImageCore(PersistenceManager pm,
- long currentSnapshotID) {
- if (currentSnapshotID <= EMPTY_PATHS_SNAPSHOT_ID) {
- return Collections.emptyMap();
- }
-
- Query query = pm.newQuery(MAuthzPathsMapping.class);
- query.setFilter("this.authzSnapshotID == currentSnapshotID");
- query.declareParameters("long currentSnapshotID");
- Collection<MAuthzPathsMapping> authzToPathsMappings =
- (Collection<MAuthzPathsMapping>) query.execute(currentSnapshotID);
-
- if (authzToPathsMappings.isEmpty()) {
- return Collections.emptyMap();
- }
-
- Map<String, Collection<String>> retVal = new HashMap<>(authzToPathsMappings.size());
- for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) {
- retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPathStrings());
- }
-
- return retVal;
- }
-
- /**
* Extract all paths and convert them into HMSPaths obect
* @param pm Persistence manager
* @param currentSnapshotID Image ID we are interested in
http://git-wip-us.apache.org/repos/asf/sentry/blob/71283bda/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index 24b11f7..b410027 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -25,6 +25,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -2451,36 +2452,48 @@ public class TestSentryStore extends org.junit.Assert {
@Test
public void testPersistFullPathsImage() throws Exception {
Map<String, Collection<String>> authzPaths = new HashMap<>();
+ String[] prefixes = {"/user/hive/warehouse"};
// Makes sure that authorizable object could be associated
// with different paths and can be properly persisted into database.
authzPaths.put("db1.table1", Sets.newHashSet("/user/hive/warehouse/db2.db/table1.1",
- "/user/hive/warehouse/db2.db/table1.2"));
+ "/user/hive/warehouse/db2.db/table1.2"));
// Makes sure that the same MPaths object could be associated
// with multiple authorizable and can be properly persisted into database.
authzPaths.put("db1.table2", Sets.newHashSet("/user/hive/warehouse/db2.db/table2.1",
- "/user/hive/warehouse/db2.db/table2.2"));
+ "/user/hive/warehouse/db2.db/table2.2"));
authzPaths.put("db2.table2", Sets.newHashSet("/user/hive/warehouse/db2.db/table2.1",
- "/user/hive/warehouse/db2.db/table2.3"));
+ "/user/hive/warehouse/db2.db/table2.3"));
long notificationID = 11;
sentryStore.persistFullPathsImage(authzPaths, notificationID);
- PathsImage pathsImage = sentryStore.retrieveFullPathsImage();
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
long savedNotificationID = sentryStore.getLastProcessedNotificationID();
- assertEquals(1, pathsImage.getCurImgNum());
- Map<String, Collection<String>> pathImage = pathsImage.getPathImage();
+ assertEquals(1, pathsUpdate.getImgNum());
+ TPathsDump pathDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry> nodeMap = pathDump.getNodeMap();
+
+ assertEquals(10, nodeMap.size());
+ //First element is "/"
+ int rootId = pathDump.getRootId();
+ TPathEntry root = nodeMap.get(rootId);
+ assertEquals("/", root.getPathElement());
+
+ Map<String, Collection<String>>pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, true);
+
assertEquals(3, pathImage.size());
for (Map.Entry<String, Collection<String>> entry : pathImage.entrySet()) {
assertEquals(2, entry.getValue().size());
}
assertEquals(2, pathImage.get("db2.table2").size());
assertTrue(CollectionUtils.isEqualCollection(Lists.newArrayList("/user/hive/warehouse/db2.db/table1.1",
- "/user/hive/warehouse/db2.db/table1.2"),
- pathImage.get("db1.table1")));
+ "/user/hive/warehouse/db2.db/table1.2"),
+ pathImage.get("db1.table1")));
assertTrue(CollectionUtils.isEqualCollection(Lists.newArrayList("/user/hive/warehouse/db2.db/table2.1",
- "/user/hive/warehouse/db2.db/table2.2"),
- pathImage.get("db1.table2")));
+ "/user/hive/warehouse/db2.db/table2.2"),
+ pathImage.get("db1.table2")));
assertTrue(CollectionUtils.isEqualCollection(Lists.newArrayList("/user/hive/warehouse/db2.db/table2.1",
- "/user/hive/warehouse/db2.db/table2.3"),
- pathImage.get("db2.table2")));
+ "/user/hive/warehouse/db2.db/table2.3"),
+ pathImage.get("db2.table2")));
assertEquals(6, sentryStore.getMPaths().size());
assertEquals(notificationID, savedNotificationID);
}
@@ -2509,8 +2522,19 @@ public class TestSentryStore extends org.junit.Assert {
long latestID = sentryStore.getLastProcessedNotificationID();
assertEquals(notificationID, latestID);
+ String []prefixes = {"/hive"};
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ TPathsDump pathDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry> nodeMap = pathDump.getNodeMap();
+ assertEquals(4, nodeMap.size());
+ int rootId = pathDump.getRootId();
+ TPathEntry root = nodeMap.get(rootId);
+ assertEquals("/", root.getPathElement());
+
+ Map<String, Collection<String>>pathsImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathsImage, true);
+
// Check that retrieving a full paths image returns both paths updates
- Map<String, Collection<String>> pathsImage = sentryStore.retrieveFullPathsImage().getPathImage();
assertEquals(2, pathsImage.size());
assertEquals(1, pathsImage.get("db1").size());
assertTrue(pathsImage.get("db1").contains("/hive/db1"));
@@ -2530,6 +2554,28 @@ public class TestSentryStore extends org.junit.Assert {
assertEquals("u2", pathsChanges.get(1).getNotificationHash());
}
+ private void buildPathsImageMap(Map<Integer, TPathEntry> nodeMap, TPathEntry node, String path,
+ Map<String, Collection<String>> pathsImage, boolean includeLeadingSlash) {
+ path = path.isEmpty()?node.getPathElement(): (path + node.getPathElement() + "/");
+ TPathEntry tempNode = node;
+ for(int childVal:node.getChildren()) {
+ node = nodeMap.get(childVal);
+ buildPathsImageMap(nodeMap, node, path, pathsImage, includeLeadingSlash);
+ }
+ if(tempNode.getChildren().isEmpty() && node.getAuthzObjs() != null) {
+ for (String authz : node.getAuthzObjs()) {
+ if (!pathsImage.containsKey(authz)) {
+ pathsImage.put(authz, new LinkedList<>());
+ }
+ if(includeLeadingSlash) {
+ pathsImage.get(authz).add(path.replaceAll("(/+$)", ""));
+ } else {
+ pathsImage.get(authz).add(path.replaceAll("(^/+)|(/+$)", ""));
+ }
+ }
+ }
+ }
+
@Test
public void testPersistDuplicatedNotificationIdShouldBeAllowed() throws Exception {
long notificationID = 1;
@@ -2560,8 +2606,16 @@ public class TestSentryStore extends org.junit.Assert {
sentryStore.addAuthzPathsMapping("db1.table",
Sets.newHashSet("db1/tbl1", "db1/tbl2"), addUpdate);
- PathsImage pathsImage = sentryStore.retrieveFullPathsImage();
- Map<String, Collection<String>> pathImage = pathsImage.getPathImage();
+ String[]prefixes = {"/"};
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap();
+ TPathEntry root = nodeMap.get(pathsDump.getRootId());
+ Map<String, Collection<String>> pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, false);
+
+ assertEquals(4, nodeMap.size());//Tree size
+ assertEquals("/", root.getPathElement());
assertEquals(1, pathImage.size());
assertEquals(2, pathImage.get("db1.table").size());
assertEquals(2, sentryStore.getMPaths().size());
@@ -2578,7 +2632,15 @@ public class TestSentryStore extends org.junit.Assert {
delUpdate.newPathChange("db1.table")
.addToDelPaths(Arrays.asList("db1", "tbl1"));
sentryStore.deleteAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1"), delUpdate);
- pathImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(3, nodeMap.size());//Tree size
assertEquals(1, pathImage.size());
assertEquals(1, pathImage.get("db1.table").size());
assertEquals(1, sentryStore.getMPaths().size());
@@ -2595,7 +2657,15 @@ public class TestSentryStore extends org.junit.Assert {
delAllupdate.newPathChange("db1.table")
.addToDelPaths(Lists.newArrayList(PathsUpdate.ALL_PATHS));
sentryStore.deleteAllAuthzPathsMapping("db1.table", delAllupdate);
- pathImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(1, nodeMap.size());//Tree size
assertEquals(0, pathImage.size());
assertEquals(0, sentryStore.getMPaths().size());
@@ -2617,7 +2687,16 @@ public class TestSentryStore extends org.junit.Assert {
"user/hive/warehouse/db1.db/table1/p1"));
authzPaths.put("db1.table2", Sets.newHashSet("user/hive/warehouse/db1.db/table2"));
sentryStore.persistFullPathsImage(authzPaths, lastNotificationId);
- Map<String, Collection<String>> pathsImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ String[]prefixes = {"/user/hive/warehouse"};
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap();
+ TPathEntry root = nodeMap.get(pathsDump.getRootId());
+ Map<String, Collection<String>> pathsImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathsImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(8, nodeMap.size());//Tree size
assertEquals(2, pathsImage.size());
@@ -2629,7 +2708,15 @@ public class TestSentryStore extends org.junit.Assert {
.addToAddPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1"));
sentryStore.renameAuthzPathsMapping("db1.table1", "db1.newTable1",
"user/hive/warehouse/db1.db/table1", "user/hive/warehouse/db1.db/newTable1", renameUpdate);
- pathsImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathsImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathsImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(9, nodeMap.size());//Tree size
assertEquals(2, pathsImage.size());
assertEquals(3, sentryStore.getMPaths().size());
assertTrue(pathsImage.containsKey("db1.newTable1"));
@@ -2650,7 +2737,15 @@ public class TestSentryStore extends org.junit.Assert {
renameUpdate.newPathChange("db1.newTable2")
.addToAddPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1"));
sentryStore.renameAuthzObj("db1.newTable1", "db1.newTable2", renameUpdate);
- pathsImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathsImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathsImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(9, nodeMap.size());//Tree size
assertEquals(2, pathsImage.size());
assertEquals(3, sentryStore.getMPaths().size());
assertTrue(pathsImage.containsKey("db1.newTable2"));
@@ -2675,7 +2770,15 @@ public class TestSentryStore extends org.junit.Assert {
"user/hive/warehouse/db1.db/newTable1",
"user/hive/warehouse/db1.db/newTable2",
update);
- pathsImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathsImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathsImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(9, nodeMap.size());//Tree size
assertEquals(2, pathsImage.size());
assertEquals(3, sentryStore.getMPaths().size());
assertTrue(pathsImage.containsKey("db1.newTable2"));
@@ -2694,7 +2797,6 @@ public class TestSentryStore extends org.junit.Assert {
@Test
public void testPersistAndReplaceANewPathsImage() throws Exception {
Map<String, Collection<String>> authzPaths = new HashMap<>();
- PathsImage pathsImage;
long notificationID = 1;
// First image to persist (this will be replaced later)
@@ -2703,8 +2805,17 @@ public class TestSentryStore extends org.junit.Assert {
authzPaths.put("db1.table2", Sets.newHashSet("/user/hive/warehouse/db2.db/table2.1",
"/user/hive/warehouse/db2.db/table2.2"));
sentryStore.persistFullPathsImage(authzPaths, notificationID);
- pathsImage = sentryStore.retrieveFullPathsImage();
- assertEquals(1, pathsImage.getCurImgNum());
+ String[]prefixes = {"/user/hive/warehouse"};
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap();
+ TPathEntry root = nodeMap.get(pathsDump.getRootId());
+ Map<String, Collection<String>> pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, true);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(9, nodeMap.size());//Tree size
+ assertEquals(1, pathsUpdate.getImgNum());
// Second image to persist (it should replace first image)
authzPaths.clear();
@@ -2716,9 +2827,17 @@ public class TestSentryStore extends org.junit.Assert {
"/another-warehouse/db2.db/table2.3"));
sentryStore.persistFullPathsImage(authzPaths, notificationID+1);
- pathsImage = sentryStore.retrieveFullPathsImage();
- assertEquals(2, pathsImage.getCurImgNum());
- Map<String, Collection<String>> pathImage = pathsImage.getPathImage();
+ prefixes = new String[]{"/another-warehouse"};
+ pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, true);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(8, nodeMap.size());//Tree size
+ assertEquals(2, pathsUpdate.getImgNum());
assertEquals(3, pathImage.size());
for (Map.Entry<String, Collection<String>> entry : pathImage.entrySet()) {
@@ -2763,8 +2882,16 @@ public class TestSentryStore extends org.junit.Assert {
newAddUpdate.newPathChange("db2.table").addToAddPaths(Arrays.asList("db2", "tbl1"));
newAddUpdate.newPathChange("db2.table").addToAddPaths(Arrays.asList("db2", "tbl2"));
sentryStore.addAuthzPathsMapping("db2.table", Sets.newHashSet("db2/tbl1", "db2/tbl2"), newAddUpdate);
- PathsImage pathsImage = sentryStore.retrieveFullPathsImage();
- Map<String, Collection<String>> pathImage = pathsImage.getPathImage();
+ String[]prefixes = {"/"};
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap();
+ TPathEntry root = nodeMap.get(pathsDump.getRootId());
+ Map<String, Collection<String>> pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(10, nodeMap.size());//Tree size
assertEquals(2, pathImage.size());
assertEquals(2, pathImage.get("db2.table").size());
assertEquals(4, sentryStore.getMPaths().size());
@@ -2774,8 +2901,15 @@ public class TestSentryStore extends org.junit.Assert {
UniquePathsUpdate delUpdate = new UniquePathsUpdate("u3", notificationID, false);
delUpdate.newPathChange("db2.table").addToDelPaths(Arrays.asList("db2", "tbl1"));
sentryStore.deleteAuthzPathsMapping("db2.table", Sets.newHashSet("db2/tbl1"), delUpdate);
- pathsImage = sentryStore.retrieveFullPathsImage();
- pathImage = pathsImage.getPathImage();
+ pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(9, nodeMap.size());//Tree size
assertEquals(2, pathImage.size());
assertEquals(1, pathImage.get("db2.table").size());
assertEquals(3, sentryStore.getMPaths().size());
@@ -2815,12 +2949,21 @@ public class TestSentryStore extends org.junit.Assert {
.addToAddPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1"));
sentryStore.renameAuthzPathsMapping("db3.table1", "db1.newTable1",
"/another-warehouse/db3.db/table1.1", "user/hive/warehouse/db1.db/newTable1", renameUpdate);
- Map<String, Collection<String>> pathsImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ String[]prefixes = {"/"};
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap();
+ TPathEntry root = nodeMap.get(pathsDump.getRootId());
+ Map<String, Collection<String>> pathsImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathsImage, true);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(11, nodeMap.size());//Tree size
assertEquals(2, pathsImage.size());
assertEquals(4, sentryStore.getMPaths().size());
assertTrue(pathsImage.containsKey("db1.newTable1"));
assertTrue(CollectionUtils.isEqualCollection(Lists.newArrayList("/another-warehouse/db3.db/table1.2",
- "user/hive/warehouse/db1.db/newTable1"),
+ "/user/hive/warehouse/db1.db/newTable1"),
pathsImage.get("db1.newTable1")));
// Update path of 'db1.newTable2' from 'db1.newTable1' to 'db1.newTable2'
@@ -2834,7 +2977,15 @@ public class TestSentryStore extends org.junit.Assert {
"user/hive/warehouse/db1.db/newTable1",
"user/hive/warehouse/db1.db/newTable2",
update);
- pathsImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathsImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathsImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(12, nodeMap.size());//Tree size
assertEquals(3, pathsImage.size());
assertEquals(5, sentryStore.getMPaths().size());
assertTrue(pathsImage.containsKey("db1.newTable2"));
@@ -3275,7 +3426,16 @@ public class TestSentryStore extends org.junit.Assert {
"user/hive/warehouse/db1.db/table1/p1"));
authzPaths.put("db1.table2", Sets.newHashSet("user/hive/warehouse/db1.db/table2"));
sentryStore.persistFullPathsImage(authzPaths, lastNotificationId);
- Map<String, Collection<String>> pathsImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ String[]prefixes = {"/user/hive/warehouse"};
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap();
+ TPathEntry root = nodeMap.get(pathsDump.getRootId());
+ Map<String, Collection<String>> pathsImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathsImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(8, nodeMap.size());//Tree size
assertEquals(2, pathsImage.size());
if (lastNotificationId == null) {
@@ -3291,7 +3451,15 @@ public class TestSentryStore extends org.junit.Assert {
.addToAddPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1"));
sentryStore.renameAuthzPathsMapping("db1.table1", "db1.newTable1",
"user/hive/warehouse/db1.db/table1", "user/hive/warehouse/db1.db/newTable1", renameUpdate);
- pathsImage = sentryStore.retrieveFullPathsImage().getPathImage();
+ pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathsImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathsImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(9, nodeMap.size());//Tree size
assertEquals(2, pathsImage.size());
assertEquals(3, sentryStore.getMPaths().size());
assertTrue(pathsImage.containsKey("db1.newTable1"));
@@ -3333,8 +3501,16 @@ public class TestSentryStore extends org.junit.Assert {
assertEquals(sentryStore.isAuthzPathsMappingEmpty(), true);
sentryStore.addAuthzPathsMapping("db1.table",
Sets.newHashSet("db1/tbl1", "db1/tbl2"), addUpdate);
- PathsImage pathsImage = sentryStore.retrieveFullPathsImage();
- Map<String, Collection<String>> pathImage = pathsImage.getPathImage();
+ String[]prefixes = {"/"};
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap();
+ TPathEntry root = nodeMap.get(pathsDump.getRootId());
+ Map<String, Collection<String>> pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(4, nodeMap.size());//Tree size
assertEquals(1, pathImage.size());
assertEquals(2, pathImage.get("db1.table").size());
assertEquals(2, sentryStore.getMPaths().size());
@@ -3364,8 +3540,16 @@ public class TestSentryStore extends org.junit.Assert {
localSentryStore.addAuthzPathsMapping("db1.table",
Sets.newHashSet("db1/tbl1", "db1/tbl2"), addUpdate);
- PathsImage pathsImage = localSentryStore.retrieveFullPathsImage();
- Map<String, Collection<String>> pathImage = pathsImage.getPathImage();
+ String[]prefixes = {"/"};
+ PathsUpdate pathsUpdate = localSentryStore.retrieveFullPathsImageUpdate(prefixes);
+ TPathsDump pathsDump = pathsUpdate.toThrift().getPathsDump();
+ Map<Integer, TPathEntry>nodeMap = pathsDump.getNodeMap();
+ TPathEntry root = nodeMap.get(pathsDump.getRootId());
+ Map<String, Collection<String>> pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(4, nodeMap.size());//Tree size
assertEquals(1, pathImage.size());
assertEquals(2, pathImage.get("db1.table").size());
assertEquals(2, localSentryStore.getMPaths().size());
@@ -3379,7 +3563,15 @@ public class TestSentryStore extends org.junit.Assert {
delUpdate.newPathChange("db1.table")
.addToDelPaths(Arrays.asList("db1", "tbl1"));
localSentryStore.deleteAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1"), delUpdate);
- pathImage = localSentryStore.retrieveFullPathsImage().getPathImage();
+ pathsUpdate = localSentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(3, nodeMap.size());//Tree size
assertEquals(1, pathImage.size());
assertEquals(1, pathImage.get("db1.table").size());
assertEquals(1, localSentryStore.getMPaths().size());
@@ -3393,7 +3585,15 @@ public class TestSentryStore extends org.junit.Assert {
delAllupdate.newPathChange("db1.table")
.addToDelPaths(Lists.newArrayList(PathsUpdate.ALL_PATHS));
localSentryStore.deleteAllAuthzPathsMapping("db1.table", delAllupdate);
- pathImage = localSentryStore.retrieveFullPathsImage().getPathImage();
+ pathsUpdate = localSentryStore.retrieveFullPathsImageUpdate(prefixes);
+ pathsDump = pathsUpdate.toThrift().getPathsDump();
+ nodeMap = pathsDump.getNodeMap();
+ root = nodeMap.get(pathsDump.getRootId());
+ pathImage = new HashMap<>();
+ buildPathsImageMap(nodeMap, root, "", pathImage, false);
+
+ assertEquals("/", root.getPathElement());
+ assertEquals(1, nodeMap.size());//Tree size
assertEquals(0, pathImage.size());
assertEquals(0, localSentryStore.getMPaths().size());