You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ha...@apache.org on 2017/03/10 19:15:48 UTC
sentry git commit: SENTRY-1566: Make full Perm/Path snapshot
available for NN plugin (Hao Hao, Reviewed by: Alexander Kolbasov and Lei Xu)
Repository: sentry
Updated Branches:
refs/heads/sentry-ha-redesign 731a5d15e -> 8f5c17dbc
SENTRY-1566: Make full Perm/Path snapshot available for NN plugin (Hao Hao, Reviewed by: Alexander Kolbasov and Lei Xu)
Change-Id: I2788046192b71a3988f5d8bf1fa903eff18a7c8d
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/8f5c17db
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/8f5c17db
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/8f5c17db
Branch: refs/heads/sentry-ha-redesign
Commit: 8f5c17dbce6417b0c98ba6c2c31fa9c7fd8b222f
Parents: 731a5d1
Author: hahao <ha...@cloudera.com>
Authored: Thu Mar 9 15:37:38 2017 -0800
Committer: hahao <ha...@cloudera.com>
Committed: Fri Mar 10 11:14:17 2017 -0800
----------------------------------------------------------------------
.../org/apache/sentry/hdfs/ImageRetriever.java | 45 ++++
.../org/apache/sentry/hdfs/PathsUpdate.java | 3 +-
.../apache/sentry/hdfs/PathImageRetriever.java | 72 ++++++
.../apache/sentry/hdfs/PermImageRetriever.java | 93 ++++++++
.../sentry/hdfs/SentryHdfsMetricsUtil.java | 27 ++-
.../org/apache/sentry/hdfs/SentryPlugin.java | 53 +----
.../org/apache/sentry/hdfs/UpdateForwarder.java | 32 ++-
.../sentry/hdfs/UpdateablePermissions.java | 7 +-
.../apache/sentry/hdfs/TestUpdateForwarder.java | 15 +-
.../db/service/persistent/PathsImage.java | 48 ++++
.../db/service/persistent/PermissionsImage.java | 57 +++++
.../db/service/persistent/SentryStore.java | 219 ++++++++++++-------
.../db/service/persistent/TestSentryStore.java | 56 ++++-
13 files changed, 545 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java
new file mode 100644
index 0000000..0e40756
--- /dev/null
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/ImageRetriever.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.hdfs;
+
+import static org.apache.sentry.hdfs.Updateable.Update;
+
+/**
+ * ImageRetriever obtains a complete snapshot of either Sentry Permissions
+ * ({@code PermissionsUpdate}) or Sentry representation of Hive Paths
+ * ({@code PathsUpdate}).
+ * <p>
+ * The snapshot image should represent a consistent state.
+ * The {@link #retrieveFullImage(long)} method obtains such state snapshot from
+ * a persistent storage.
+ * The Snapshots are propagated to a consumer of Sentry, such as HDFS NameNode,
+ * whenever the consumer needs to synchronize its full state.
+ */
+public interface ImageRetriever<K extends Update> {
+
+ /**
+ * Retrieve a complete snapshot of type {@code Update} from a persistent storage.
+ *
+ * @param seqNum
+ * @return a complete snapshot of type {@link Update}, e.g {@link PermissionsUpdate}
+ * or {@link PathsUpdate}
+ * @throws Exception
+ */
+ K retrieveFullImage(long seqNum) throws Exception;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
index 992c8b7..14e967a 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -39,7 +39,6 @@ import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
* A wrapper class over the TPathsUpdate thrift generated class. Please see
* {@link Updateable.Update} for more information
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java
new file mode 100644
index 0000000..16a1604
--- /dev/null
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathImageRetriever.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.hdfs;
+
+import com.codahale.metrics.Timer;
+import org.apache.sentry.hdfs.service.thrift.TPathChanges;
+import org.apache.sentry.provider.db.service.persistent.PathsImage;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * PathImageRetriever obtains a complete snapshot of Hive Paths from a persistent
+ * storage and translate it into {@code PathsUpdate} that the consumers, such as
+ * HDFS NameNod, can understand.
+ */
+public class PathImageRetriever implements ImageRetriever<PathsUpdate> {
+
+ private final SentryStore sentryStore;
+
+ PathImageRetriever(SentryStore sentryStore) {
+ this.sentryStore = sentryStore;
+ }
+
+ @Override
+ public PathsUpdate retrieveFullImage(long seqNum) throws Exception {
+ try (final Timer.Context timerContext =
+ SentryHdfsMetricsUtil.getRetrievePathFullImageTimer.time()) {
+
+ // Reads a up-to-date complete snapshot of Hive paths from the
+ // persistent storage, along with the sequence number of latest
+ // delta change the snapshot corresponds to.
+ PathsImage pathsImage = sentryStore.retrieveFullPathsImage();
+ long curSeqNum = pathsImage.getCurSeqNum();
+ Map<String, Set<String>> pathImage = pathsImage.getPathImage();
+
+ // Translates the complete Hive paths snapshot into a PathsUpdate.
+ // Adds all <hiveObj, paths> mapping to be included in this paths update.
+ // And label it with the latest delta change sequence number for consumer
+ // to be aware of the next delta change it should continue with.
+ // TODO: use curSeqNum from DB instead of seqNum when doing SENTRY-1613
+ PathsUpdate pathsUpdate = new PathsUpdate(seqNum, true);
+ for (Map.Entry<String, Set<String>> pathEnt : pathImage.entrySet()) {
+ TPathChanges pathChange = pathsUpdate.newPathChange(pathEnt.getKey());
+
+ for (String path : pathEnt.getValue()) {
+ pathChange.addToAddPaths(PathsUpdate.splitPath(path));
+ }
+ }
+
+ SentryHdfsMetricsUtil.getPathChangesHistogram.update(pathsUpdate
+ .getPathChanges().size());
+ return pathsUpdate;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java
new file mode 100644
index 0000000..3017c9e
--- /dev/null
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermImageRetriever.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.hdfs;
+
+import com.codahale.metrics.Timer;
+import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate;
+import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
+import org.apache.sentry.hdfs.service.thrift.TRoleChanges;
+import org.apache.sentry.provider.db.service.persistent.PermissionsImage;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * PermImageRetriever obtains a complete snapshot of Sentry permission from a persistent
+ * storage and translate it into {@code PermissionsUpdate} that the consumers, such as
+ * HDFS NameNod, can understand.
+ */
+public class PermImageRetriever implements ImageRetriever<PermissionsUpdate> {
+
+ private final SentryStore sentryStore;
+
+ PermImageRetriever(SentryStore sentryStore) {
+ this.sentryStore = sentryStore;
+ }
+
+ @Override
+ public PermissionsUpdate retrieveFullImage(long seqNum) throws Exception {
+ try(Timer.Context timerContext =
+ SentryHdfsMetricsUtil.getRetrievePermFullImageTimer.time()) {
+
+ // Read the most up-to-date snapshot of Sentry perm information,
+ // with a corresponding delta change sequence number.
+ PermissionsImage permImage = sentryStore.retrieveFullPermssionsImage();
+ long curSeqNum = permImage.getCurSeqNum();
+ Map<String, Map<String, String>> privilegeImage =
+ permImage.getPrivilegeImage();
+ Map<String, List<String>> roleImage =
+ permImage.getRoleImage();
+
+ // Translates the complete Sentry permission snapshot into a PermissionsUpdate.
+ // Adds the <hiveObj, <role, privileges>> mapping and the <role, groups> mapping
+ // to be included in the permission update.
+ // And label it with the latest delta change sequence number for consumer
+ // to be aware of the next delta change it should continue with.
+ TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, curSeqNum,
+ new HashMap<String, TPrivilegeChanges>(),
+ new HashMap<String, TRoleChanges>());
+
+ for (Map.Entry<String, Map<String, String>> privEnt : privilegeImage.entrySet()) {
+ String authzObj = privEnt.getKey();
+ Map<String,String> privs = privEnt.getValue();
+ tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges(
+ authzObj, privs, new HashMap<String, String>()));
+ }
+
+ for (Map.Entry<String, List<String>> privEnt : roleImage.entrySet()) {
+ String role = privEnt.getKey();
+ List<String> groups = privEnt.getValue();
+ tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups,
+ new LinkedList<String>()));
+ }
+
+ PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate);
+ // TODO: use curSeqNum from DB instead of seqNum when doing SENTRY-1567
+ permissionsUpdate.setSeqNum(seqNum);
+ SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update(
+ tPermUpdate.getPrivilegeChangesSize());
+ SentryHdfsMetricsUtil.getRoleChangesHistogram.update(
+ tPermUpdate.getRoleChangesSize());
+ return permissionsUpdate;
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
index e68c708..be14569 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java
@@ -60,19 +60,30 @@ public class SentryHdfsMetricsUtil {
MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification",
"path-changes-size"));
- // Metrics for retrieveFullImage in SentryPlugin.PermImageRetriever
- // The time used for each retrieveFullImage
- public static final Timer getRetrieveFullImageTimer = sentryMetrics.getTimer(
- MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image"));
- // The size of privilege changes for each retrieveFullImage
+ // Metrics for retrievePermFullImage in PermImageRetriever
+ // The time used for each retrievePermFullImage
+ public static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image"));
+ // The size of privilege changes for each retrievePermFullImage
public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram(
- MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image",
+ MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image",
"privilege-changes-size"));
- // The size of role changes for each retrieveFullImage call
+ // The size of role changes for each retrievePermFullImage call
public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram(
- MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image",
+ MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image",
"role-changes-size"));
+ // Metrics for retrievePathFullImage in PathImageRetriever
+ // The time used for each retrievePathFullImage
+ public static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer(
+ MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image"));
+
+ // The size of path changes for each retrievePathFullImage
+ public static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram(
+ MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image",
+ "path-changes-size"));
+
+
// Metrics for notifySentry HMS update in MetaStorePlugin
// The timer used for each notifySentry
public static final Timer getNotifyHMSUpdateTimer = sentryMetrics.getTimer(
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
index f68f690..029f9d5 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
@@ -18,23 +18,17 @@
package org.apache.sentry.hdfs;
-import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
-import com.codahale.metrics.Timer;
import org.apache.hadoop.conf.Configuration;
import org.apache.sentry.core.common.utils.SigUtils;
import org.apache.sentry.hdfs.ServiceConstants.ServerConfig;
-import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever;
-import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate;
import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
import org.apache.sentry.hdfs.service.thrift.TRoleChanges;
import org.apache.sentry.provider.db.SentryPolicyStorePlugin;
-import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock;
import org.apache.sentry.provider.db.service.persistent.SentryStore;
import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleAddGroupsRequest;
import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleDeleteGroupsRequest;
@@ -43,7 +37,6 @@ import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleRevokePrivil
import org.apache.sentry.provider.db.service.thrift.TDropPrivilegesRequest;
import org.apache.sentry.provider.db.service.thrift.TDropSentryRoleRequest;
import org.apache.sentry.provider.db.service.thrift.TRenamePrivilegesRequest;
-import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
import org.apache.sentry.provider.db.service.thrift.TSentryGroup;
import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege;
import org.apache.sentry.service.thrift.HMSFollower;
@@ -122,48 +115,6 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen
public static volatile SentryPlugin instance;
- static class PermImageRetriever implements ExternalImageRetriever<PermissionsUpdate> {
-
- private final SentryStore sentryStore;
-
- public PermImageRetriever(SentryStore sentryStore) {
- this.sentryStore = sentryStore;
- }
-
- @Override
- public PermissionsUpdate retrieveFullImage(long currSeqNum) throws Exception {
- try(Timer.Context timerContext =
- SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time()) {
-
- SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time();
- Map<String, HashMap<String, String>> privilegeImage = sentryStore.retrieveFullPrivilegeImage();
- Map<String, LinkedList<String>> roleImage = sentryStore.retrieveFullRoleImage();
-
- TPermissionsUpdate tPermUpdate = new TPermissionsUpdate(true, currSeqNum,
- new HashMap<String, TPrivilegeChanges>(),
- new HashMap<String, TRoleChanges>());
- for (Map.Entry<String, HashMap<String, String>> privEnt : privilegeImage.entrySet()) {
- String authzObj = privEnt.getKey();
- HashMap<String,String> privs = privEnt.getValue();
- tPermUpdate.putToPrivilegeChanges(authzObj, new TPrivilegeChanges(
- authzObj, privs, new HashMap<String, String>()));
- }
- for (Map.Entry<String, LinkedList<String>> privEnt : roleImage.entrySet()) {
- String role = privEnt.getKey();
- LinkedList<String> groups = privEnt.getValue();
- tPermUpdate.putToRoleChanges(role, new TRoleChanges(role, groups, new LinkedList<String>()));
- }
- PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate);
- permissionsUpdate.setSeqNum(currSeqNum);
- SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update(
- tPermUpdate.getPrivilegeChangesSize());
- SentryHdfsMetricsUtil.getRoleChangesHistogram.update(
- tPermUpdate.getRoleChangesSize());
- return permissionsUpdate;
- }
- }
- }
-
private UpdateForwarder<PathsUpdate> pathsUpdater;
private UpdateForwarder<PermissionsUpdate> permsUpdater;
// TODO: Each perm change sequence number should be generated during persistence at sentry store.
@@ -202,10 +153,10 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen
permImageRetriever = new PermImageRetriever(sentryStore);
pathsUpdater = UpdateForwarder.create(conf, new UpdateableAuthzPaths(
- pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs);
+ pathPrefixes), new PathsUpdate(0, false), null, 100, initUpdateRetryDelayMs, false);
permsUpdater = UpdateForwarder.create(conf,
new UpdateablePermissions(permImageRetriever), new PermissionsUpdate(0, false),
- permImageRetriever, 100, initUpdateRetryDelayMs);
+ permImageRetriever, 100, initUpdateRetryDelayMs, true);
LOGGER.info("Sentry HDFS plugin initialized !!");
instance = this;
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
index 6d5c607..22c5769 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateForwarder.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -37,12 +37,6 @@ import org.slf4j.LoggerFactory;
public class UpdateForwarder<K extends Updateable.Update> implements
Updateable<K>, Closeable {
- interface ExternalImageRetriever<K> {
-
- K retrieveFullImage(long currSeqNum) throws Exception;
-
- }
-
private final AtomicLong lastSeenSeqNum = new AtomicLong(0);
protected final AtomicLong lastCommittedSeqNum = new AtomicLong(0);
// Updates should be handled in order
@@ -60,7 +54,7 @@ public class UpdateForwarder<K extends Updateable.Update> implements
// UpdateLog is disabled when getMaxUpdateLogSize() = 0;
private final int maxUpdateLogSize;
- private final ExternalImageRetriever<K> imageRetreiver;
+ private final ImageRetriever<K> imageRetreiver;
private volatile Updateable<K> updateable;
@@ -72,16 +66,16 @@ public class UpdateForwarder<K extends Updateable.Update> implements
private static final String UPDATABLE_TYPE_NAME = "update_forwarder";
public UpdateForwarder(Configuration conf, Updateable<K> updateable,
- ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize) {
- this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY);
+ ImageRetriever<K> imageRetreiver, int maxUpdateLogSize, boolean shouldInit) {
+ this(conf, updateable, imageRetreiver, maxUpdateLogSize, INIT_UPDATE_RETRY_DELAY, shouldInit);
}
protected UpdateForwarder(Configuration conf, Updateable<K> updateable, //NOPMD
- ExternalImageRetriever<K> imageRetreiver, int maxUpdateLogSize,
- int initUpdateRetryDelay) {
+ ImageRetriever<K> imageRetreiver, int maxUpdateLogSize,
+ int initUpdateRetryDelay, boolean shouldInit) {
this.maxUpdateLogSize = maxUpdateLogSize;
this.imageRetreiver = imageRetreiver;
- if (imageRetreiver != null) {
+ if (shouldInit) {
spawnInitialUpdater(updateable, initUpdateRetryDelay);
} else {
this.updateable = updateable;
@@ -89,17 +83,17 @@ public class UpdateForwarder<K extends Updateable.Update> implements
}
public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf,
- Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver,
- int maxUpdateLogSize) throws SentryPluginException {
+ Updateable<K> updateable, K update, ImageRetriever<K> imageRetreiver,
+ int maxUpdateLogSize, boolean shouldInit) throws SentryPluginException {
return create(conf, updateable, update, imageRetreiver, maxUpdateLogSize,
- INIT_UPDATE_RETRY_DELAY);
+ INIT_UPDATE_RETRY_DELAY, shouldInit);
}
public static <K extends Updateable.Update> UpdateForwarder<K> create(Configuration conf,
- Updateable<K> updateable, K update, ExternalImageRetriever<K> imageRetreiver,
- int maxUpdateLogSize, int initUpdateRetryDelay) throws SentryPluginException {
+ Updateable<K> updateable, K update, ImageRetriever<K> imageRetreiver,
+ int maxUpdateLogSize, int initUpdateRetryDelay, boolean shouldInit) throws SentryPluginException {
return new UpdateForwarder<K>(conf, updateable, imageRetreiver,
- maxUpdateLogSize, initUpdateRetryDelay);
+ maxUpdateLogSize, initUpdateRetryDelay, shouldInit);
}
private void spawnInitialUpdater(final Updateable<K> updateable,
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java
index fe2baa6..03c67d6 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/UpdateablePermissions.java
@@ -20,16 +20,14 @@ package org.apache.sentry.hdfs;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReadWriteLock;
-import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever;
-
public class UpdateablePermissions implements Updateable<PermissionsUpdate>{
private static final String UPDATABLE_TYPE_NAME = "perm_update";
private AtomicLong seqNum = new AtomicLong();
- private final ExternalImageRetriever<PermissionsUpdate> imageRetreiver;
+ private final ImageRetriever<PermissionsUpdate> imageRetreiver;
public UpdateablePermissions(
- ExternalImageRetriever<PermissionsUpdate> imageRetreiver) {
+ ImageRetriever<PermissionsUpdate> imageRetreiver) {
this.imageRetreiver = imageRetreiver;
}
@@ -62,5 +60,4 @@ public class UpdateablePermissions implements Updateable<PermissionsUpdate>{
public String getUpdateableTypeName() {
return UPDATABLE_TYPE_NAME;
}
-
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java
index 0f0d0a7..d12b134 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestUpdateForwarder.java
@@ -27,7 +27,6 @@ import org.apache.thrift.TException;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever;
import org.apache.sentry.hdfs.Updateable.Update;
import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
import org.junit.After;
@@ -134,7 +133,7 @@ public class TestUpdateForwarder {
}
}
- static class DummyImageRetreiver implements ExternalImageRetriever<DummyUpdate> {
+ static class DummyImageRetreiver implements ImageRetriever<DummyUpdate> {
private String state;
public void setState(String state) {
@@ -164,7 +163,7 @@ public class TestUpdateForwarder {
DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
imageRetreiver.setState("a,b,c");
updateForwarder = UpdateForwarder.create(
- testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10);
+ testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 10, true);
Assert.assertEquals(-2, updateForwarder.getLastUpdatedSeqNum());
List<DummyUpdate> allUpdates = updateForwarder.getAllUpdatesFrom(0);
Assert.assertTrue(allUpdates.size() == 1);
@@ -184,7 +183,7 @@ public class TestUpdateForwarder {
DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
imageRetreiver.setState("a,b,c");
updateForwarder = UpdateForwarder.create(
- testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5);
+ testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true);
updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d"));
while(!updateForwarder.areAllUpdatesCommited()) {
Thread.sleep(100);
@@ -205,7 +204,7 @@ public class TestUpdateForwarder {
Assume.assumeTrue(!testConf.getBoolean(ServerConfig.SENTRY_HA_ENABLED,
false));
updateForwarder = UpdateForwarder.create(
- testConf, new DummyUpdatable(), new DummyUpdate(), null, 5);
+ testConf, new DummyUpdatable(), new DummyUpdate(), null, 5, false);
updateForwarder.handleUpdateNotification(new DummyUpdate(-1, true).setState("a"));
while(!updateForwarder.areAllUpdatesCommited()) {
Thread.sleep(100);
@@ -232,7 +231,7 @@ public class TestUpdateForwarder {
DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
imageRetreiver.setState("a,b,c");
updateForwarder = UpdateForwarder.create(
- testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5);
+ testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true);
updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d"));
while(!updateForwarder.areAllUpdatesCommited()) {
Thread.sleep(100);
@@ -282,7 +281,7 @@ public class TestUpdateForwarder {
DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
imageRetreiver.setState("a,b,c");
updateForwarder = UpdateForwarder.create(
- testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5);
+ testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true);
updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d"));
while(!updateForwarder.areAllUpdatesCommited()) {
Thread.sleep(100);
@@ -328,7 +327,7 @@ public class TestUpdateForwarder {
DummyImageRetreiver imageRetreiver = new DummyImageRetreiver();
imageRetreiver.setState("a,b,c");
updateForwarder = UpdateForwarder.create(
- testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5);
+ testConf, new DummyUpdatable(), new DummyUpdate(), imageRetreiver, 5, true);
updateForwarder.handleUpdateNotification(new DummyUpdate(5, false).setState("d"));
while(!updateForwarder.areAllUpdatesCommited()) {
Thread.sleep(100);
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java
new file mode 100644
index 0000000..fd56ce2
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PathsImage.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.provider.db.service.persistent;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A container for complete hive paths snapshot.
+ * <p>
+ * It is composed by a hiveObj to Paths mapping and the sequence number/change ID
+ * of latest delta change that the snapshot maps to.
+ */
+public class PathsImage {
+
+ // A full snapshot of hiveObj to Paths mapping.
+ private final Map<String, Set<String>> pathImage;
+ private final long curSeqNum;
+
+ PathsImage(Map<String, Set<String>> pathImage, long curSeqNum) {
+ this.pathImage = pathImage;
+ this.curSeqNum = curSeqNum;
+ }
+
+ public long getCurSeqNum() {
+ return curSeqNum;
+ }
+
+ public Map<String, Set<String>> getPathImage() {
+ return pathImage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java
new file mode 100644
index 0000000..f03e93f
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/PermissionsImage.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.provider.db.service.persistent;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A container for complete sentry permission snapshot.
+ * <p>
+ * It is composed by a role to groups mapping, and hiveObj to < role, privileges > mapping.
+ * It also has the sequence number/change ID of latest delta change that the snapshot maps to.
+ */
+public class PermissionsImage {
+
+ // A full snapshot of sentry role to groups mapping.
+ private final Map<String, List<String>> roleImage;
+
+ // A full snapshot of hiveObj to <role, privileges> mapping.
+ private final Map<String, Map<String, String>> privilegeImage;
+ private final long curSeqNum;
+
+ PermissionsImage(Map<String, List<String>> roleImage,
+ Map<String, Map<String, String>> privilegeImage, long curSeqNum) {
+ this.roleImage = roleImage;
+ this.privilegeImage = privilegeImage;
+ this.curSeqNum = curSeqNum;
+ }
+
+ public long getCurSeqNum() {
+ return curSeqNum;
+ }
+
+ public Map<String, Map<String, String>> getPrivilegeImage() {
+ return privilegeImage;
+ }
+
+ public Map<String, List<String>> getRoleImage() {
+ return roleImage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index a5a835e..6ea6d3f 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -2260,107 +2260,158 @@ public class SentryStore {
ServerConfig.ADMIN_GROUPS, new String[]{}));
}
- public Map<String, HashMap<String, String>> retrieveFullPrivilegeImage() throws Exception {
+ /**
+ * Retrieves an up-to-date sentry permission snapshot.
+ * <p>
+ * It reads hiveObj to < role, privileges > mapping from {@link MSentryPrivilege}
+ * table and role to groups mapping from {@link MSentryGroup}.
+ * It also gets the changeID of latest delta update, from {@link MSentryPathChange}, that
+ * the snapshot corresponds to.
+ *
+ * @return a {@link PathsImage} contains the mapping of hiveObj to
+ * < role, privileges > and the mapping of role to < Groups >.
+ * For empty image returns {@link #EMPTY_CHANGE_ID} and empty maps.
+ * @throws Exception
+ */
+ public PermissionsImage retrieveFullPermssionsImage() throws Exception {
return tm.executeTransaction(
- new TransactionBlock<Map<String, HashMap<String, String>>>() {
- public Map<String, HashMap<String, String>> execute(PersistenceManager pm)
- throws Exception {
- Map<String, HashMap<String, String>> retVal = new HashMap<>();
- Query query = pm.newQuery(MSentryPrivilege.class);
- QueryParamBuilder paramBuilder = newQueryParamBuilder();
- paramBuilder
- .addNotNull(SERVER_NAME)
- .addNotNull(DB_NAME)
- .addNull(URI);
+ new TransactionBlock<PermissionsImage>() {
+ public PermissionsImage execute(PersistenceManager pm)
+ throws Exception {
+ // curChangeID could be 0, if Sentry server has been running before
+ // enable SentryPlugin(HDFS Sync feature).
+ long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPermChange.class);
+ Map<String, List<String>> roleImage = retrieveFullRoleImageCore(pm);
+ Map<String, Map<String, String>> privilegeMap = retrieveFullPrivilegeImageCore(pm);
- query.setFilter(paramBuilder.toString());
- query.setOrdering("serverName ascending, dbName ascending, tableName ascending");
- @SuppressWarnings("unchecked")
- List<MSentryPrivilege> privileges =
- (List<MSentryPrivilege>) query.executeWithMap(paramBuilder.getArguments());
- for (MSentryPrivilege mPriv : privileges) {
- String authzObj = mPriv.getDbName();
- if (!isNULL(mPriv.getTableName())) {
- authzObj = authzObj + "." + mPriv.getTableName();
- }
- HashMap<String, String> pUpdate = retVal.get(authzObj);
- if (pUpdate == null) {
- pUpdate = new HashMap<>();
- retVal.put(authzObj, pUpdate);
- }
- for (MSentryRole mRole : mPriv.getRoles()) {
- String existingPriv = pUpdate.get(mRole.getRoleName());
- if (existingPriv == null) {
- pUpdate.put(mRole.getRoleName(), mPriv.getAction().toUpperCase());
- } else {
- pUpdate.put(mRole.getRoleName(), existingPriv + ","
- + mPriv.getAction().toUpperCase());
- }
- }
- }
- return retVal;
+ return new PermissionsImage(roleImage, privilegeMap, curChangeID);
+ }
+ });
+ }
+
+ /**
+ * Retrieves an up-to-date sentry privileges snapshot from {@code MSentryPrivilege} table.
+ * The snapshot is represented by mapping of hiveObj to role privileges.
+ *
+ * @param pm PersistenceManager
+ * @return a mapping of hiveObj to < role, privileges >
+ * @throws Exception
+ */
+ private Map<String, Map<String, String>> retrieveFullPrivilegeImageCore(PersistenceManager pm)
+ throws Exception {
+
+ Map<String, Map<String, String>> retVal = new HashMap<>();
+ Query query = pm.newQuery(MSentryPrivilege.class);
+ QueryParamBuilder paramBuilder = newQueryParamBuilder();
+ paramBuilder.addNotNull(SERVER_NAME)
+ .addNotNull(DB_NAME)
+ .addNull(URI);
+
+ query.setFilter(paramBuilder.toString());
+ query.setOrdering("serverName ascending, dbName ascending, tableName ascending");
+ @SuppressWarnings("unchecked")
+ List<MSentryPrivilege> privileges =
+ (List<MSentryPrivilege>) query.executeWithMap(paramBuilder.getArguments());
+ for (MSentryPrivilege mPriv : privileges) {
+ String authzObj = mPriv.getDbName();
+ if (!isNULL(mPriv.getTableName())) {
+ authzObj = authzObj + "." + mPriv.getTableName();
+ }
+ Map<String, String> pUpdate = retVal.get(authzObj);
+ if (pUpdate == null) {
+ pUpdate = new HashMap<>();
+ retVal.put(authzObj, pUpdate);
+ }
+ for (MSentryRole mRole : mPriv.getRoles()) {
+ String existingPriv = pUpdate.get(mRole.getRoleName());
+ if (existingPriv == null) {
+ pUpdate.put(mRole.getRoleName(), mPriv.getAction().toUpperCase());
+ } else {
+ pUpdate.put(mRole.getRoleName(), existingPriv + "," + mPriv.getAction().toUpperCase());
}
- });
+ }
+ }
+ return retVal;
}
/**
- * @return Mapping of Role -> [Groups]
+ * Retrieves an up-to-date sentry role snapshot from {@code MSentryGroup} table.
+ * The snapshot is represented by a role to groups map.
+ *
+ * @param pm PersistenceManager
+ * @return a mapping of Role to < Groups >
+ * @throws Exception
*/
- public Map<String, LinkedList<String>> retrieveFullRoleImage() throws Exception {
- return tm.executeTransaction(
- new TransactionBlock<Map<String, LinkedList<String>>>() {
- public Map<String, LinkedList<String>> execute(PersistenceManager pm) throws Exception {
- Query query = pm.newQuery(MSentryGroup.class);
- @SuppressWarnings("unchecked")
- List<MSentryGroup> groups = (List<MSentryGroup>) query.execute();
- if (groups.isEmpty()) {
- return Collections.emptyMap();
- }
+ private Map<String, List<String>> retrieveFullRoleImageCore(PersistenceManager pm)
+ throws Exception {
+ Query query = pm.newQuery(MSentryGroup.class);
+ @SuppressWarnings("unchecked")
+ List<MSentryGroup> groups = (List<MSentryGroup>) query.execute();
+ if (groups.isEmpty()) {
+ return Collections.emptyMap();
+ }
- Map<String, LinkedList<String>> retVal = new HashMap<>();
- for (MSentryGroup mGroup : groups) {
- for (MSentryRole role : mGroup.getRoles()) {
- LinkedList<String> rUpdate = retVal.get(role.getRoleName());
- if (rUpdate == null) {
- rUpdate = new LinkedList<String>();
- retVal.put(role.getRoleName(), rUpdate);
- }
- rUpdate.add(mGroup.getGroupName());
- }
- }
- return retVal;
- }
- });
+ Map<String, List<String>> retVal = new HashMap<>();
+ for (MSentryGroup mGroup : groups) {
+ for (MSentryRole role : mGroup.getRoles()) {
+ List<String> rUpdate = retVal.get(role.getRoleName());
+ if (rUpdate == null) {
+ rUpdate = new LinkedList<>();
+ retVal.put(role.getRoleName(), rUpdate);
+ }
+ rUpdate.add(mGroup.getGroupName());
+ }
+ }
+ return retVal;
+ }
+
+ /**
+ * Retrieves an up-to-date hive paths snapshot.
+ * <p>
+ * It reads hiveObj to paths mapping from {@link MAuthzPathsMapping} table and
+ * gets the changeID of latest delta update, from {@link MSentryPathChange}, that
+ * the snapshot corresponds to.
+ *
+ * @return an up-to-date hive paths snapshot contains mapping of hiveObj to < Paths >.
+ * For empty image return {@link #EMPTY_CHANGE_ID} and a empty map.
+ * @throws Exception
+ */
+ public PathsImage retrieveFullPathsImage() throws Exception {
+ return (PathsImage) tm.executeTransaction(
+ new TransactionBlock() {
+ public Object execute(PersistenceManager pm) throws Exception {
+ // curChangeID could be 0 for the first full snapshot fetching
+ // from HMS. It does not have corresponding delta update.
+ long curChangeID = getLastProcessedChangeIDCore(pm, MSentryPathChange.class);
+ Map<String, Set<String>> pathImage = retrieveFullPathsImageCore(pm);
+
+ return new PathsImage(pathImage, curChangeID);
+ }
+ });
}
/**
- * This returns a Mapping of Authz -> [Paths]
+ * Retrieves an up-to-date hive paths snapshot from {@code MAuthzPathsMapping} table.
+ * The snapshot is represented by a hiveObj to paths map.
+ *
+ * @return a mapping of hiveObj to < Paths >.
*/
- public Map<String, Set<String>> retrieveFullPathsImage() {
- Map<String, Set<String>> result = new HashMap<>();
- try {
- result = (Map<String, Set<String>>) tm.executeTransaction(
- new TransactionBlock<Object>() {
- public Object execute(PersistenceManager pm) throws Exception {
- Map<String, Set<String>> retVal = new HashMap<>();
- Query query = pm.newQuery(MAuthzPathsMapping.class);
- List<MAuthzPathsMapping> authzToPathsMappings = (List<MAuthzPathsMapping>) query.execute();
- for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) {
- retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths());
- }
- return retVal;
- }
- });
- } catch (Exception e) {
- LOGGER.error(e.getMessage(), e);
+ private Map<String, Set<String>> retrieveFullPathsImageCore(PersistenceManager pm) {
+ Map<String, Set<String>> retVal = new HashMap<>();
+ Query query = pm.newQuery(MAuthzPathsMapping.class);
+ Iterable<MAuthzPathsMapping> authzToPathsMappings =
+ (Iterable<MAuthzPathsMapping>) query.execute();
+
+ for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) {
+ retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths());
}
- return result;
+ return retVal;
}
/**
- * Persist a full hive snapshot into Sentry DB in a single transaction.
+ * Persist an up-to-date hive snapshot into Sentry DB in a single transaction.
*
- * @param authzPaths Mapping of hiveObj -> [Paths]
+ * @param authzPaths Mapping of hiveObj to < Paths <
* @throws Exception
*/
public void persistFullPathsImage(final Map<String, Set<String>> authzPaths) throws Exception {
http://git-wip-us.apache.org/repos/asf/sentry/blob/8f5c17db/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index 91f15c0..75f855c 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -2188,21 +2188,67 @@ public class TestSentryStore extends org.junit.Assert {
}
@Test
+ public void testRetrieveFullPermssionsImage() throws Exception {
+
+ // Create roles
+ String roleName1 = "privs-r1", roleName2 = "privs-r2";
+ String groupName1 = "privs-g1";
+ String grantor = "g1";
+ sentryStore.createSentryRole(roleName1);
+ sentryStore.createSentryRole(roleName2);
+
+ // Grant Privileges to the roles
+ TSentryPrivilege privilege1 = new TSentryPrivilege();
+ privilege1.setPrivilegeScope("TABLE");
+ privilege1.setServerName("server1");
+ privilege1.setDbName("db1");
+ privilege1.setTableName("tbl1");
+ privilege1.setAction("SELECT");
+ privilege1.setCreateTime(System.currentTimeMillis());
+ sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName1, privilege1);
+ sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName2, privilege1);
+
+ TSentryPrivilege privilege2 = new TSentryPrivilege();
+ privilege2.setPrivilegeScope("SERVER");
+ privilege2.setServerName("server1");
+ privilege1.setDbName("db2");
+ privilege1.setAction("ALL");
+ privilege2.setCreateTime(System.currentTimeMillis());
+ sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName2, privilege2);
+
+ // Grant roles to the groups
+ Set<TSentryGroup> groups = Sets.newHashSet();
+ TSentryGroup group = new TSentryGroup();
+ group.setGroupName(groupName1);
+ groups.add(group);
+ sentryStore.alterSentryRoleAddGroups(grantor, roleName1, groups);
+ sentryStore.alterSentryRoleAddGroups(grantor, roleName2, groups);
+
+ PermissionsImage permImage = sentryStore.retrieveFullPermssionsImage();
+ Map<String, Map<String, String>> privs = permImage.getPrivilegeImage();
+ Map<String, List<String>> roles = permImage.getRoleImage();
+ assertEquals(2, privs.get("db1.tbl1").size());
+ assertEquals(2, roles.size());
+ }
+
+ @Test
public void testAuthzPathsMapping() throws Exception {
sentryStore.createAuthzPathsMapping("db1.table1", Sets.newHashSet("/user/hive/warehouse/db1.db/table1"));
sentryStore.createAuthzPathsMapping("db1.table2", Sets.newHashSet("/user/hive/warehouse/db1.db/table2"));
- Map<String, Set<String>> pathsImage = sentryStore.retrieveFullPathsImage();
- assertEquals(2, pathsImage.size());
- assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathsImage.get("db1.table1"));
+ PathsImage pathsImage = sentryStore.retrieveFullPathsImage();
+ Map<String, Set<String>> pathImage = pathsImage.getPathImage();
+ assertEquals(2, pathImage.size());
+ assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathImage.get("db1.table1"));
Map<String, Set<String>> authzPaths = new HashMap<>();
authzPaths.put("db2.table1", Sets.newHashSet("/user/hive/warehouse/db2.db/table1"));
authzPaths.put("db2.table2", Sets.newHashSet("/user/hive/warehouse/db2.db/table2"));
sentryStore.persistFullPathsImage(authzPaths);
pathsImage = sentryStore.retrieveFullPathsImage();
- assertEquals(4, pathsImage.size());
- assertEquals(Sets.newHashSet("/user/hive/warehouse/db2.db/table1"), pathsImage.get("db2.table1"));
+ pathImage = pathsImage.getPathImage();
+ assertEquals(4, pathImage.size());
+ assertEquals(Sets.newHashSet("/user/hive/warehouse/db2.db/table1"), pathImage.get("db2.table1"));
}
public void testQueryParamBuilder() {