You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by va...@apache.org on 2017/09/08 01:17:47 UTC
sentry git commit: SENTRY-1931: NameNode only gets full snapshot once
(Sergio Pena, Reviewed by: Alexander Kolbasov, Vamsee Yarlagadda)
Repository: sentry
Updated Branches:
refs/heads/master c302da781 -> 398a8a937
SENTRY-1931: NameNode only gets full snapshot once (Sergio Pena, Reviewed by: Alexander Kolbasov, Vamsee Yarlagadda)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/398a8a93
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/398a8a93
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/398a8a93
Branch: refs/heads/master
Commit: 398a8a9378cb43fea7f66153df917b9eb3abd40c
Parents: c302da7
Author: Vamsee Yarlagadda <va...@cloudera.com>
Authored: Thu Sep 7 18:15:57 2017 -0700
Committer: Vamsee Yarlagadda <va...@cloudera.com>
Committed: Thu Sep 7 18:15:57 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java | 6 ++++--
.../src/main/java/org/apache/sentry/hdfs/PathsUpdate.java | 5 +++++
.../main/java/org/apache/sentry/hdfs/PermissionsUpdate.java | 5 +++++
.../src/main/java/org/apache/sentry/hdfs/Updateable.java | 2 ++
.../main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java | 6 +++++-
.../main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java | 2 +-
.../main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java | 3 ++-
.../main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java | 2 +-
.../java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java | 2 +-
.../test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java | 6 ++++--
10 files changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java
index 4503950..3b7fcef 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/DeltaRetriever.java
@@ -29,7 +29,7 @@ import static org.apache.sentry.hdfs.Updateable.Update;
* Sentry permissions are represented as {@link PermissionsUpdate} and HMS Paths
* are represented as {@link PathsUpdate}. The delta update contains change
* from a state to another.
- * The {@link #retrieveDelta(long)} method obtains such delta update from a persistent storage.
+ * The {@link #retrieveDelta(long,long)} method obtains such delta update from a persistent storage.
* Delta update is propagated to a consumer of Sentry, such as HDFS NameNode whenever
* the consumer needs to synchronize the update.
*/
@@ -38,13 +38,15 @@ public interface DeltaRetriever<K extends Update> {
/**
* Retrieves all delta updates of type {@link Update} newer than or equal to
* the given sequence number/change ID (inclusive) from a persistent storage.
+ * An image number may be used by the implementation to request deltas for a specific image.
* An empty collection can be returned.
*
* @param seqNum the given seq number
+ * @param imgNum the given img number
* @return delta updates of type K
* @throws Exception when there is an error in operation on persistent storage
*/
- List<K> retrieveDelta(long seqNum) throws Exception;
+ List<K> retrieveDelta(long seqNum, long imgNum) throws Exception;
/**
* Checks if there the delta update is available, given the sequence number/change
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
index ccc10ef..3996747 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
@@ -100,6 +100,11 @@ public class PathsUpdate implements Updateable.Update {
return tPathsUpdate.getImgNum();
}
+ @Override
+ public void setImgNum(long imgNum) {
+ tPathsUpdate.setImgNum(imgNum);
+ }
+
public TPathsUpdate toThrift() {
return tPathsUpdate;
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
index 7aa60a3..392f9a6 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java
@@ -68,6 +68,11 @@ public class PermissionsUpdate implements Updateable.Update {
}
@Override
+ public void setImgNum(long imgNum) {
+ throw new UnsupportedOperationException("setImgNum not used");
+ }
+
+ @Override
public boolean hasFullImage() {
return tPermUpdate.isHasfullImage();
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java
index 795aac7..bbf3f13 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/Updateable.java
@@ -40,6 +40,8 @@ public interface Updateable<K extends Updateable.Update> {
long getImgNum();
+ void setImgNum(long imgNum);
+
byte[] serialize() throws IOException;
void deserialize(byte data[]) throws IOException;
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
index 819b6b2..cd97939 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPaths.java
@@ -84,7 +84,11 @@ public class UpdateableAuthzPaths implements AuthzPaths, Updateable<PathsUpdate>
lock.writeLock().lock();
}
seqNum.set(update.getSeqNum());
- imgNum.set(update.getImgNum());
+
+ // Update the image ID only if the update has a new one
+ if (imgNum.get() < update.getImgNum()) {
+ imgNum.set(update.getImgNum());
+ }
LOG.debug("##### Updated paths seq Num [{}] img Num [{}]", seqNum.get(), imgNum.get());
}
} finally {
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
index 5d74421..bb85c13 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/DBUpdateForwarder.java
@@ -103,7 +103,7 @@ class DBUpdateForwarder<K extends Updateable.Update> {
// Checks if newer deltas exist in the persistent storage.
// If there are, return the list of delta updates.
if (seqNum > SEQUENCE_NUMBER_UPDATE_UNINITIALIZED && deltaRetriever.isDeltaAvailable(seqNum)) {
- List<K> deltas = deltaRetriever.retrieveDelta(seqNum);
+ List<K> deltas = deltaRetriever.retrieveDelta(seqNum, imgNum);
if (!deltas.isEmpty()) {
LOGGER.info("Newer delta updates are found up to sequence number: {}", curSeqNum);
return deltas;
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
index fda7455..b2e45f9 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java
@@ -44,7 +44,7 @@ public class PathDeltaRetriever implements DeltaRetriever<PathsUpdate> {
}
@Override
- public List<PathsUpdate> retrieveDelta(long seqNum) throws Exception {
+ public List<PathsUpdate> retrieveDelta(long seqNum, long imgNum) throws Exception {
try (final Context timerContext =
SentryHdfsMetricsUtil.getDeltaPathChangesTimer.time()) {
List<MSentryPathChange> mSentryPathChanges =
@@ -66,6 +66,7 @@ public class PathDeltaRetriever implements DeltaRetriever<PathsUpdate> {
PathsUpdate pathsUpdate = new PathsUpdate();
pathsUpdate.JSONDeserialize(mSentryPathChange.getPathChange());
pathsUpdate.setSeqNum(changeID);
+ pathsUpdate.setImgNum(imgNum);
updates.add(pathsUpdate);
}
return updates;
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
index df6a0b0..6974d37 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java
@@ -44,7 +44,7 @@ public class PermDeltaRetriever implements DeltaRetriever<PermissionsUpdate> {
}
@Override
- public List<PermissionsUpdate> retrieveDelta(long seqNum) throws Exception {
+ public List<PermissionsUpdate> retrieveDelta(long seqNum, long imgNum) throws Exception {
try (final Context timerContext =
SentryHdfsMetricsUtil.getDeltaPermChangesTimer.time()) {
Collection<MSentryPermChange> mSentryPermChanges =
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java
index 8fbc100..830d00e 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDBUpdateForwarder.java
@@ -117,7 +117,7 @@ public class TestDBUpdateForwarder {
Mockito.when(imageRetriever.getLatestImageID()).thenReturn(1L);
Mockito.when(deltaRetriever.getLatestDeltaID()).thenReturn(3L);
Mockito.when(deltaRetriever.isDeltaAvailable(2L)).thenReturn(true);
- Mockito.when(deltaRetriever.retrieveDelta(2L))
+ Mockito.when(deltaRetriever.retrieveDelta(2L, 1L))
.thenReturn(Arrays.asList(new PathsUpdate(3, 1, false)));
List<PathsUpdate> updates = updater.getAllUpdatesFrom(2, 1);
http://git-wip-us.apache.org/repos/asf/sentry/blob/398a8a93/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java
index 63b8caf..60696cc 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java
@@ -44,7 +44,7 @@ public class TestDeltaRetriever {
.thenReturn(Collections.<MSentryPathChange>emptyList());
PathDeltaRetriever deltaRetriever = new PathDeltaRetriever(sentryStoreMock);
- List<PathsUpdate> pathsUpdates = deltaRetriever.retrieveDelta(1);
+ List<PathsUpdate> pathsUpdates = deltaRetriever.retrieveDelta(1, 1);
assertTrue(pathsUpdates.isEmpty());
}
@@ -63,12 +63,14 @@ public class TestDeltaRetriever {
.thenReturn(deltaPathChanges);
deltaRetriever = new PathDeltaRetriever(sentryStoreMock);
- pathsUpdates = deltaRetriever.retrieveDelta(1);
+ pathsUpdates = deltaRetriever.retrieveDelta(1, 3);
assertEquals(2, pathsUpdates.size());
assertEquals(1, pathsUpdates.get(0).getSeqNum());
assertEquals(true, pathsUpdates.get(0).hasFullImage());
+ assertEquals(3, pathsUpdates.get(0).getImgNum());
assertEquals(2, pathsUpdates.get(1).getSeqNum());
assertEquals(false, pathsUpdates.get(1).hasFullImage());
+ assertEquals(3, pathsUpdates.get(1).getImgNum());
}
}