You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by va...@apache.org on 2017/06/07 01:42:24 UTC
sentry git commit: SENTRY-1795: Delta tables should not have holes
(Lei(Eddy) Xu, Reviewed by: Alex Kolbasov, Vamsee Yarlagadda)
Repository: sentry
Updated Branches:
refs/heads/sentry-ha-redesign 732561bdd -> 768f46030
SENTRY-1795: Delta tables should not have holes (Lei(Eddy) Xu, Reviewed by: Alex Kolbasov, Vamsee Yarlagadda)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/768f4603
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/768f4603
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/768f4603
Branch: refs/heads/sentry-ha-redesign
Commit: 768f460303df44db38bf7a2218648b39e07b498c
Parents: 732561b
Author: Vamsee Yarlagadda <va...@cloudera.com>
Authored: Tue Jun 6 18:38:53 2017 -0700
Committer: Vamsee Yarlagadda <va...@cloudera.com>
Committed: Tue Jun 6 18:38:53 2017 -0700
----------------------------------------------------------------------
.../sentry/provider/db/service/model/MSentryPathChange.java | 6 ++----
.../sentry/provider/db/service/model/MSentryPermChange.java | 5 ++---
.../org/apache/sentry/provider/db/service/model/package.jdo | 4 ++--
.../provider/db/service/persistent/DeltaTransactionBlock.java | 6 ++++--
.../sentry/provider/db/service/persistent/TestSentryStore.java | 3 ++-
5 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/768f4603/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
index d11f37f..58878e5 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
@@ -68,14 +68,12 @@ public class MSentryPathChange implements MSentryChange {
private long createTimeMs;
private long notificationID;
- public MSentryPathChange(PathsUpdate pathChange) throws TException {
+ public MSentryPathChange(long changeID, PathsUpdate pathChange) throws TException {
// Each PathsUpdate maps to a MSentryPathChange object.
// The PathsUpdate is generated from a HMS notification log,
// the notification ID is stored as seqNum and
// the notification update is serialized as JSON string.
- //
- // See SENTRY-1643. changeID is set after increasing 1 of the "max(changeID)" fetched from
- // the table, to avoid holes between changeIDs. it is subjected to change.
+ this.changeID = changeID;
this.notificationID = pathChange.getSeqNum();
this.pathChange = pathChange.JSONSerialize();
this.createTimeMs = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/sentry/blob/768f4603/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
index 1cb1a1f..e29e780 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
@@ -65,9 +65,8 @@ public class MSentryPermChange implements MSentryChange {
private String permChange;
private long createTimeMs;
- public MSentryPermChange(PermissionsUpdate permChange) throws TException {
- // See SENTRY-1643. changeID is set after increasing 1 of the "max(changeID)" fetched from
- // the table, to avoid holes between changeIDs. it is subjected to change.
+ public MSentryPermChange(long changeID, PermissionsUpdate permChange) throws TException {
+ this.changeID = changeID;
this.permChange = permChange.JSONSerialize();
this.createTimeMs = System.currentTimeMillis();
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/768f4603/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
index 96ab462..459939b 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
@@ -271,7 +271,7 @@
</class>
<class name="MSentryPermChange" table="SENTRY_PERM_CHANGE" identity-type="application" detachable="true">
- <field name="changeID" primary-key="true" value-strategy="increment" key-cache-size="1">
+ <field name="changeID" primary-key="true">
<column name="CHANGE_ID" jdbc-type="BIGINT" allows-null="false"/>
</field>
<field name ="permChange">
@@ -283,7 +283,7 @@
</class>
<class name="MSentryPathChange" table="SENTRY_PATH_CHANGE" identity-type="application" detachable="true">
- <field name="changeID" primary-key="true" value-strategy="increment" key-cache-size="1">
+ <field name="changeID" primary-key="true">
<column name="CHANGE_ID" jdbc-type="BIGINT" allows-null="false"/>
</field>
<field name="notificationID">
http://git-wip-us.apache.org/repos/asf/sentry/blob/768f4603/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java
index 77282da..4fe62bf 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java
@@ -85,9 +85,11 @@ public class DeltaTransactionBlock implements TransactionBlock<Object> {
// changeID is trying to be persisted twice, the transaction would
// fail.
if (update instanceof PermissionsUpdate) {
- pm.makePersistent(new MSentryPermChange((PermissionsUpdate) update));
+ long lastChangeID = SentryStore.getLastProcessedChangeIDCore(pm, MSentryPermChange.class);
+ pm.makePersistent(new MSentryPermChange(lastChangeID + 1, (PermissionsUpdate) update));
} else if (update instanceof PathsUpdate) {
- pm.makePersistent(new MSentryPathChange((PathsUpdate) update));
+ long lastChangeID = SentryStore.getLastProcessedChangeIDCore(pm, MSentryPathChange.class);
+ pm.makePersistent(new MSentryPathChange(lastChangeID + 1, (PathsUpdate) update));
// Notification id from PATH_UPDATE entry is made persistent in
// SENTRY_LAST_NOTIFICATION_ID table.
pm.makePersistent(new MSentryHmsNotification(update.getSeqNum()));
http://git-wip-us.apache.org/repos/asf/sentry/blob/768f4603/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index 23b685b..788062c 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -120,6 +120,7 @@ public class TestSentryStore extends org.junit.Assert {
policyFilePath = new File(dataDir, "local_policy_file.ini");
conf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE,
policyFilePath.getPath());
+ conf.setInt(ServerConfig.SENTRY_STORE_TRANSACTION_RETRY, 10);
sentryStore = new SentryStore(conf);
}
@@ -2960,7 +2961,7 @@ public class TestSentryStore extends org.junit.Assert {
new PermissionsUpdate(seqNumGenerator.getAndIncrement(), false);
tbs.add(new DeltaTransactionBlock(update));
try {
- tm.executeTransaction(tbs);
+ tm.executeTransactionBlocksWithRetry(tbs);
} catch (Exception e) {
LOGGER.error("Failed to execute permission update transaction", e);
fail(String.format("Transaction failed: %s", e.getMessage()));