You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ha...@apache.org on 2017/02/22 19:30:45 UTC
sentry git commit: SENTRY-1611: Purge MSentryPerm/PathChange tables
(Lei (Eddy) Xu, Reviewed by: Hao Hao and Alexander Kolbasov)
Repository: sentry
Updated Branches:
refs/heads/sentry-ha-redesign 3817f18be -> b596fe9b8
SENTRY-1611: Purge MSentryPerm/PathChange tables (Lei (Eddy) Xu, Reviewed by: Hao Hao and Alexander Kolbasov)
Change-Id: I59ec5b83abd7f86cadc434e46caf9be142dd3ab6
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/b596fe9b
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/b596fe9b
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/b596fe9b
Branch: refs/heads/sentry-ha-redesign
Commit: b596fe9b884db583215a58a3309677590969b71f
Parents: 3817f18
Author: hahao <ha...@cloudera.com>
Authored: Wed Feb 22 11:28:49 2017 -0800
Committer: hahao <ha...@cloudera.com>
Committed: Wed Feb 22 11:28:49 2017 -0800
----------------------------------------------------------------------
.../db/service/model/MSentryChange.java | 24 ++++
.../db/service/model/MSentryPathChange.java | 2 +-
.../db/service/model/MSentryPermChange.java | 2 +-
.../db/service/persistent/SentryStore.java | 117 ++++++++++++++++---
.../db/service/persistent/TestSentryStore.java | 33 ++++++
5 files changed, 162 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java
new file mode 100644
index 0000000..6011ef4
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryChange.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.provider.db.service.model;
+
+/**
+ * The base class for various delta changes stored in Sentry DB.
+ */
+public interface MSentryChange {
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
index 12e9a09..0ca7fe2 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java
@@ -57,7 +57,7 @@ import javax.jdo.annotations.PrimaryKey;
*/
@PersistenceCapable
-public class MSentryPathChange {
+public class MSentryPathChange implements MSentryChange {
@PrimaryKey
private long changeID;
http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
index 3662bfd..476fbcb 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java
@@ -55,7 +55,7 @@ import javax.jdo.annotations.PrimaryKey;
* can re-enhance.
*/
@PersistenceCapable
-public class MSentryPermChange {
+public class MSentryPermChange implements MSentryChange {
@PrimaryKey
private long changeID;
http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index 4cc123f..2ae9f3b 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -425,6 +425,8 @@ public class SentryStore {
pm.newQuery(MSentryGroup.class).deletePersistentAll();
pm.newQuery(MSentryUser.class).deletePersistentAll();
pm.newQuery(MSentryPrivilege.class).deletePersistentAll();
+ pm.newQuery(MSentryPermChange.class).deletePersistentAll();
+ pm.newQuery(MSentryPathChange.class).deletePersistentAll();
return null;
}
});
@@ -435,6 +437,56 @@ public class SentryStore {
}
/**
+ * Purge a given delta change table, with a specified number of changes to be kept.
+ *
+ * @param cls the class of a perm/path delta change {@link MSentryPermChange} or
+ * {@link MSentryPathChange}.
+ * @param pm a {@link PersistenceManager} instance.
+ * @param changesToKeep the number of changes the caller want to keep.
+ * @param <T> the type of delta change class.
+ */
+ @VisibleForTesting
+ <T extends MSentryChange> void purgeDeltaChangeTableCore(
+ Class<T> cls, PersistenceManager pm, long changesToKeep) {
+ Preconditions.checkArgument(changesToKeep >= 0,
+ "changes to keep must be a non-negative number");
+ long lastChangedID = getLastProcessedChangeIDCore(pm, cls);
+ long maxIDDeleted = lastChangedID - changesToKeep;
+
+ Query query = pm.newQuery(cls);
+
+ // It is an approximation of "SELECT ... LIMIT CHANGE_TO_KEEP" in SQL, because JDO w/ derby
+ // does not support "LIMIT".
+ // See: http://www.datanucleus.org/products/datanucleus/jdo/jdoql_declarative.html
+ query.setFilter("changeID <= maxChangedIdDeleted");
+ query.declareParameters("long maxChangedIdDeleted");
+ long numDeleted = query.deletePersistentAll(maxIDDeleted);
+ LOGGER.info(String.format("Purged %d of %s to changeID=%d",
+ numDeleted, cls.getSimpleName(), maxIDDeleted));
+ }
+
+ /**
+ * Purge delta change tables, {@link MSentryPermChange} and {@link MSentryPathChange}.
+ */
+ public void purgeDeltaChangeTables() {
+ LOGGER.info("Purging MSentryPathUpdate and MSentyPermUpdate tables.");
+ try {
+ tm.executeTransaction(new TransactionBlock<Object>() {
+ @Override
+ public Object execute(PersistenceManager pm) throws Exception {
+ purgeDeltaChangeTableCore(MSentryPermChange.class, pm, 1);
+ LOGGER.info("MSentryPermChange table has been purged.");
+ purgeDeltaChangeTableCore(MSentryPathChange.class, pm, 1);
+ LOGGER.info("MSentryPathUpdate table has been purged.");
+ return null;
+ }
+ });
+ } catch (Exception e) {
+ LOGGER.error("Delta change cleaning process encounter an error.", e);
+ }
+ }
+
+ /**
* Alter a given sentry role to grant a privilege.
*
* @param grantorPrincipal User name
@@ -3289,34 +3341,34 @@ public class SentryStore {
}
/**
- * Get the last processed perm change ID.
+ * Get the last processed change ID for perm/path delta changes.
*
* @param pm the PersistenceManager
- * @return the last processed perm changedID
+ * @param changeCls the class of a delta c
+ *
+ * @return the last processed changedID for the delta changes.
*/
- private long getLastProcessedPermChangeIDCore(PersistenceManager pm) {
- Query query = pm.newQuery(MSentryPermChange.class);
- query.setResult("max(this.changeID)");
+ private <T extends MSentryChange> long getLastProcessedChangeIDCore(
+ PersistenceManager pm, Class<T> changeCls) {
+ Query query = pm.newQuery(changeCls);
+ query.setResult("max(changeID)");
Long changeID = (Long) query.execute();
- if (changeID == null) {
- return EMPTY_CHANGE_ID;
- } else {
- return changeID;
- }
+ return changeID == null ? EMPTY_CHANGE_ID : changeID;
}
/**
- * Get the MSentryPermChange object by ChangeID. Internally invoke
- * getLastProcessedPermChangeIDCore().
+ * Get the last processed change ID for perm delta changes.
*
- * @return MSentryPermChange
+ * Internally invoke {@link #getLastProcessedChangeIDCore(PersistenceManager, Class)}
+ *
+ * @return latest perm change ID.
*/
@VisibleForTesting
long getLastProcessedPermChangeID() throws Exception {
return tm.executeTransaction(
new TransactionBlock<Long>() {
public Long execute(PersistenceManager pm) throws Exception {
- return getLastProcessedPermChangeIDCore(pm);
+ return getLastProcessedChangeIDCore(pm, MSentryPermChange.class);
}
});
}
@@ -3348,6 +3400,35 @@ public class SentryStore {
}
/**
+ * Fetch all {@link MSentryChange} in the database.
+ *
+ * @param cls the class of the Sentry delta change.
+ * @return a list of Sentry delta changes.
+ * @throws Exception
+ */
+ @SuppressWarnings("unchecked")
+ private <T extends MSentryChange> List<T> getMSentryChanges(final Class<T> cls) throws Exception {
+ return tm.executeTransaction(
+ new TransactionBlock<List<T>>() {
+ public List<T> execute(PersistenceManager pm) throws Exception {
+ Query query = pm.newQuery(cls);
+ return (List<T>) query.execute();
+ }
+ });
+ }
+
+ /**
+ * Fetch all {@link MSentryPermChange} in the database. It should only be used in the tests.
+ *
+ * @return a list of permission changes.
+ * @throws Exception
+ */
+ @VisibleForTesting
+ List<MSentryPermChange> getMSentryPermChanges() throws Exception {
+ return getMSentryChanges(MSentryPermChange.class);
+ }
+
+ /**
* Get the MSentryPathChange object by ChangeID.
*/
public MSentryPathChange getMSentryPathChangeByID(final long changeID) throws Exception {
@@ -3371,6 +3452,14 @@ public class SentryStore {
}
/**
+ * Fetch all {@link MSentryPathChange} in the database. It should only be used in the tests.
+ */
+ @VisibleForTesting
+ List<MSentryPathChange> getMSentryPathChanges() throws Exception {
+ return getMSentryChanges(MSentryPathChange.class);
+ }
+
+ /**
* Execute Perm/Path UpdateTransaction and corresponding actual
* action transaction, e.g dropSentryRole, in a single transaction.
* The order of the transaction does not matter because there is no
http://git-wip-us.apache.org/repos/asf/sentry/blob/b596fe9b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index dfaac15..d9fce69 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -2479,4 +2479,37 @@ public class TestSentryStore extends org.junit.Assert {
policyFile.write(policyFilePath);
}
+ @Test
+ public void testPurgeDeltaChanges() throws Exception {
+ String role = "purgeRole";
+ String grantor = "g1";
+ String table = "purgeTable";
+
+ assertEquals(0, sentryStore.getMSentryPermChanges().size());
+ assertEquals(0, sentryStore.getMSentryPathChanges().size());
+
+ sentryStore.createSentryRole(role);
+
+ final int numPermChanges = 5;
+ for (int i = 0; i < numPermChanges; i++) {
+ TSentryPrivilege privilege = new TSentryPrivilege();
+ privilege.setPrivilegeScope("Column");
+ privilege.setServerName("server");
+ privilege.setDbName("db");
+ privilege.setTableName(table);
+ privilege.setColumnName("column");
+ privilege.setAction(AccessConstants.SELECT);
+ privilege.setCreateTime(System.currentTimeMillis());
+
+ PermissionsUpdate update = new PermissionsUpdate(i + 1, false);
+ sentryStore.alterSentryRoleGrantPrivilege(grantor, role, privilege, update);
+ }
+ assertEquals(numPermChanges, sentryStore.getMSentryPermChanges().size());
+
+ sentryStore.purgeDeltaChangeTables();
+ assertEquals(1, sentryStore.getMSentryPermChanges().size());
+
+ // TODO: verify MSentryPathChange being purged.
+ // assertEquals(1, sentryStore.getMSentryPathChanges().size());
+ }
}