You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ka...@apache.org on 2018/10/09 14:35:54 UTC
sentry git commit: SENTRY-2423: Increase the allocation size for
auto-increment of id's for Snapshot tables. (Kalyan Kumar Kalvagadda reviewed
by Arjun Mishra and Sergio Pena)
Repository: sentry
Updated Branches:
refs/heads/master 2c9a927a9 -> 74676535b
SENTRY-2423: Increase the allocation size for auto-increment of id's for Snapshot tables. (Kalyan Kumar Kalvagadda reviewed by Arjun Mishra and Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/74676535
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/74676535
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/74676535
Branch: refs/heads/master
Commit: 74676535b066b48939ae9ad9c1a1061986450f3d
Parents: 2c9a927
Author: Kalyan Kumar Kalvagadda <kk...@cloudera.com>
Authored: Tue Oct 9 09:32:49 2018 -0500
Committer: Kalyan Kumar Kalvagadda <kk...@cloudera.com>
Committed: Tue Oct 9 09:32:49 2018 -0500
----------------------------------------------------------------------
.../sentry/service/common/ServiceConstants.java | 8 ++++++
.../provider/db/service/model/package.jdo | 4 +--
.../db/service/persistent/SentryStore.java | 3 ++
.../db/service/persistent/TestSentryStore.java | 30 ++++++++++++++++++++
4 files changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java
----------------------------------------------------------------------
diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java
index e90fe2d..092060c 100644
--- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java
+++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java
@@ -256,6 +256,14 @@ public class ServiceConstants {
*/
public static final String SENTRY_DB_EXPLICIT_GRANTS_PERMITTED = "sentry.db.explicit.grants.permitted";
public static final String SENTRY_DB_EXPLICIT_GRANTS_PERMITTED_DEFAULT = "";
+
+ /**
+ * This value sets the allocation size used by datanucleus for the values it auto generates.
+ * This is used when the strategy is explicitly mentioned in JDO. With native(default) this configuration
+ * is not used.
+ */
+ public static final String SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE = "sentry.db.valuegeneration.allocation.size";
+ public static final int SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE_DEFAULT = 100;
}
public static final String SENTRY_ZK_JAAS_NAME = "Sentry";
http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
index 6539e33..20ec0de 100644
--- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
@@ -256,7 +256,7 @@
</class>
<class name="MAuthzPathsMapping" identity-type="datastore" table="AUTHZ_PATHS_MAPPING" detachable="true">
- <datastore-identity>
+ <datastore-identity strategy="increment">
<column name="AUTHZ_OBJ_ID"/>
</datastore-identity>
<index name="AUTHZ_SNAPSHOT_ID_INDEX" unique="false">
@@ -288,7 +288,7 @@
</class>
<class name="MPath" identity-type="datastore" table="AUTHZ_PATH" detachable="true">
- <datastore-identity>
+ <datastore-identity strategy="increment">
<column name="PATH_ID"/>
</datastore-identity>
<field name="path">
http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index 1722109..7a736ca 100644
--- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -249,6 +249,9 @@ public class SentryStore implements SentryStoreInterface {
// Disallow operations outside of transactions
prop.setProperty("datanucleus.NontransactionalRead", "false");
prop.setProperty("datanucleus.NontransactionalWrite", "false");
+ int allocationSize = conf.getInt(ServerConfig.SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE, ServerConfig.
+ SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE_DEFAULT);
+ prop.setProperty("datanucleus.valuegeneration.increment.allocationSize", Integer.toString(allocationSize));
return prop;
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index a299e00..4a9afe3 100644
--- a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -4571,6 +4571,36 @@ public class TestSentryStore extends org.junit.Assert {
assertEquals(0, allPrivileges.get(USER3).size());
}
+ @Test
+ public void testPersistFullPathsImageWithHugeData() throws Exception {
+ Map<String, Collection<String>> authzPaths = new HashMap<>();
+ String[] prefixes = {"/user/hive/warehouse"};
+ // Makes sure that authorizable object could be associated
+ // with different paths and can be properly persisted into database.
+ for(int db_index = 1 ; db_index <= 10; db_index++) {
+ String db_name = "db" + db_index;
+ for( int table_index = 1; table_index <= 15; table_index++) {
+ Set<String> paths = Sets.newHashSet();
+ String table_name = "tb" + table_index;
+ String location = "/u/h/w/" + db_name + "/" + table_name;
+ for (int part_index = 1; part_index <= 30; part_index++) {
+ paths.add(location + "/" + part_index);
+ }
+ authzPaths.put(db_name+table_name, paths);
+ }
+ }
+ long notificationID = 110000;
+ sentryStore.persistFullPathsImage(authzPaths, notificationID);
+ PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes);
+ long savedNotificationID = sentryStore.getLastProcessedNotificationID();
+ assertEquals(1, pathsUpdate.getImgNum());
+ TPathsDump pathDump = pathsUpdate.toThrift().getPathsDump();
+ assertNotNull(pathDump);
+ Map<Integer, TPathEntry> nodeMap = pathDump.getNodeMap();
+ assertTrue(nodeMap.size() > 0);
+ assertEquals(notificationID, savedNotificationID);
+ }
+
private TSentryPrivilege toTSentryPrivilege(String action, String scope, String server,
String dbName, String tableName) {
TSentryPrivilege privilege = new TSentryPrivilege();