You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ha...@apache.org on 2016/06/24 22:19:50 UTC
sentry git commit: SENTRY-1325: Store HMSPaths in Sentry DB to allow
fast failover (Hao Hao, Reviewed by Colin Patrick McCabe and Anne Yu)
Repository: sentry
Updated Branches:
refs/heads/sentry-ha-redesign 5630fc5ce -> e7ec257a7
SENTRY-1325: Store HMSPaths in Sentry DB to allow fast failover (Hao Hao, Reviewed by Colin Patrick McCabe and Anne Yu)
Change-Id: Idc4bb72436b6b4be098496ef074a05a9390ae135
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/e7ec257a
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/e7ec257a
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/e7ec257a
Branch: refs/heads/sentry-ha-redesign
Commit: e7ec257a76c7175a49e95b17ca4676dbd74a8d67
Parents: 5630fc5
Author: hahao <ha...@cloudera.com>
Authored: Fri Jun 24 15:18:24 2016 -0700
Committer: hahao <ha...@cloudera.com>
Committed: Fri Jun 24 15:18:24 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/sentry/hdfs/HMSPaths.java | 93 +++++++++++++----
.../org/apache/sentry/hdfs/HMSPathsDumper.java | 2 +-
.../db/service/model/MAuthzPathsMapping.java | 102 +++++++++++++++++++
.../provider/db/service/model/package.jdo | 25 +++++
.../db/service/persistent/SentryStore.java | 71 +++++++++++++
.../db/service/persistent/TestSentryStore.java | 12 +++
6 files changed, 287 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
index 6d2ab23..e4850f4 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPaths.java
@@ -116,7 +116,7 @@ public class HMSPaths implements AuthzPaths {
// Path of child element to the path entry mapping.
// e.g. 'b' -> '/a/b'
- private final Map<String, Entry> children;
+ private final Map<String, Entry> children = new HashMap<String, Entry>();
Entry(Entry parent, String pathElement, EntryType type,
String authzObj) {
@@ -125,7 +125,6 @@ public class HMSPaths implements AuthzPaths {
this.pathElement = pathElement;
this.authzObjs = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
addAuthzObj(authzObj);
- children = new HashMap<String, Entry>();
}
Entry(Entry parent, String pathElement, EntryType type,
@@ -135,7 +134,6 @@ public class HMSPaths implements AuthzPaths {
this.pathElement = pathElement;
this.authzObjs = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
addAuthzObjs(authzObjs);
- children = new HashMap<String, Entry>();
}
// Get all the mapping of the children element to
@@ -179,6 +177,66 @@ public class HMSPaths implements AuthzPaths {
getFullPath(), type, Joiner.on(",").join(authzObjs));
}
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+
+ Entry other = (Entry) obj;
+ if (parent == null) {
+ if (other.parent != null) {
+ return false;
+ }
+ } else if (!parent.equals(other.parent)) {
+ return false;
+ }
+
+ if (type == null) {
+ if (other.type != null) {
+ return false;
+ }
+ } else if (!type.equals(other.type)) {
+ return false;
+ }
+
+ if (pathElement == null) {
+ if (other.pathElement != null) {
+ return false;
+ }
+ } else if (!pathElement.equals(other.pathElement)) {
+ return false;
+ }
+
+ if (authzObjs == null) {
+ if (other.authzObjs != null) {
+ return false;
+ }
+ } else if (!authzObjs.equals(other.authzObjs)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((parent == null) ? 0 : parent.hashCode());
+ result = prime * result + ((type == null) ? 0 : type.hashCode());
+ result = prime * result + ((pathElement == null) ? 0 : pathElement.hashCode());
+ result = prime * result + ((authzObjs == null) ? 0 : authzObjs.hashCode());
+
+ return result;
+ }
+
/**
* Create a child entry based on the path, type and authzObj that
* associates with it.
@@ -332,12 +390,10 @@ public class HMSPaths implements AuthzPaths {
return authzObjs;
}
-
-
public Entry findPrefixEntry(List<String> pathElements) {
Preconditions.checkArgument(pathElements != null,
"pathElements cannot be NULL");
- return (getType() == EntryType.PREFIX)
+ return (getType() == EntryType.PREFIX)
? this : findPrefixEntry(pathElements, 0);
}
@@ -413,7 +469,10 @@ public class HMSPaths implements AuthzPaths {
// The hive authorized objects to path entries mapping.
// One authorized object can map to a set of path entries.
- private Map<String, Set<Entry>> authzObjToPath;
+ private Map<String, Set<Entry>> authzObjToEntries;
+
+ public HMSPaths() {
+ }
public HMSPaths(String[] pathPrefixes) {
boolean rootPrefix = false;
@@ -433,7 +492,7 @@ public class HMSPaths implements AuthzPaths {
}
}
- authzObjToPath = new TreeMap<String, Set<Entry>>(String.CASE_INSENSITIVE_ORDER);
+ authzObjToEntries = new TreeMap<String, Set<Entry>>(String.CASE_INSENSITIVE_ORDER);
}
void _addAuthzObject(String authzObj, List<String> authzObjPaths) {
@@ -441,7 +500,7 @@ public class HMSPaths implements AuthzPaths {
}
void addAuthzObject(String authzObj, List<List<String>> authzObjPathElements) {
- Set<Entry> previousEntries = authzObjToPath.get(authzObj);
+ Set<Entry> previousEntries = authzObjToEntries.get(authzObj);
Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size());
for (List<String> pathElements : authzObjPathElements) {
Entry e = root.createAuthzObjPath(pathElements, authzObj);
@@ -451,7 +510,7 @@ public class HMSPaths implements AuthzPaths {
LOG.warn("Ignoring path, no prefix");
}
}
- authzObjToPath.put(authzObj, newEntries);
+ authzObjToEntries.put(authzObj, newEntries);
if (previousEntries != null) {
previousEntries.removeAll(newEntries);
if (!previousEntries.isEmpty()) {
@@ -464,7 +523,7 @@ public class HMSPaths implements AuthzPaths {
void addPathsToAuthzObject(String authzObj,
List<List<String>> authzObjPathElements, boolean createNew) {
- Set<Entry> entries = authzObjToPath.get(authzObj);
+ Set<Entry> entries = authzObjToEntries.get(authzObj);
if (entries != null) {
Set<Entry> newEntries = new HashSet<Entry>(authzObjPathElements.size());
for (List<String> pathElements : authzObjPathElements) {
@@ -501,7 +560,7 @@ public class HMSPaths implements AuthzPaths {
*/
void deletePathsFromAuthzObject(String authzObj,
List<List<String>> authzObjPathElements) {
- Set<Entry> entries = authzObjToPath.get(authzObj);
+ Set<Entry> entries = authzObjToEntries.get(authzObj);
if (entries != null) {
Set<Entry> toDelEntries = new HashSet<Entry>(authzObjPathElements.size());
for (List<String> pathElements : authzObjPathElements) {
@@ -522,7 +581,7 @@ public class HMSPaths implements AuthzPaths {
}
void deleteAuthzObject(String authzObj) {
- Set<Entry> entries = authzObjToPath.remove(authzObj);
+ Set<Entry> entries = authzObjToEntries.remove(authzObj);
if (entries != null) {
for (Entry entry : entries) {
entry.deleteAuthzObject(authzObj);
@@ -593,12 +652,12 @@ public class HMSPaths implements AuthzPaths {
}
}
if(samePaths) {
- Set<Entry> eSet = authzObjToPath.get(oldName);
+ Set<Entry> eSet = authzObjToEntries.get(oldName);
if (eSet == null) {
LOG.warn("Unexpected state in renameAuthzObject, cannot find oldName in authzObjToPath: oldName=" + oldName + " newName=" + newName +
" oldPath=" + oldPathElems + " newPath=" + newPathElems);
} else {
- authzObjToPath.put(newName, eSet);
+ authzObjToEntries.put(newName, eSet);
for (Entry e : eSet) {
if (e.getAuthzObjs().contains(oldName)) {
e.addAuthzObj(newName);
@@ -632,8 +691,8 @@ public class HMSPaths implements AuthzPaths {
this.root = root;
}
- void setAuthzObjToPathMapping(Map<String, Set<Entry>> mapping) {
- authzObjToPath = mapping;
+ void setAuthzObjToEntryMapping(Map<String, Set<Entry>> mapping) {
+ authzObjToEntries = mapping;
}
@Override
http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java
index e759ff1..7e56952 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/HMSPathsDumper.java
@@ -86,7 +86,7 @@ public class HMSPathsDumper implements AuthzPathsDumper<HMSPaths> {
cloneToEntry(tRootEntry, rootEntry, pathDump.getNodeMap(), authzObjToPath,
rootEntry.getType() == EntryType.PREFIX);
newHmsPaths.setRootEntry(rootEntry);
- newHmsPaths.setAuthzObjToPathMapping(authzObjToPath);
+ newHmsPaths.setAuthzObjToEntryMapping(authzObjToPath);
return newHmsPaths;
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java
new file mode 100644
index 0000000..56e456e
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MAuthzPathsMapping.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.provider.db.service.model;
+
+import javax.jdo.annotations.PersistenceCapable;
+import java.util.Set;
+
+/**
+ * Transactional database backend for storing HMS Paths Updates. Any changes to this object
+ * require re-running the maven build so DN can re-enhance.
+ */
+@PersistenceCapable
+public class MAuthzPathsMapping {
+
+ private String authzObjName;
+ private Set<String> paths;
+ private long createTimeMs;
+
+ public MAuthzPathsMapping(String authzObjName, Set<String> paths, long createTimeMs) {
+ this.authzObjName = authzObjName;
+ this.paths = paths;
+ this.createTimeMs = createTimeMs;
+ }
+
+ public long getCreateTime() {
+ return createTimeMs;
+ }
+
+ public void setCreateTime(long createTime) {
+ this.createTimeMs = createTime;
+ }
+
+ public String getAuthzObjName() {
+ return authzObjName;
+ }
+
+ public void setAuthzObjName(String authzObjName) {
+ this.authzObjName = authzObjName;
+ }
+
+ public void setPaths(Set<String> paths) {
+ this.paths = paths;
+ }
+
+ public Set<String> getPaths() {
+ return paths;
+ }
+
+ @Override
+ public String toString() {
+ return "MSentryPathsUpdate [authzObj=" + authzObjName + "], paths=[" + paths.toString()
+ + "], createTimeMs=[" + String.valueOf(createTimeMs) + "]";
+ }
+
+ @Override
+ public int hashCode() {
+ return authzObjName == null ? 0 : authzObjName.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+
+ if (obj == null) {
+ return false;
+ }
+
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+
+ MAuthzPathsMapping other = (MAuthzPathsMapping) obj;
+
+ if (authzObjName == null) {
+ if (other.authzObjName != null) {
+ return false;
+ }
+ } else if (!authzObjName.equals(other.authzObjName)) {
+ return false;
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
index b3b9494..fb5470f 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo
@@ -237,6 +237,31 @@
</field>
</class>
+ <class name="MAuthzPathsMapping" identity-type="datastore" table="AUTHZ_PATHS_MAPPING" detachable="true">
+ <datastore-identity>
+ <column name="AUTHZ_OBJ_ID"/>
+ </datastore-identity>
+ <!--
+ authzObjName is composed by hive database name, and table name. e.g. "default.tb1". Since
+ both hive database name, and table name have restrictions to be at most 128 characters long,
+ 384 characters length should be enough for AUTHZ_OBJ_NAM.
+ -->
+ <field name="authzObjName">
+ <column name="AUTHZ_OBJ_NAME" length="384" jdbc-type="VARCHAR"/>
+ <index name="AuthzObjName" unique="true"/>
+ </field>
+ <field name="createTimeMs">
+ <column name="CREATE_TIME_MS" jdbc-type="BIGINT"/>
+ </field>
+ <field name = "paths">
+ <collection element-type="java.lang.String"/>
+ <join/>
+ <element>
+ <column name="PATHS"/>
+ </element>
+ </field>
+ </class>
+
</package>
</jdo>
http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
index b7ef0e9..7dad496 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java
@@ -56,6 +56,7 @@ import org.apache.sentry.core.common.exception.SentryAlreadyExistsException;
import org.apache.sentry.core.common.exception.SentryGrantDeniedException;
import org.apache.sentry.core.common.exception.SentryInvalidInputException;
import org.apache.sentry.core.common.exception.SentryNoSuchObjectException;
+import org.apache.sentry.provider.db.service.model.MAuthzPathsMapping;
import org.apache.sentry.provider.db.service.model.MSentryGroup;
import org.apache.sentry.provider.db.service.model.MSentryPrivilege;
import org.apache.sentry.provider.db.service.model.MSentryRole;
@@ -1986,6 +1987,76 @@ public class SentryStore {
}
/**
+ * This returns a Mapping of Authz -> [Paths]
+ */
+ public Map<String, Set<String>> retrieveFullPathsImage() {
+ Map<String, Set<String>> retVal = new HashMap<>();
+ boolean rollbackTransaction = true;
+ PersistenceManager pm = null;
+
+ try {
+ pm = openTransaction();
+ Query query = pm.newQuery(MAuthzPathsMapping.class);
+ List<MAuthzPathsMapping> authzToPathsMappings = (List<MAuthzPathsMapping>) query.execute();
+ for (MAuthzPathsMapping authzToPaths : authzToPathsMappings) {
+ retVal.put(authzToPaths.getAuthzObjName(), authzToPaths.getPaths());
+ }
+
+ rollbackTransaction = false;
+ commitTransaction(pm);
+ return retVal;
+ } finally {
+ if (rollbackTransaction) {
+ rollbackTransaction(pm);
+ }
+ }
+ }
+
+ public CommitContext createAuthzPathsMapping(String hiveObj,
+ Set<String> paths) throws SentryNoSuchObjectException, SentryAlreadyExistsException {
+
+ boolean rollbackTransaction = true;
+ PersistenceManager pm = null;
+
+ try {
+ pm = openTransaction();
+ createAuthzPathsMappingCore(pm, hiveObj, paths);
+ CommitContext commit = commitUpdateTransaction(pm);
+ rollbackTransaction = false;
+ return commit;
+ } finally {
+ if (rollbackTransaction) {
+ rollbackTransaction(pm);
+ }
+ }
+ }
+
+ private void createAuthzPathsMappingCore(PersistenceManager pm, String authzObj,
+ Set<String> paths) throws SentryAlreadyExistsException {
+
+ MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMapping(pm, authzObj);
+
+ if (mAuthzPathsMapping == null) {
+ mAuthzPathsMapping =
+ new MAuthzPathsMapping(authzObj, paths, System.currentTimeMillis());
+ pm.makePersistent(mAuthzPathsMapping);
+ } else {
+ throw new SentryAlreadyExistsException("AuthzObj: " + authzObj);
+ }
+ }
+
+ /**
+ * Get the MAuthzPathsMapping object from authzObj
+ */
+ public MAuthzPathsMapping getMAuthzPathsMapping(PersistenceManager pm, String authzObj) {
+ Query query = pm.newQuery(MAuthzPathsMapping.class);
+ query.setFilter("this.authzObjName == t");
+ query.declareParameters("java.lang.String t");
+ query.setUnique(true);
+ return (MAuthzPathsMapping) query.execute(authzObj);
+ }
+
+ /**
* This thread exists to clean up "orphaned" privilege rows in the database.
* These rows aren't removed automatically due to the fact that there is
* a many-to-many mapping between the roles and privileges, and the
http://git-wip-us.apache.org/repos/asf/sentry/blob/e7ec257a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
index bc7fe12..3ef1cf7 100644
--- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
+++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java
@@ -2079,6 +2079,18 @@ public class TestSentryStore extends org.junit.Assert {
}
}
+ @Test
+ public void testAuthzPathsMapping() throws Exception {
+ long seqId = sentryStore.createAuthzPathsMapping("db1.table1", Sets.newHashSet("/user/hive/warehouse/db1.db/table1")).getSequenceId();
+ long actualSeqId = sentryStore.createAuthzPathsMapping("db1.table2", Sets.newHashSet("/user/hive/warehouse/db1.db/table2")).getSequenceId();
+ assertEquals(seqId + 1, actualSeqId);
+
+ Map<String, Set<String>> pathsImage = sentryStore.retrieveFullPathsImage();
+ assertEquals(2, pathsImage.size());
+ assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathsImage.get("db1.table1"));
+ }
+
+
protected static void addGroupsToUser(String user, String... groupNames) {
policyFile.addGroupsToUser(user, groupNames);
}