You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ja...@apache.org on 2014/06/10 16:05:42 UTC
git commit: SENTRY-277: Add Pig+HCat test for Metastore auth plugin
Repository: incubator-sentry
Updated Branches:
refs/heads/master c57c53179 -> 3071da2fc
SENTRY-277: Add Pig+HCat test for Metastore auth plugin
(Prasad Mujumdar via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/3071da2f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/3071da2f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/3071da2f
Branch: refs/heads/master
Commit: 3071da2fca0a5e9123622334cdab8de22c5cee89
Parents: c57c531
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Tue Jun 10 07:04:59 2014 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Tue Jun 10 07:04:59 2014 -0700
----------------------------------------------------------------------
pom.xml | 16 +++
.../metastore/MetastoreAuthzBinding.java | 18 ++-
sentry-tests/sentry-tests-hive/pom.xml | 15 +++
.../apache/sentry/tests/e2e/hive/Context.java | 25 +++-
...actMetastoreTestWithStaticConfiguration.java | 31 +++++
.../e2e/metastore/TestMetaStoreWithPigHCat.java | 113 +++++++++++++++++++
6 files changed, 211 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 564a2ad..05943c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,6 +81,7 @@ limitations under the License.
<solr.version>4.8.0</solr.version>
<solr.sentry.handlers.version>4.4.0-cdh5.1.0-SNAPSHOT</solr.sentry.handlers.version>
<zookeeper.version>3.4.5-cdh5.1.0-SNAPSHOT</zookeeper.version>
+ <pig.version>0.12.0-cdh5.1.0-SNAPSHOT</pig.version>
</properties>
<dependencyManagement>
@@ -126,6 +127,16 @@ limitations under the License.
<version>${hive.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-core</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-pig-adapter</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
@@ -363,6 +374,11 @@ limitations under the License.
<artifactId>mockito-all</artifactId>
<version>${mockito.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ <version>${pig.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java
index 2737793..197017b 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/MetastoreAuthzBinding.java
@@ -240,13 +240,21 @@ public class MetastoreAuthzBinding extends MetaStorePreEventListener {
}
private void authorizeAddPartition(PreAddPartitionEvent context)
- throws InvalidOperationException, MetaException {
- // check if we need to validate URI permissions when storage location is
- // non-default
+ throws InvalidOperationException, MetaException, NoSuchObjectException {
HierarcyBuilder inputBuilder = new HierarcyBuilder();
- if (!StringUtils.isEmpty(context.getPartition().getSd().getLocation())) {
- inputBuilder.addUriToOutput(getAuthServer(), context.getPartition()
+
+ // check if we need to validate URI permissions when storage location is
+ // non-default, ie something not under the parent table
+ String partitionLocation = context.getPartition().getSd().getLocation();
+ if (!StringUtils.isEmpty(partitionLocation)) {
+ String tableLocation = context
+ .getHandler()
+ .get_table(context.getPartition().getDbName(),
+ context.getPartition().getTableName()).getSd().getLocation();
+ if (!partitionLocation.startsWith(tableLocation)) {
+ inputBuilder.addUriToOutput(getAuthServer(), context.getPartition()
.getSd().getLocation());
+ }
}
authorizeMetastoreAccess(HiveOperation.ALTERTABLE_ADDPARTS,
inputBuilder.build(),
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-tests/sentry-tests-hive/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/pom.xml b/sentry-tests/sentry-tests-hive/pom.xml
index 03802a3..cd95252 100644
--- a/sentry-tests/sentry-tests-hive/pom.xml
+++ b/sentry-tests/sentry-tests-hive/pom.xml
@@ -102,6 +102,16 @@ limitations under the License.
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-pig-adapter</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>test</scope>
@@ -173,6 +183,11 @@ limitations under the License.
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java
index 83fe397..4985912 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/Context.java
@@ -43,6 +43,8 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -251,6 +253,20 @@ public class Context {
return client;
}
+ public PigServer getPigServer(String userName, final ExecType exType)
+ throws Exception {
+ UserGroupInformation clientUgi = UserGroupInformation
+ .createRemoteUser(userName);
+ PigServer pigServer = (PigServer) ShimLoader.getHadoopShims().doAs(
+ clientUgi, new PrivilegedExceptionAction<Object>() {
+ @Override
+ public PigServer run() throws Exception {
+ return new PigServer(exType, new HiveConf());
+ }
+ });
+ return pigServer;
+ }
+
/**
* Execute "set x" and extract value from key=val format result Verify the
* extracted value
@@ -270,8 +286,13 @@ public class Context {
resultValues[1]);
}
- public static void verifyMetastoreAuthException(MetaException e)
+ public static void verifyMetastoreAuthException(Throwable e)
throws Exception {
- assertTrue(e.getMessage().contains(METASTORE_AUTH_ERROR_MSG));
+ if (e instanceof MetaException) {
+ assertTrue(e.getMessage().contains(METASTORE_AUTH_ERROR_MSG));
+ } else {
+ throw new Exception("Excepted MetaException but got "
+ + e.getClass().getName(), e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
index 2e70b14..fe10f5d 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java
@@ -17,10 +17,13 @@
*/
package org.apache.sentry.tests.e2e.metastore;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -29,7 +32,12 @@ import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.pig.PigServer;
import org.apache.sentry.tests.e2e.dbprovider.PolicyProviderForTest;
import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration;
import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory.HiveServer2Type;
@@ -153,4 +161,27 @@ public abstract class AbstractMetastoreTestWithStaticConfiguration extends
client.createDatabase(db);
}
+ public void execHiveSQL(String sqlStmt, String userName) throws Exception {
+ HiveConf hiveConf = new HiveConf();
+ Driver driver = new Driver(hiveConf);
+ SessionState.start(new CliSessionState(hiveConf));
+ driver.run(sqlStmt);
+ driver.close();
+ SessionState.get().close();
+ }
+
+ public void execPigLatin(String userName, final PigServer pigServer,
+ final String pigLatin) throws Exception {
+ UserGroupInformation clientUgi = UserGroupInformation
+ .createRemoteUser(userName);
+ ShimLoader.getHadoopShims().doAs(clientUgi,
+ new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Void run() throws Exception {
+ pigServer.registerQuery(pigLatin);
+ return null;
+ }
+ });
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/3071da2f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetaStoreWithPigHCat.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetaStoreWithPigHCat.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetaStoreWithPigHCat.java
new file mode 100644
index 0000000..46feace
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetaStoreWithPigHCat.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.metastore;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hive.hcatalog.pig.HCatStorer;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.sentry.tests.e2e.dbprovider.PolicyProviderForTest;
+import org.apache.sentry.tests.e2e.hive.Context;
+import org.apache.sentry.tests.e2e.hive.StaticUserGroup;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+
+public class TestMetaStoreWithPigHCat extends
+ AbstractMetastoreTestWithStaticConfiguration {
+ private PolicyProviderForTest policyFile;
+ private File dataFile;
+ private static final String dbName = "db_1";
+ private static final String db_all_role = "all_db1";
+ private static final String uri_role = "uri_role";
+
+ @Before
+ public void setup() throws Exception {
+ context = createContext();
+ dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME);
+ FileOutputStream to = new FileOutputStream(dataFile);
+ Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to);
+ to.close();
+
+ policyFile = PolicyProviderForTest.setAdminOnServer1(ADMINGROUP);
+ policyFile
+ .addRolesToGroup(USERGROUP1, db_all_role)
+ .addRolesToGroup(USERGROUP2, "read_db_role")
+ .addPermissionsToRole(db_all_role, "server=server1->db=" + dbName)
+ .addPermissionsToRole("read_db_role",
+ "server=server1->db=" + dbName + "->table=*->action=SELECT")
+ .setUserGroupMapping(StaticUserGroup.getStaticMapping());
+ writePolicyFile(policyFile);
+
+ HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
+ client.dropDatabase(dbName, true, true, true);
+ createMetastoreDB(client, dbName);
+ client.close();
+ }
+
+ /**
+ * Verify add partition via Pig+HCatStore
+ */
+ @Test
+ public void testPartionLoad() throws Exception {
+ String tabName = "tab1";
+ execHiveSQL("CREATE TABLE " + dbName + "." + tabName
+ + " (id int) PARTITIONED BY (part_col STRING)", USER1_1);
+
+ // user with ALL on DB should be able to add partion using Pig/HCatStore
+ PigServer pigServer = context.getPigServer(USER1_1, ExecType.LOCAL);
+ execPigLatin(USER1_1, pigServer, "A = load '" + dataFile.getPath()
+ + "' as (id:int);");
+ execPigLatin(USER1_1, pigServer, "store A into '" + dbName + "." + tabName
+ + "' using " + HCatStorer.class.getName() + " ('part_col=part1');");
+ HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
+ assertEquals(1, client.listPartitionNames(dbName, tabName, (short) 10)
+ .size());
+ client.close();
+
+ // user without ALL on DB should NOT be able to add partition with
+ // Pig/HCatStore
+ pigServer = context.getPigServer(USER2_1, ExecType.LOCAL);
+ execPigLatin(USER2_1, pigServer, "A = load '" + dataFile.getPath()
+ + "' as (id:int);");
+ try {
+ execPigLatin(USER2_1, pigServer, "store A into '" + dbName + "."
+ + tabName
+ + "' using " + HCatStorer.class.getName() + " ('part_col=part2');");
+ // TODO: The HCatStore seems to be swallowing the exception. Thus we
+ // manually verify that partition is not created by above call.
+ client = context.getMetaStoreClient(ADMIN1);
+ assertEquals(1, client.listPartitionNames(dbName, tabName, (short)10).size());
+ client.close();
+ // fail("HCatStore should fail for non-privilege user");
+
+ } catch (IOException e) {
+ Context.verifyMetastoreAuthException(e.getCause());
+ }
+
+ }
+
+}