You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2014/07/03 03:55:54 UTC
svn commit: r1607515 - in /hive/trunk/itests: ./ hive-unit-hadoop2/
hive-unit-hadoop2/src/ hive-unit-hadoop2/src/test/
hive-unit-hadoop2/src/test/java/ hive-unit-hadoop2/src/test/java/org/
hive-unit-hadoop2/src/test/java/org/apache/ hive-unit-hadoop2/s...
Author: szehon
Date: Thu Jul 3 01:55:53 2014
New Revision: 1607515
URL: http://svn.apache.org/r1607515
Log:
HIVE-7247: Fix itests using hadoop-1 profile (Szehon reviewed by Brock)
Added:
hive/trunk/itests/hive-unit-hadoop2/
hive/trunk/itests/hive-unit-hadoop2/pom.xml
hive/trunk/itests/hive-unit-hadoop2/src/
hive/trunk/itests/hive-unit-hadoop2/src/test/
hive/trunk/itests/hive-unit-hadoop2/src/test/java/
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
Removed:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
Modified:
hive/trunk/itests/hive-unit/pom.xml
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
hive/trunk/itests/pom.xml
Added: hive/trunk/itests/hive-unit-hadoop2/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit-hadoop2/pom.xml?rev=1607515&view=auto
==============================================================================
--- hive/trunk/itests/hive-unit-hadoop2/pom.xml (added)
+++ hive/trunk/itests/hive-unit-hadoop2/pom.xml Thu Jul 3 01:55:53 2014
@@ -0,0 +1,191 @@
+ <?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it</artifactId>
+ <version>0.14.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>hive-it-unit-hadoop2</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive Integration - Unit Tests - Hadoop 2</name>
+
+ <properties>
+ <hive.path.to.root>../..</hive.path.to.root>
+ </properties>
+
+ <dependencies>
+ <!-- intra-project -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- dependencies are always listed in sorted order by groupId, artifectId -->
+ <!-- test intra-project -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-beeline</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-cli</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it-util</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-serde</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
+ <!-- test inter-project -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito-all.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it-unit</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <!-- hadoop-2 dependencies -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop-23.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop-23.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>${hadoop-23.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop-23.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.hadoop2.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>setup-metastore-scripts</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <mkdir dir="${test.tmp.dir}/scripts/metastore" />
+ <copy todir="${test.tmp.dir}/scripts/metastore">
+ <fileset dir="${basedir}/${hive.path.to.root}/metastore/scripts/"/>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
Added: hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java?rev=1607515&view=auto
==============================================================================
--- hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java (added)
+++ hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java Thu Jul 3 01:55:53 2014
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.security;
+
+import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;
+import static org.apache.hadoop.fs.permission.AclEntryType.GROUP;
+import static org.apache.hadoop.fs.permission.AclEntryType.OTHER;
+import static org.apache.hadoop.fs.permission.AclEntryType.USER;
+
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryScope;
+import org.apache.hadoop.fs.permission.AclEntryType;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import com.google.common.collect.Lists;
+
+public class TestExtendedAcls extends FolderPermissionBase {
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ conf = new HiveConf(TestExtendedAcls.class);
+ //setup the mini DFS with acl's enabled.
+ conf.set("dfs.namenode.acls.enabled", "true");
+ baseSetup();
+ }
+
+ List<AclEntry> aclSpec1 = Lists.newArrayList(
+ aclEntry(ACCESS, USER, FsAction.ALL),
+ aclEntry(ACCESS, GROUP, FsAction.ALL),
+ aclEntry(ACCESS, OTHER, FsAction.ALL),
+ aclEntry(ACCESS, USER, "bar", FsAction.READ_WRITE),
+ aclEntry(ACCESS, USER, "foo", FsAction.READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, "bar", FsAction.READ_WRITE),
+ aclEntry(ACCESS, GROUP, "foo", FsAction.READ_EXECUTE));
+
+ List<AclEntry> aclSpec2 = Lists.newArrayList(
+ aclEntry(ACCESS, USER, FsAction.ALL),
+ aclEntry(ACCESS, GROUP, FsAction.ALL),
+ aclEntry(ACCESS, OTHER, FsAction.READ_EXECUTE),
+ aclEntry(ACCESS, USER, "bar2", FsAction.READ_WRITE),
+ aclEntry(ACCESS, USER, "foo2", FsAction.READ_EXECUTE),
+ aclEntry(ACCESS, GROUP, "bar2", FsAction.READ),
+ aclEntry(ACCESS, GROUP, "foo2", FsAction.READ_EXECUTE));
+
+ @Override
+ public void setPermission(String locn, int permIndex) throws Exception {
+ switch (permIndex) {
+ case 0:
+ setAcl(locn, aclSpec1);
+ break;
+ case 1:
+ setAcl(locn, aclSpec2);
+ break;
+ default:
+ throw new RuntimeException("Only 2 permissions by this test");
+ }
+ }
+
+ @Override
+ public void verifyPermission(String locn, int permIndex) throws Exception {
+ switch (permIndex) {
+ case 0:
+ FsPermission perm = fs.getFileStatus(new Path(locn)).getPermission();
+ Assert.assertEquals(perm.toString(), "rwxrwxrwx");
+
+ List<AclEntry> actual = getAcl(locn);
+ verifyAcls(aclSpec1, actual);
+ break;
+ case 1:
+ perm = fs.getFileStatus(new Path(locn)).getPermission();
+ Assert.assertEquals(perm.toString(), "rwxrwxr-x");
+
+ List<AclEntry> acls = getAcl(locn);
+ verifyAcls(aclSpec2, acls);
+ break;
+ default:
+ throw new RuntimeException("Only 2 permissions by this test");
+ }
+ }
+
+ /**
+ * Create a new AclEntry with scope, type and permission (no name).
+ *
+ * @param scope
+ * AclEntryScope scope of the ACL entry
+ * @param type
+ * AclEntryType ACL entry type
+ * @param permission
+ * FsAction set of permissions in the ACL entry
+ * @return AclEntry new AclEntry
+ */
+ private AclEntry aclEntry(AclEntryScope scope, AclEntryType type,
+ FsAction permission) {
+ return new AclEntry.Builder().setScope(scope).setType(type)
+ .setPermission(permission).build();
+ }
+
+ /**
+ * Create a new AclEntry with scope, type, name and permission.
+ *
+ * @param scope
+ * AclEntryScope scope of the ACL entry
+ * @param type
+ * AclEntryType ACL entry type
+ * @param name
+ * String optional ACL entry name
+ * @param permission
+ * FsAction set of permissions in the ACL entry
+ * @return AclEntry new AclEntry
+ */
+ private AclEntry aclEntry(AclEntryScope scope, AclEntryType type,
+ String name, FsAction permission) {
+ return new AclEntry.Builder().setScope(scope).setType(type).setName(name)
+ .setPermission(permission).build();
+ }
+
+ private void verifyAcls(List<AclEntry> expectedList, List<AclEntry> actualList) {
+ for (AclEntry expected : expectedList) {
+ if (expected.getName() != null) {
+ //the non-named acl's are coming as regular permission, and not as aclEntries.
+ boolean found = false;
+ for (AclEntry actual : actualList) {
+ if (actual.equals(expected)) {
+ found = true;
+ }
+ }
+ if (!found) {
+ Assert.fail("Following Acl does not have a match: " + expected);
+ }
+ }
+ }
+ }
+
+ private void setAcl(String locn, List<AclEntry> aclSpec) throws Exception {
+ fs.setAcl(new Path(locn), aclSpec);
+ }
+
+ private List<AclEntry> getAcl(String locn) throws Exception {
+ return fs.getAclStatus(new Path(locn)).getEntries();
+ }
+}
Modified: hive/trunk/itests/hive-unit/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/pom.xml?rev=1607515&r1=1607514&r2=1607515&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/pom.xml (original)
+++ hive/trunk/itests/hive-unit/pom.xml Thu Jul 3 01:55:53 2014
@@ -264,6 +264,18 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${maven.jar.plugin.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java?rev=1607515&r1=1607514&r2=1607515&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java Thu Jul 3 01:55:53 2014
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
Modified: hive/trunk/itests/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/pom.xml?rev=1607515&r1=1607514&r2=1607515&view=diff
==============================================================================
--- hive/trunk/itests/pom.xml (original)
+++ hive/trunk/itests/pom.xml Thu Jul 3 01:55:53 2014
@@ -44,6 +44,7 @@
<profile>
<id>hadoop-2</id>
<modules>
+ <module>hive-unit-hadoop2</module>
<module>hive-minikdc</module>
</modules>
</profile>