You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2014/07/03 03:55:54 UTC

svn commit: r1607515 - in /hive/trunk/itests: ./ hive-unit-hadoop2/ hive-unit-hadoop2/src/ hive-unit-hadoop2/src/test/ hive-unit-hadoop2/src/test/java/ hive-unit-hadoop2/src/test/java/org/ hive-unit-hadoop2/src/test/java/org/apache/ hive-unit-hadoop2/s...

Author: szehon
Date: Thu Jul  3 01:55:53 2014
New Revision: 1607515

URL: http://svn.apache.org/r1607515
Log:
HIVE-7247: Fix itests using hadoop-1 profile (Szehon reviewed by Brock)

Added:
    hive/trunk/itests/hive-unit-hadoop2/
    hive/trunk/itests/hive-unit-hadoop2/pom.xml
    hive/trunk/itests/hive-unit-hadoop2/src/
    hive/trunk/itests/hive-unit-hadoop2/src/test/
    hive/trunk/itests/hive-unit-hadoop2/src/test/java/
    hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/
    hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/
    hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/
    hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/
    hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/
    hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/
    hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
Removed:
    hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
Modified:
    hive/trunk/itests/hive-unit/pom.xml
    hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
    hive/trunk/itests/pom.xml

Added: hive/trunk/itests/hive-unit-hadoop2/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit-hadoop2/pom.xml?rev=1607515&view=auto
==============================================================================
--- hive/trunk/itests/hive-unit-hadoop2/pom.xml (added)
+++ hive/trunk/itests/hive-unit-hadoop2/pom.xml Thu Jul  3 01:55:53 2014
@@ -0,0 +1,191 @@
+	<?xml version="1.0" encoding="UTF-8"?>
+	<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hive</groupId>
+    <artifactId>hive-it</artifactId>
+    <version>0.14.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>hive-it-unit-hadoop2</artifactId>
+  <packaging>jar</packaging>
+  <name>Hive Integration - Unit Tests - Hadoop 2</name>
+
+  <properties>
+    <hive.path.to.root>../..</hive.path.to.root>
+  </properties>
+
+  <dependencies>
+    <!-- intra-project -->
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-common</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-jdbc</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-service</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
+    <!-- dependencies are always listed in sorted order by groupId, artifectId -->
+    <!-- test intra-project -->
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-beeline</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-cli</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-it-util</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-metastore</artifactId>
+      <version>${project.version}</version>
+      <classifier>tests</classifier>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-serde</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-service</artifactId>
+      <version>${project.version}</version>
+      <classifier>tests</classifier>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <version>${project.version}</version>
+      <classifier>tests</classifier>
+    </dependency>
+    <!-- test inter-project -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>${mockito-all.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-it-unit</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <!-- hadoop-2 dependencies -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop-23.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <version>${hadoop-23.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+      <version>${hadoop-23.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <version>${hadoop-23.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-server</artifactId>
+      <version>${hbase.hadoop2.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minicluster</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-servlet</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>setup-metastore-scripts</id>
+            <phase>process-test-resources</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <mkdir dir="${test.tmp.dir}/scripts/metastore" />
+                <copy todir="${test.tmp.dir}/scripts/metastore">
+                  <fileset dir="${basedir}/${hive.path.to.root}/metastore/scripts/"/>
+                </copy>
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

Added: hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java?rev=1607515&view=auto
==============================================================================
--- hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java (added)
+++ hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestExtendedAcls.java Thu Jul  3 01:55:53 2014
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.security;
+
+import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;
+import static org.apache.hadoop.fs.permission.AclEntryType.GROUP;
+import static org.apache.hadoop.fs.permission.AclEntryType.OTHER;
+import static org.apache.hadoop.fs.permission.AclEntryType.USER;
+
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryScope;
+import org.apache.hadoop.fs.permission.AclEntryType;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import com.google.common.collect.Lists;
+
+public class TestExtendedAcls extends FolderPermissionBase {
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    conf = new HiveConf(TestExtendedAcls.class);
+    //setup the mini DFS with acl's enabled.
+    conf.set("dfs.namenode.acls.enabled", "true");
+    baseSetup();
+  }
+
+  List<AclEntry> aclSpec1 = Lists.newArrayList(
+      aclEntry(ACCESS, USER, FsAction.ALL),
+      aclEntry(ACCESS, GROUP, FsAction.ALL),
+      aclEntry(ACCESS, OTHER, FsAction.ALL),
+      aclEntry(ACCESS, USER, "bar", FsAction.READ_WRITE),
+      aclEntry(ACCESS, USER, "foo", FsAction.READ_EXECUTE),
+      aclEntry(ACCESS, GROUP, "bar", FsAction.READ_WRITE),
+      aclEntry(ACCESS, GROUP, "foo", FsAction.READ_EXECUTE));
+
+  List<AclEntry> aclSpec2 = Lists.newArrayList(
+      aclEntry(ACCESS, USER, FsAction.ALL),
+      aclEntry(ACCESS, GROUP, FsAction.ALL),
+      aclEntry(ACCESS, OTHER, FsAction.READ_EXECUTE),
+      aclEntry(ACCESS, USER, "bar2", FsAction.READ_WRITE),
+      aclEntry(ACCESS, USER, "foo2", FsAction.READ_EXECUTE),
+      aclEntry(ACCESS, GROUP, "bar2", FsAction.READ),
+      aclEntry(ACCESS, GROUP, "foo2", FsAction.READ_EXECUTE));
+
+  @Override
+  public void setPermission(String locn, int permIndex) throws Exception {
+    switch (permIndex) {
+      case 0:
+        setAcl(locn, aclSpec1);
+        break;
+      case 1:
+        setAcl(locn, aclSpec2);
+        break;
+      default:
+        throw new RuntimeException("Only 2 permissions by this test");
+    }
+  }
+
+  @Override
+  public void verifyPermission(String locn, int permIndex) throws Exception {
+    switch (permIndex) {
+      case 0:
+        FsPermission perm = fs.getFileStatus(new Path(locn)).getPermission();
+        Assert.assertEquals(perm.toString(), "rwxrwxrwx");
+
+        List<AclEntry> actual = getAcl(locn);
+        verifyAcls(aclSpec1, actual);
+        break;
+      case 1:
+        perm = fs.getFileStatus(new Path(locn)).getPermission();
+        Assert.assertEquals(perm.toString(), "rwxrwxr-x");
+
+        List<AclEntry> acls = getAcl(locn);
+        verifyAcls(aclSpec2, acls);
+        break;
+      default:
+        throw new RuntimeException("Only 2 permissions by this test");
+    }
+  }
+
+  /**
+   * Create a new AclEntry with scope, type and permission (no name).
+   *
+   * @param scope
+   *          AclEntryScope scope of the ACL entry
+   * @param type
+   *          AclEntryType ACL entry type
+   * @param permission
+   *          FsAction set of permissions in the ACL entry
+   * @return AclEntry new AclEntry
+   */
+  private AclEntry aclEntry(AclEntryScope scope, AclEntryType type,
+      FsAction permission) {
+    return new AclEntry.Builder().setScope(scope).setType(type)
+        .setPermission(permission).build();
+  }
+
+  /**
+   * Create a new AclEntry with scope, type, name and permission.
+   *
+   * @param scope
+   *          AclEntryScope scope of the ACL entry
+   * @param type
+   *          AclEntryType ACL entry type
+   * @param name
+   *          String optional ACL entry name
+   * @param permission
+   *          FsAction set of permissions in the ACL entry
+   * @return AclEntry new AclEntry
+   */
+  private AclEntry aclEntry(AclEntryScope scope, AclEntryType type,
+      String name, FsAction permission) {
+    return new AclEntry.Builder().setScope(scope).setType(type).setName(name)
+        .setPermission(permission).build();
+  }
+
+  private void verifyAcls(List<AclEntry> expectedList, List<AclEntry> actualList) {
+    for (AclEntry expected : expectedList) {
+      if (expected.getName() != null) {
+        //the non-named acl's are coming as regular permission, and not as aclEntries.
+        boolean found = false;
+        for (AclEntry actual : actualList) {
+          if (actual.equals(expected)) {
+            found = true;
+          }
+        }
+        if (!found) {
+          Assert.fail("Following Acl does not have a match: " + expected);
+        }
+      }
+    }
+  }
+
+  private void setAcl(String locn, List<AclEntry> aclSpec) throws Exception {
+    fs.setAcl(new Path(locn), aclSpec);
+  }
+
+  private List<AclEntry> getAcl(String locn) throws Exception {
+    return fs.getAclStatus(new Path(locn)).getEntries();
+  }
+}

Modified: hive/trunk/itests/hive-unit/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/pom.xml?rev=1607515&r1=1607514&r2=1607515&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/pom.xml (original)
+++ hive/trunk/itests/hive-unit/pom.xml Thu Jul  3 01:55:53 2014
@@ -264,6 +264,18 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+         <artifactId>maven-jar-plugin</artifactId>
+         <version>${maven.jar.plugin.version}</version>
+         <executions>
+           <execution>
+             <goals>
+               <goal>test-jar</goal>
+             </goals>
+           </execution>
+         </executions>
+       </plugin>
     </plugins>
   </build>
 

Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java?rev=1607515&r1=1607514&r2=1607515&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java Thu Jul  3 01:55:53 2014
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;

Modified: hive/trunk/itests/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/pom.xml?rev=1607515&r1=1607514&r2=1607515&view=diff
==============================================================================
--- hive/trunk/itests/pom.xml (original)
+++ hive/trunk/itests/pom.xml Thu Jul  3 01:55:53 2014
@@ -44,6 +44,7 @@
     <profile>
       <id>hadoop-2</id>
       <modules>
+        <module>hive-unit-hadoop2</module>
         <module>hive-minikdc</module>
       </modules>
     </profile>