You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/08/14 08:51:18 UTC
svn commit: r1372745 - in /hive/trunk: ./ ivy/ shims/
shims/src/common-secure/test/ shims/src/common-secure/test/org/
shims/src/common-secure/test/org/apache/
shims/src/common-secure/test/org/apache/hadoop/
shims/src/common-secure/test/org/apache/hadoo...
Author: hashutosh
Date: Tue Aug 14 06:51:18 2012
New Revision: 1372745
URL: http://svn.apache.org/viewvc?rev=1372745&view=rev
Log:
HIVE-3341 [jira] Making hive tests run against different MR versions
(Sushanth Sowmyan via Ashutosh Chauhan)
Summary:
HIVE-3341 Making hive tests run against different MR versions
After we build hive, we want to have the ability to run unit tests against specific hadoop versions. Currently, the classpath constructed has multiple hadoop jars, which makes compiling okay, but running non-deterministic.
An example is HIVE-3156, where running against 0.23 shows issues with a couple of tests (which should either be shimmed out, or separated into directories the way it's done in the shims/ directory) - It would also be nice to find these issues out at test-compile time itself, rather than having them fail at test runtime.
With this patch, we can set ant variable hadoop.mr.rev to 20, 20S or 23 to test against a particular version.
Test Plan: current tests continue to work - this is a build system change
Reviewers: JIRA, ashutoshc, cwsteinbach
Reviewed By: ashutoshc
Differential Revision: https://reviews.facebook.net/D4557
Added:
hive/trunk/shims/src/common-secure/test/
hive/trunk/shims/src/common-secure/test/org/
hive/trunk/shims/src/common-secure/test/org/apache/
hive/trunk/shims/src/common-secure/test/org/apache/hadoop/
hive/trunk/shims/src/common-secure/test/org/apache/hadoop/hive/
hive/trunk/shims/src/common-secure/test/org/apache/hadoop/hive/thrift/
hive/trunk/shims/src/common-secure/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java
- copied, changed from r1372744, hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java
hive/trunk/shims/src/common-secure/test/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
- copied unchanged from r1372744, hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
Removed:
hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java
hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
Modified:
hive/trunk/build-common.xml
hive/trunk/build.properties
hive/trunk/ivy/common-configurations.xml
hive/trunk/shims/build.xml
Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1372745&r1=1372744&r2=1372745&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Tue Aug 14 06:51:18 2012
@@ -91,11 +91,15 @@
<!-- Include build/dist/lib on the classpath before Ivy and exclude hive jars from Ivy to make sure we get the local changes when we test Hive -->
<fileset dir="${build.dir.hive}/dist/lib" includes="*.jar" erroronmissingdir="false" excludes="**/hive_contrib*.jar,**/hive-contrib*.jar,**/lib*.jar"/>
- <fileset dir="${hive.root}/build/ivy/lib/test" includes="*.jar" erroronmissingdir="false" excludes="**/hive_*.jar,**/hive-*.jar"/>
- <fileset dir="${hive.root}/build/ivy/lib/default" includes="*.jar" erroronmissingdir="false" excludes="**/hive_*.jar,**/hive-*.jar" />
<fileset dir="${hive.root}/testlibs" includes="*.jar"/>
- <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
+ <fileset dir="${build.ivy.lib.dir}/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" erroronmissingdir="false" />
<pathelement location="${build.classes}" />
+
+ <!-- we strip out hadoop jars present in places other than the hadoop shimmed dir-->
+ <fileset dir="${hive.root}/build/ivy/lib/test" includes="*.jar" erroronmissingdir="false"
+ excludes="**/hive_*.jar,**/hive-*.jar,**/hadoop-*.jar"/>
+ <fileset dir="${hive.root}/build/ivy/lib/default" includes="*.jar" erroronmissingdir="false"
+ excludes="**/hive_*.jar,**/hive-*.jar,**/hadoop-*.jar" />
</distinctelementsclasspath>
</target>
@@ -176,7 +180,9 @@
<pathelement location="${build.dir.hive}/classes"/>
<fileset dir="${build.dir.hive}" includes="*/*.jar"/>
<fileset dir="${hive.root}/lib" includes="*.jar"/>
+ <fileset dir="${build.ivy.lib.dir}/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" erroronmissingdir="false" />
<fileset dir="${build.ivy.lib.dir}/default" includes="*.jar"
+ excludes="**/hadoop-*.jar"
erroronmissingdir="false"/>
</path>
Modified: hive/trunk/build.properties
URL: http://svn.apache.org/viewvc/hive/trunk/build.properties?rev=1372745&r1=1372744&r2=1372745&view=diff
==============================================================================
--- hive/trunk/build.properties (original)
+++ hive/trunk/build.properties Tue Aug 14 06:51:18 2012
@@ -28,7 +28,7 @@ javac.args.warnings=
hadoop-0.20.version=0.20.2
hadoop-0.20S.version=1.0.0
-hadoop-0.23.version=0.23.1
+hadoop-0.23.version=2.0.0-alpha
hadoop.version=${hadoop-0.20.version}
hadoop.security.version=${hadoop-0.20S.version}
# Used to determine which set of Hadoop artifacts we depend on.
Modified: hive/trunk/ivy/common-configurations.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/common-configurations.xml?rev=1372745&r1=1372744&r2=1372745&view=diff
==============================================================================
--- hive/trunk/ivy/common-configurations.xml (original)
+++ hive/trunk/ivy/common-configurations.xml Tue Aug 14 06:51:18 2012
@@ -22,10 +22,12 @@
<conf name="runtime" description="runtime but not the artifact"/>
<conf name="test" extends="hadoop${hadoop.mr.rev}.test,compile" visibility="private" />
<conf name="hadoop20.compile" visibility="private"/>
+ <conf name="hadoop20S.compile" visibility="private"/>
<conf name="hadoop23.compile" visibility="private"/>
<conf name="hadoop20.test" visibility="private"/>
+ <conf name="hadoop20S.test" visibility="private"/>
<conf name="hadoop23.test" visibility="private"/>
<conf name="hadoop0.20.shim" visibility="private"/>
<conf name="hadoop0.20S.shim" visibility="private"/>
<conf name="hadoop0.23.shim" visibility="private"/>
-</configurations>
\ No newline at end of file
+</configurations>
Modified: hive/trunk/shims/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/build.xml?rev=1372745&r1=1372744&r2=1372745&view=diff
==============================================================================
--- hive/trunk/shims/build.xml (original)
+++ hive/trunk/shims/build.xml Tue Aug 14 06:51:18 2012
@@ -40,20 +40,7 @@ to call at top-level: ant deploy-contrib
<property name="shims.0.20S.hadoop.ivy.dir" value="${hive.root}/build/ivy/lib/hadoop0.20S.shim" />
<property name="shims.0.23.hadoop.ivy.dir" value="${hive.root}/build/ivy/lib/hadoop0.23.shim" />
- <target name="set-test-classpath">
- <path id="test.classpath">
- <pathelement location="${test.build.classes}" />
- <pathelement location="" />
- <pathelement location="${test.src.data.dir}/conf"/>
- <pathelement location="${hive.conf.dir}"/>
- <fileset dir="${hive.root}/testlibs" includes="*.jar" />
- <fileset dir="${shims.0.20S.hadoop.ivy.dir}" includes="*.jar" excludes="slf4j-*.jar" />
- <fileset dir="../lib" includes="*.jar" />
- <path refid="classpath" />
- </path>
- </target>
-
- <target name="build_shims"
+ <target name="build-shims"
description="Build shims against a particular hadoop version">
<echo message="Project: ${ant.project.name}"/>
<echo message="Compiling ${sources} against hadoop ${hadoop.version.ant-internal} (${hadoop.root})"/>
@@ -86,7 +73,7 @@ to call at top-level: ant deploy-contrib
<for param="shimName" list="${shims.include}">
<sequential>
<echo>Building shims @{shimName}</echo>
- <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+ <antcall target="build-shims" inheritRefs="false" inheritAll="false">
<param name="hadoop.version.ant-internal" value="${shims.@{shimName}.version}" />
<param name="sources" value="${shims.@{shimName}.sources}" />
<param name="hadoop.ivy.dir" value="${shims.@{shimName}.hadoop.ivy.dir}" />
@@ -95,49 +82,99 @@ to call at top-level: ant deploy-contrib
</sequential>
</for>
</target>
-
- <target name="compile_secure_test"
- depends="set-test-classpath"
- description="Test shims against a particular hadoop version">
+
+ <target name="compile-test-worker"
+ description="worker to compile tests, takes worker.test.src.dir parameter"
+ depends="set-test-classpath">
<echo message="Project: ${ant.project.name}"/>
- <getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version.ant-internal}" />
- <echo message="Compiling shim tests against hadoop ${hadoop.version.ant-internal} (${hadoop.root})"/>
- <javac
- encoding="${build.encoding}"
- srcdir="${test.src.dir}"
- includes="org/apache/hadoop/**/*.java"
- excludes="**/TestSerDe.java"
- destdir="${test.build.classes}"
- debug="${javac.debug}"
- optimize="${javac.optimize}"
- target="${javac.version}"
- source="${javac.version}"
- deprecation="${javac.deprecation}"
- includeantruntime="false">
- <compilerarg line="${javac.args} ${javac.args.warnings}" />
- <classpath refid="test.classpath"/>
- </javac>
- <javac
- encoding="${build.encoding}"
- srcdir="${test.build.src}"
- includes="org/apache/hadoop/**/*.java"
- destdir="${test.build.classes}"
- debug="${javac.debug}"
- optimize="${javac.optimize}"
- target="${javac.version}"
- source="${javac.version}"
- deprecation="${javac.deprecation}"
- includeantruntime="false">
- <compilerarg line="${javac.args} ${javac.args.warnings}" />
- <classpath refid="test.classpath"/>
- </javac>
+ <echo message="Compiling shim tests against hadoop ${hadoop.mr.rev}"/>
+ <if>
+ <available file="${worker.test.src.dir}" type="dir"/>
+ <then>
+ <echo message=" Test srcdir : ${worker.test.src.dir} "/>
+ <javac
+ encoding="${build.encoding}"
+ srcdir="${worker.test.src.dir}"
+ includes="org/apache/hadoop/**/*.java"
+ excludes="**/TestSerDe.java"
+ destdir="${test.build.classes}"
+ debug="${javac.debug}"
+ optimize="${javac.optimize}"
+ target="${javac.version}"
+ source="${javac.version}"
+ deprecation="${javac.deprecation}"
+ includeantruntime="false">
+ <compilerarg line="${javac.args} ${javac.args.warnings}" />
+ <classpath refid="test.classpath"/>
+ </javac>
+ </then>
+ <else>
+ <echo message=" Skipping nonexistent test srcdir : ${worker.test.src.dir} "/>
+ </else>
+ </if>
</target>
-
- <target name="compile-test" depends="compile,ivy-retrieve-test">
- <echo message="Project: ${ant.project.name}"/>
- <!-- TODO: move tests to version directory -->
- <antcall target="compile_secure_test" inheritRefs="false" inheritAll="false">
- <param name="hadoop.version.ant-internal" value="${hadoop.security.version}" />
+
+ <target name="determine-versions-for-tests">
+ <condition property="is.20">
+ <equals arg1="${hadoop.mr.rev}" arg2="20" />
+ </condition>
+ <condition property="is.20S">
+ <equals arg1="${hadoop.mr.rev}" arg2="20S" />
+ </condition>
+ <condition property="is.23">
+ <equals arg1="${hadoop.mr.rev}" arg2="23" />
+ </condition>
+ <condition property="is.secure">
+ <or>
+ <istrue value="${is.20S}" />
+ <istrue value="${is.23}" />
+ </or>
+ </condition>
+ </target>
+
+ <target name="compile-23-test" if="is.23">
+ <antcall target="compile-test-worker" inheritRefs="false" inheritAll="false">
+ <param name="worker.test.src.dir" value="${basedir}/src/0.23/test" />
+ </antcall>
+ </target>
+
+ <target name="compile-20S-test" if="is.20S">
+ <antcall target="compile-test-worker" inheritRefs="false" inheritAll="false">
+ <param name="worker.test.src.dir" value="${basedir}/src/0.20S/test" />
</antcall>
</target>
+
+ <target name="compile-20-test" if="is.20">
+ <antcall target="compile-test-worker" inheritRefs="false" inheritAll="false">
+ <param name="worker.test.src.dir" value="${basedir}/src/0.20/test" />
+ </antcall>
+ </target>
+
+ <target name="compile-common-secure-test" if="is.secure">
+ <antcall target="compile-test-worker" inheritRefs="false" inheritAll="false">
+ <param name="worker.test.src.dir" value="${basedir}/src/common-secure/test" />
+ </antcall>
+ </target>
+
+ <target name="compile-common-test">
+ <antcall target="compile-test-worker" inheritRefs="false" inheritAll="false">
+ <param name="worker.test.src.dir" value="${basedir}/src/test" />
+ </antcall>
+ <antcall target="compile-test-worker" inheritRefs="false" inheritAll="false">
+ <param name="worker.test.src.dir" value="${basedir}/src/common/test" />
+ </antcall>
+ <antcall target="compile-test-worker" inheritRefs="false" inheritAll="false">
+ <param name="worker.test.src.dir" value="${test.build.src}" />
+ </antcall>
+ </target>
+
+ <target name="compile-test" depends="compile,ivy-retrieve-test,determine-versions-for-tests">
+ <echo message="Project: ${ant.project.name}"/>
+ <antcall target="compile-common-test" />
+ <antcall target="compile-common-secure-test" />
+ <antcall target="compile-20-test" />
+ <antcall target="compile-20S-test" />
+ <antcall target="compile-23-test" />
+ </target>
+
</project>
Copied: hive/trunk/shims/src/common-secure/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java (from r1372744, hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java)
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java?p2=hive/trunk/shims/src/common-secure/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java&p1=hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java&r1=1372744&r2=1372745&rev=1372745&view=diff
==============================================================================
--- hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java (original)
+++ hive/trunk/shims/src/common-secure/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java Tue Aug 14 06:51:18 2012
@@ -28,6 +28,7 @@ import java.security.PrivilegedException
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
+import java.util.List;
import junit.framework.TestCase;
@@ -96,7 +97,7 @@ public class TestHadoop20SAuthBridge ext
private void configureSuperUserIPAddresses(Configuration conf,
String superUserShortName) throws IOException {
- ArrayList<String> ipList = new ArrayList<String>();
+ List<String> ipList = new ArrayList<String>();
Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
.getNetworkInterfaces();
while (netInterfaceList.hasMoreElements()) {