You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by am...@apache.org on 2010/11/26 10:58:34 UTC
svn commit: r1039288 - in /hadoop/mapreduce/branches/branch-0.22:
CHANGES.txt src/contrib/build-contrib.xml src/contrib/build.xml
src/contrib/streaming/build.xml src/test/aop/build/aop.xml
Author: amareshwari
Date: Fri Nov 26 09:58:34 2010
New Revision: 1039288
URL: http://svn.apache.org/viewvc?rev=1039288&view=rev
Log:
Merge -r 1039284:1039285 from trunk
Modified:
hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
hadoop/mapreduce/branches/branch-0.22/src/contrib/build-contrib.xml
hadoop/mapreduce/branches/branch-0.22/src/contrib/build.xml
hadoop/mapreduce/branches/branch-0.22/src/contrib/streaming/build.xml
hadoop/mapreduce/branches/branch-0.22/src/test/aop/build/aop.xml
Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1039288&r1=1039287&r2=1039288&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Fri Nov 26 09:58:34 2010
@@ -428,6 +428,9 @@ Release 0.21.1 - Unreleased
MAPREDUCE-1905. Fixes Context.setStatus() and progress() apis.
(amareshwari)
+ MAPREDUCE-1809. Ant build changes for Streaming system tests in contrib
+ projects. (Vinay Kumar Thota via amareshwari)
+
Release 0.21.0 - 2010-08-13
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/build-contrib.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/build-contrib.xml?rev=1039288&r1=1039287&r2=1039288&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/build-contrib.xml (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/build-contrib.xml Fri Nov 26 09:58:34 2010
@@ -36,9 +36,16 @@
<property name="src.test" location="${root}/src/test"/>
<property name="src.test.data" location="${root}/src/test/data"/>
<property name="src.examples" location="${root}/src/examples"/>
+ <property name="build-fi.dir" location="${hadoop.root}/build-fi"/>
+ <property name="system-test-build-dir" location="${build-fi.dir}/system"/>
+ <!-- Property added for contrib system tests -->
+ <property name="src.test.system" location="${root}/src/test/system"/>
<available file="${src.examples}" type="dir" property="examples.available"/>
<available file="${src.test}" type="dir" property="test.available"/>
+ <!-- Property added for contrib system tests -->
+ <available file="${src.test.system}" type="dir"
+ property="test.system.available"/>
<property name="conf.dir" location="${hadoop.root}/conf"/>
<property name="test.junit.output.format" value="plain"/>
@@ -63,6 +70,10 @@
<property name="dest.jar" value="hadoop-${version}-${name}.jar"/>
<fileset id="lib.jars" dir="${root}" includes="lib/*.jar"/>
+ <!-- Property added for contrib system tests -->
+ <property name="build.test.system" location="${build.dir}/system"/>
+ <property name="build.system.classes"
+ location="${build.test.system}/classes"/>
<!-- IVY properties set here -->
@@ -87,10 +98,15 @@
<pathelement location="${build.classes}"/>
<fileset refid="lib.jars"/>
<pathelement location="${hadoop.root}/build/classes"/>
+ <pathelement location="${system-test-build-dir}/classes"/>
+ <pathelement location="${system-test-build-dir}/tools"/>
<pathelement location="${hadoop.root}/build/tools"/>
<fileset dir="${hadoop.root}/lib">
<include name="**/*.jar" />
</fileset>
+ <fileset dir="${build.ivy.lib.dir}">
+ <include name="**/*.jar" />
+ </fileset>
<path refid="${ant.project.name}.common-classpath"/>
<pathelement path="${clover.jar}"/>
</path>
@@ -112,6 +128,36 @@
<path refid="contrib-classpath"/>
</path>
+ <!-- The system test classpath -->
+ <path id="test.system.classpath">
+ <pathelement location="${hadoop.root}/src/contrib/${name}/src/test/system" />
+ <pathelement location="${build.test.system}" />
+ <pathelement location="${build.test.system}/classes"/>
+ <pathelement location="${build.examples}"/>
+ <pathelement location="${system-test-build-dir}/classes" />
+ <pathelement location="${system-test-build-dir}/test/mapred/classes" />
+ <pathelement location="${system-test-build-dir}" />
+ <pathelement location="${system-test-build-dir}/tools" />
+ <pathelement location="${hadoop.home}"/>
+ <pathelement location="${hadoop.conf.dir}"/>
+ <pathelement location="${hadoop.conf.dir.deployed}"/>
+ <pathelement location="${hadoop.root}/build"/>
+ <pathelement location="${hadoop.root}/build/examples"/>
+ <pathelement location="${hadoop.root}/build/test/classes" />
+ <path refid="contrib-classpath"/>
+ <fileset dir="${system-test-build-dir}">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ <fileset dir="${system-test-build-dir}/test/mapred/testjar">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ <fileset dir="${hadoop.root}/build/contrib/${name}">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+ </path>
<!-- to be overridden by sub-projects -->
<target name="check-contrib"/>
@@ -128,6 +174,9 @@
<mkdir dir="${build.test}/extraconf"/>
<mkdir dir="${build.examples}"/>
<mkdir dir="${hadoop.log.dir}"/>
+ <!-- The below two tags added for contrib system tests -->
+ <mkdir dir="${build.test.system}"/>
+ <mkdir dir="${build.system.classes}"/>
<antcall target="init-contrib"/>
</target>
@@ -141,6 +190,7 @@
encoding="${build.encoding}"
srcdir="${src.dir}"
includes="**/*.java"
+ excludes="system/**/*.java"
destdir="${build.classes}"
debug="${javac.debug}"
deprecation="${javac.deprecation}">
@@ -174,12 +224,28 @@
encoding="${build.encoding}"
srcdir="${src.test}"
includes="**/*.java"
+ excludes="system/**/*.java"
destdir="${build.test}"
debug="${javac.debug}">
<classpath refid="test.classpath"/>
</javac>
</target>
+ <!-- ================================================================== -->
+ <!-- Compile system test code -->
+ <!-- ================================================================== -->
+ <target name="compile-test-system" depends="compile-examples, ivy-retrieve-test"
+ if="test.system.available">
+ <echo message="contrib: ${name}"/>
+ <javac
+ encoding="${build.encoding}"
+ srcdir="${src.test.system}"
+ includes="**/*.java"
+ destdir="${build.system.classes}"
+ debug="${javac.debug}">
+ <classpath refid="test.system.classpath"/>
+ </javac>
+ </target>
<!-- ====================================================== -->
<!-- Make a Hadoop contrib's jar -->
@@ -264,15 +330,133 @@
<formatter type="${test.junit.output.format}" />
<batchtest todir="${build.test}" unless="testcase">
<fileset dir="${src.test}"
- includes="**/Test*.java" excludes="**/${test.exclude}.java" />
+ includes="**/Test*.java" excludes="**/${test.exclude}.java, system/**/*.java" />
</batchtest>
<batchtest todir="${build.test}" if="testcase">
- <fileset dir="${src.test}" includes="**/${testcase}.java"/>
+ <fileset dir="${src.test}" includes="**/${testcase}.java" excludes="system/**/*.java" />
</batchtest>
</junit>
<antcall target="checkfailure"/>
</target>
+ <!-- ================================================================== -->
+ <!-- Run system tests -->
+ <!-- ================================================================== -->
+ <target name="test-system" depends="compile-test-system, jar"
+ if="test.system.available">
+ <delete dir="${build.test.system}/extraconf"/>
+ <mkdir dir="${build.test.system}/extraconf"/>
+ <property name="test.src.dir" location="${hadoop.root}/src/test"/>
+ <property name="test.junit.printsummary" value="yes" />
+ <property name="test.junit.haltonfailure" value="no" />
+ <property name="test.junit.maxmemory" value="512m" />
+ <property name="test.junit.fork.mode" value="perTest" />
+ <property name="test.all.tests.file" value="${test.src.dir}/all-tests" />
+ <property name="test.build.dir" value="${hadoop.root}/build/test"/>
+ <property name="basedir" value="${hadoop.root}"/>
+ <property name="test.timeout" value="900000"/>
+ <property name="test.junit.output.format" value="plain"/>
+ <property name="test.tools.input.dir" value="${basedir}/src/test/tools/data"/>
+ <property name="c++.src" value="${basedir}/src/c++"/>
+ <property name="test.include" value="Test*"/>
+ <property name="c++.libhdfs.src" value="${c++.src}/libhdfs"/>
+ <property name="test.build.data" value="${build.test.system}/data"/>
+ <property name="test.cache.data" value="${build.test.system}/cache"/>
+ <property name="test.debug.data" value="${build.test.system}/debug"/>
+ <property name="test.log.dir" value="${build.test.system}/logs"/>
+ <exec executable="sed" inputstring="${os.name}"
+ outputproperty="nonspace.os">
+ <arg value="s/ /_/g"/>
+ </exec>
+ <property name="build.platform"
+ value="${nonspace.os}-${os.arch}-${sun.arch.data.model}"/>
+ <property name="build.native"
+ value="${hadoop.root}/build/native/${build.platform}"/>
+ <property name="lib.dir" value="${hadoop.root}/lib"/>
+ <property name="install.c++.examples"
+ value="${hadoop.root}/build/c++-examples/${build.platform}"/>
+ <condition property="tests.testcase">
+ <and>
+ <isset property="testcase" />
+ </and>
+ </condition>
+ <property name="test.junit.jvmargs" value="-ea" />
+ <macro-system-test-runner test.file="${test.all.tests.file}"
+ classpath="test.system.classpath"
+ test.dir="${build.test.system}"
+ fileset.dir="${hadoop.root}/src/contrib/${name}/src/test/system"
+ hadoop.conf.dir.deployed="${hadoop.conf.dir.deployed}">
+ </macro-system-test-runner>
+</target>
+<macrodef name="macro-system-test-runner">
+ <attribute name="test.file" />
+ <attribute name="classpath" />
+ <attribute name="test.dir" />
+ <attribute name="fileset.dir" />
+ <attribute name="hadoop.conf.dir.deployed" default="" />
+ <sequential>
+ <delete dir="@{test.dir}/data"/>
+ <mkdir dir="@{test.dir}/data"/>
+ <delete dir="@{test.dir}/logs"/>
+ <mkdir dir="@{test.dir}/logs"/>
+ <copy file="${test.src.dir}/hadoop-policy.xml"
+ todir="@{test.dir}/extraconf" />
+ <copy file="${test.src.dir}/fi-site.xml"
+ todir="@{test.dir}/extraconf" />
+ <junit showoutput="${test.output}"
+ printsummary="${test.junit.printsummary}"
+ haltonfailure="${test.junit.haltonfailure}"
+ fork="yes"
+ forkmode="${test.junit.fork.mode}"
+ maxmemory="${test.junit.maxmemory}"
+ dir="${basedir}" timeout="${test.timeout}"
+ errorProperty="tests.failed" failureProperty="tests.failed">
+ <jvmarg value="${test.junit.jvmargs}" />
+ <sysproperty key="test.build.data" value="@{test.dir}/data"/>
+ <sysproperty key="test.tools.input.dir" value = "${test.tools.input.dir}"/>
+ <sysproperty key="test.cache.data" value="${test.cache.data}"/>
+ <sysproperty key="test.debug.data" value="${test.debug.data}"/>
+ <sysproperty key="hadoop.log.dir" value="@{test.dir}/logs"/>
+ <sysproperty key="test.src.dir" value="@{fileset.dir}"/>
+ <sysproperty key="taskcontroller-path" value="${taskcontroller-path}"/>
+ <sysproperty key="taskcontroller-ugi" value="${taskcontroller-ugi}"/>
+ <sysproperty key="test.build.extraconf" value="@{test.dir}/extraconf" />
+ <sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
+ <sysproperty key="java.library.path"
+ value="${build.native}/lib:${lib.dir}/native/${build.platform}"/>
+ <sysproperty key="install.c++.examples" value="${install.c++.examples}"/>
+ <syspropertyset dynamic="no">
+ <propertyref name="hadoop.tmp.dir"/>
+ </syspropertyset>
+ <!-- set compile.c++ in the child jvm only if it is set -->
+ <syspropertyset dynamic="no">
+ <propertyref name="compile.c++"/>
+ </syspropertyset>
+
+ <!-- Pass probability specifications to the spawn JVM -->
+ <syspropertyset id="FaultProbabilityProperties">
+ <propertyref regex="fi.*"/>
+ </syspropertyset>
+ <sysproperty key="test.system.hdrc.deployed.hadoopconfdir"
+ value="@{hadoop.conf.dir.deployed}" />
+ <classpath refid="@{classpath}"/>
+ <formatter type="${test.junit.output.format}" />
+ <batchtest todir="@{test.dir}" unless="testcase">
+ <fileset dir="@{fileset.dir}"
+ excludes="**/${test.exclude}.java aop/** system/**">
+ <patternset>
+ <includesfile name="@{test.file}"/>
+ </patternset>
+ </fileset>
+ </batchtest>
+ <batchtest todir="@{test.dir}" if="testcase">
+ <fileset dir="@{fileset.dir}" includes="**/${testcase}.java"/>
+ </batchtest>
+ </junit>
+ <antcall target="checkfailure"/>
+ </sequential>
+ </macrodef>
+
<target name="docs" depends="forrest.check" description="Generate forrest-based documentation. To use, specify -Dforrest.home=<base of Apache Forrest installation> on the command line." if="forrest.home">
<!-- Nothing by default -->
</target>
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/build.xml?rev=1039288&r1=1039287&r2=1039288&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/build.xml (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/build.xml Fri Nov 26 09:58:34 2010
@@ -73,6 +73,29 @@
<fail if="testsfailed">Tests failed!</fail>
</target>
+ <!-- ====================================================== -->
+ <!-- Test all the contrib system tests -->
+ <!-- ====================================================== -->
+ <target name="test-system-contrib">
+ <property name="hadoop.root" location="${root}/../../../"/>
+ <property name="build.contrib.dir" location="${hadoop.root}/build/contrib"/>
+ <delete file="${build.contrib.dir}/testsfailed"/>
+ <subant target="test-system">
+ <property name="continueOnFailure" value="true"/>
+ <property name="hadoop.home" value="${hadoop.home}"/>
+ <property name="hadoop.conf.dir" value="${hadoop.conf.dir}"/>
+ <property name="hadoop.conf.dir.deployed"
+ value="${hadoop.conf.dir.deployed}"/>
+ <fileset dir="." includes="hdfsproxy/build.xml"/>
+ <fileset dir="." includes="streaming/build.xml"/>
+ <fileset dir="." includes="fairscheduler/build.xml"/>
+ <fileset dir="." includes="capacity-scheduler/build.xml"/>
+ <fileset dir="." includes="gridmix/build.xml"/>
+ </subant>
+ <available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
+ <fail if="testsfailed">Tests failed!</fail>
+ </target>
+
<target name="docs">
<subant target="docs">
<fileset dir="." includes="capacity-scheduler/build.xml"/>
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/streaming/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/streaming/build.xml?rev=1039288&r1=1039287&r2=1039288&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/streaming/build.xml (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/streaming/build.xml Fri Nov 26 09:58:34 2010
@@ -43,4 +43,10 @@ to call at top-level: ant deploy-contrib
</antcall>
</target>
+ <!--Run all system tests.-->
+ <target name="test-system">
+ <antcall target="hadoopbuildcontrib.test-system">
+ </antcall>
+ </target>
+
</project>
Modified: hadoop/mapreduce/branches/branch-0.22/src/test/aop/build/aop.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/aop/build/aop.xml?rev=1039288&r1=1039287&r2=1039288&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/test/aop/build/aop.xml (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/test/aop/build/aop.xml Fri Nov 26 09:58:34 2010
@@ -139,6 +139,14 @@
<target name="test-system" depends="ivy-retrieve-common, ivy-retrieve-system"
description="Run system tests">
<subant buildpath="build.xml" target="jar-test-system"/>
+ <subant target="test-system-contrib">
+ <property name="hadoop.home" value="${hadoop.home}"/>
+ <property name="hadoop.conf.dir" value="${hadoop.conf.dir}"/>
+ <property name="version" value="${version}"/>
+ <property name="hadoop.conf.dir.deployed"
+ value="${hadoop.conf.dir.deployed}"/>
+ <fileset file="${contrib.dir}/build.xml"/>
+ </subant>
<macro-test-runner test.file="${test.mapred.all.tests.file}"
suite.type="system/test"
classpath="test.system.classpath"