You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by om...@apache.org on 2009/11/13 22:53:22 UTC
svn commit: r836012 - in /hadoop/hdfs/branches/branch-0.21: ./
.eclipse.templates/.launches/ conf/ lib/ src/contrib/block_forensics/
src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/server/datan...
Author: omalley
Date: Fri Nov 13 21:53:21 2009
New Revision: 836012
URL: http://svn.apache.org/viewvc?rev=836012&view=rev
Log:
HDFS-641. Move all of the components that depend on map/reduce to
map/reduce. (omalley)
Removed:
hadoop/hdfs/branches/branch-0.21/conf/capacity-scheduler.xml.template
hadoop/hdfs/branches/branch-0.21/conf/mapred-queue-acls.xml.template
hadoop/hdfs/branches/branch-0.21/conf/mapred-site.xml.template
hadoop/hdfs/branches/branch-0.21/lib/hadoop-mapred-0.21.0-dev.jar
hadoop/hdfs/branches/branch-0.21/lib/hadoop-mapred-examples-0.21.0-dev.jar
hadoop/hdfs/branches/branch-0.21/lib/hadoop-mapred-test-0.21.0-dev.jar
hadoop/hdfs/branches/branch-0.21/src/contrib/block_forensics/
hadoop/hdfs/branches/branch-0.21/src/test/hdfs-with-mr/
hadoop/hdfs/branches/branch-0.21/src/test/mapred-site.xml
Modified:
hadoop/hdfs/branches/branch-0.21/ (props changed)
hadoop/hdfs/branches/branch-0.21/.eclipse.templates/.launches/ (props changed)
hadoop/hdfs/branches/branch-0.21/CHANGES.txt
hadoop/hdfs/branches/branch-0.21/build.xml (contents, props changed)
hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (props changed)
hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/protocol/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/branch-0.21/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
-/hadoop/hdfs/trunk:818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
Propchange: hadoop/hdfs/branches/branch-0.21/.eclipse.templates/.launches/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1 +1 @@
-/hadoop/hdfs/trunk/.eclipse.templates/.launches:824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/.eclipse.templates/.launches:817853-817863,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
Modified: hadoop/hdfs/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/CHANGES.txt?rev=836012&r1=836011&r2=836012&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.21/CHANGES.txt Fri Nov 13 21:53:21 2009
@@ -291,11 +291,14 @@
HDFS-728. Create a comprehensive functional test for append. (hairong)
- HDFS-736. commitBlockSynchronization() updates block GS and length in-place.
- (shv)
+ HDFS-736. commitBlockSynchronization() updates block GS and length
+ in-place. (shv)
HDFS-521. Create new tests for pipeline (cos)
+ HDFS-641. Move all of the components that depend on map/reduce to
+ map/reduce. (omalley)
+
BUG FIXES
HDFS-76. Better error message to users when commands fail because of
Modified: hadoop/hdfs/branches/branch-0.21/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/build.xml?rev=836012&r1=836011&r2=836012&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/build.xml (original)
+++ hadoop/hdfs/branches/branch-0.21/build.xml Fri Nov 13 21:53:21 2009
@@ -32,7 +32,6 @@
<property name="hadoop-mr.version" value="0.21.0-dev"/>
<property name="final.name" value="${name}-${version}"/>
<property name="test.hdfs.final.name" value="${name}-test-${version}"/>
- <property name="test.hdfswithmr.final.name" value="${name}-hdfswithmr-test-${version}"/>
<property name="ant.final.name" value="${name}-ant-${version}"/>
<property name="year" value="2009"/>
@@ -90,7 +89,6 @@
<property name="test.junit.maxmemory" value="512m" />
<property name="test.hdfs.build.classes" value="${test.build.dir}/classes"/>
- <property name="test.hdfs.with.mr.build.classes" value="${test.build.dir}/hdfs-with-mr/classes"/>
<property name="test.hdfs.with.mr.classpath.id" value="test.hdfs.with.mr.classpath"/>
<property name="test.hdfs.commit.tests.file" value="${test.src.dir}/commit-tests" />
@@ -194,15 +192,6 @@
<path refid="classpath"/>
</path>
- <path id="test.hdfs.with.mr.classpath">
- <path refid="test.classpath"/>
- <pathelement location="${test.hdfs.with.mr.build.classes}" />
- <pathelement location="${lib.dir}/hadoop-mapred-test-${hadoop-mr.version}.jar" />
- <pathelement location="${lib.dir}/hadoop-mapred-${hadoop-mr.version}.jar" />
- <pathelement location="${lib.dir}/hadoop-mapred-tools-${hadoop-mr.version}.jar" />
- <pathelement location="${lib.dir}/hadoop-mapred-examples-${hadoop-mr.version}.jar" />
- </path>
-
<!-- the cluster test classpath: uses conf.dir for configuration -->
<path id="test.cluster.classpath">
<path refid="classpath"/>
@@ -253,12 +242,6 @@
<exclude name="**/*.jsp" />
</fileset>
</copy>
- <unzip src="${lib.dir}/hadoop-mapred-${hadoop-mr.version}.jar"
- dest="${build.dir}">
- <patternset>
- <include name="webapps/**"/>
- </patternset>
- </unzip>
<copy todir="${conf.dir}" verbose="true">
<fileset dir="${conf.dir}" includes="**/*.template"/>
@@ -427,7 +410,7 @@
<!-- -->
<!-- ================================================================== -->
- <target name="jar-test-fault-inject" depends="jar-hdfs-test-fault-inject, jar-hdfswithmr-test-fault-inject"
+ <target name="jar-test-fault-inject" depends="jar-hdfs-test-fault-inject"
description="Make hadoop-test.jar files"/>
<target name="jar-hdfs-test-fault-inject" description="Make hadoop-test-fi.jar">
@@ -437,14 +420,6 @@
</subant>
</target>
- <target name="jar-hdfswithmr-test-fault-inject" description="Make hadoop-hdfswithmr-test-fi.jar">
- <subant buildpath="build.xml" target="jar-hdfswithmr-test">
- <property name="build.dir" value="${build-fi.dir}"/>
- <property name="test.hdfswithmr.final.name"
- value="${name}-hdsfwithmr-test-${version}-fi"/>
- </subant>
- </target>
-
<!--End of Fault Injection (FI) related session-->
<target name="compile-core" depends="clover, compile-hdfs-classes" description="Compile"/>
@@ -510,30 +485,12 @@
<copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19" todir="${test.cache.data}"/>
</target>
- <target name="compile-hdfs-with-mr-test" depends="compile-hdfs-test">
- <mkdir dir="${test.hdfs.with.mr.build.classes}"/>
- <javac
- encoding="${build.encoding}"
- srcdir="${test.src.dir}/hdfs-with-mr"
- includes="org/apache/hadoop/**/*.java"
- destdir="${test.hdfs.with.mr.build.classes}"
- debug="${javac.debug}"
- optimize="${javac.optimize}"
- target="${javac.version}"
- source="${javac.version}"
- deprecation="${javac.deprecation}">
- <compilerarg line="${javac.args} ${javac.args.warnings}" />
- <classpath refid="test.hdfs.with.mr.classpath"/>
- </javac>
- </target>
-
-
<!-- ================================================================== -->
<!-- Make hadoop-test.jar -->
<!-- ================================================================== -->
<!-- -->
<!-- ================================================================== -->
- <target name="jar-test" depends="jar-hdfs-test, jar-hdfswithmr-test" description="Make hadoop-test.jar"/>
+ <target name="jar-test" depends="jar-hdfs-test" description="Make hadoop-test.jar"/>
<target name="jar-hdfs-test" depends="compile-hdfs-test" description="Make hadoop-hdfs-test.jar">
<copy todir="${test.build.classes}">
@@ -553,24 +510,6 @@
</jar>
</target>
- <target name="jar-hdfswithmr-test" depends="compile-hdfs-with-mr-test" description="Make hadoop-hdfswithmr-test.jar">
- <copy todir="${test.build.classes}">
- <fileset dir="${test.hdfs.with.mr.build.classes}"/>
- </copy>
- <jar jarfile="${build.dir}/${test.hdfswithmr.final.name}.jar"
- basedir="${test.build.classes}">
- <manifest>
- <attribute name="Main-Class"
- value="org/apache/hadoop/test/HdfsWithMRTestDriver"/>
- <section name="org/apache/hadoop">
- <attribute name="Implementation-Title" value="${ant.project.name}"/>
- <attribute name="Implementation-Version" value="${version}"/>
- <attribute name="Implementation-Vendor" value="Apache"/>
- </section>
- </manifest>
- </jar>
- </target>
-
<condition property="tests.notestcase">
<and>
<isfalse value="${test.fault.inject}"/>
@@ -677,67 +616,15 @@
<macro-test-runner test.file="${test.hdfs.all.tests.file}" suite.type="unit"/>
</target>
- <target name="run-test-hdfs-with-mr" depends="compile-hdfs-with-mr-test" description="Run hdfs unit tests that require mapred">
-
- <delete dir="${test.build.data}"/>
- <mkdir dir="${test.build.data}"/>
- <delete dir="${test.log.dir}"/>
- <mkdir dir="${test.log.dir}"/>
- <copy file="${test.src.dir}/hadoop-policy.xml"
- todir="${test.build.extraconf}" />
- <copy file="${test.src.dir}/fi-site.xml"
- todir="${test.build.extraconf}" />
- <junit showoutput="${test.output}"
- printsummary="${test.junit.printsummary}"
- haltonfailure="${test.junit.haltonfailure}"
- fork="yes"
- forkmode="${test.junit.fork.mode}"
- maxmemory="${test.junit.maxmemory}"
- dir="${basedir}" timeout="${test.timeout}"
- errorProperty="tests.failed" failureProperty="tests.failed">
- <sysproperty key="test.build.data" value="${test.build.data}"/>
- <sysproperty key="test.cache.data" value="${test.cache.data}"/>
- <sysproperty key="test.debug.data" value="${test.debug.data}"/>
- <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
- <sysproperty key="test.src.dir" value="${test.src.dir}"/>
- <sysproperty key="test.build.extraconf" value="${test.build.extraconf}" />
- <sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
- <classpath refid="test.hdfs.with.mr.classpath"/>
- <syspropertyset id="FaultProbabilityProperties">
- <propertyref regex="fi.*"/>
- </syspropertyset>
- <formatter type="${test.junit.output.format}" />
- <batchtest todir="${test.build.dir}" if="tests.notestcase">
- <fileset dir="${test.src.dir}/hdfs-with-mr"
- includes="**/${test.include}.java"
- excludes="**/${test.exclude}.java" />
- </batchtest>
- <batchtest todir="${test.build.dir}" if="tests.testcase">
- <fileset dir="${test.src.dir}/hdfs-with-mr" includes="**/${testcase}.java"/>
- </batchtest>
- </junit>
- <antcall target="checkfailure"/>
- </target>
-
<target name="checkfailure" if="tests.failed">
<touch file="${test.build.dir}/testsfailed"/>
<fail unless="continueOnFailure">Tests failed!</fail>
</target>
- <target name="test-contrib" depends="compile, compile-hdfs-with-mr-test" description="Run contrib unit tests">
- <subant target="test">
- <property name="version" value="${version}"/>
- <property name="hadoop-version" value="${hadoop-core.version}"/>
- <property name="clover.jar" value="${clover.jar}"/>
- <fileset file="${contrib.dir}/build.xml"/>
- </subant>
- </target>
-
- <target name="test-core" description="Run core, hdfs and mapred unit tests">
+ <target name="test-core" description="Run hdfs unit tests">
<delete file="${test.build.dir}/testsfailed"/>
<property name="continueOnFailure" value="true"/>
<antcall target="run-test-hdfs"/>
- <antcall target="run-test-hdfs-with-mr"/>
<antcall target="run-test-unit"/>
<antcall target="run-test-hdfs-fault-inject"/>
<available file="${test.build.dir}/testsfailed" property="testsfailed"/>
Propchange: hadoop/hdfs/branches/branch-0.21/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/build.xml:713112
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
-/hadoop/hdfs/trunk/build.xml:818294-818298,824552,824944,825229,826149,828116,828926,829258,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/build.xml:817853-817863,818294-818298,824552,824944,825229,826149,828116,828926,829258,829880,829894,830003,831436,832043,833499,835728
Propchange: hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/contrib/hdfsproxy:713112
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
Propchange: hadoop/hdfs/branches/branch-0.21/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
-/hadoop/hdfs/trunk/src/java:818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/src/java:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
Propchange: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1,5 +1,5 @@
/hadoop/core/branches/branch-0.19/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:713112
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:776175-785643,785929-786278
/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:817353-818319,818321-818553
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java:824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java:817853-817863,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:796829-800617,800619-803337,804756-805652,808672-809439,811495-813103,813105-813630,814223-815964,818294-818298
Propchange: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -3,4 +3,4 @@
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DatanodeBlockInfo.java:776175-785643,785929-786278
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
Propchange: hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/protocol/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1 +1 @@
-/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/protocol:824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/protocol:817853-817863,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
Propchange: hadoop/hdfs/branches/branch-0.21/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
-/hadoop/hdfs/trunk/src/test/hdfs:818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/src/test/hdfs:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
Propchange: hadoop/hdfs/branches/branch-0.21/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
-/hadoop/hdfs/trunk/src/webapps/datanode:818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/src/webapps/datanode:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
Propchange: hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
-/hadoop/hdfs/trunk/src/webapps/hdfs:818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/src/webapps/hdfs:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
Propchange: hadoop/hdfs/branches/branch-0.21/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 13 21:53:21 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
-/hadoop/hdfs/trunk/src/webapps/secondary:818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728
+/hadoop/hdfs/trunk/src/webapps/secondary:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,832043,833499,835728