You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/11/02 06:35:03 UTC
svn commit: r1196458 [18/19] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ assembly/
bin/ conf/ dev-support/ hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/
hadoop-mapreduce-client/hadoop-mapreduce-cl...
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm Wed Nov 2 05:34:31 2011
@@ -21,7 +21,7 @@ Hadoop MapReduce Next Generation - Writi
\[ {{{./index.html}Go Back}} \]
-%{toc|section=1|fromDepth=1}
+%{toc|section=1|fromDepth=0}
* Purpose
@@ -323,25 +323,29 @@ Hadoop MapReduce Next Generation - Writi
multi-tenancy nature, amongst other issues, it cannot make any assumptions
of things like pre-configured ports that it can listen on.
+ * When the ApplicationMaster starts up, several parameters are made available
+ to it via the environment. These include the ContainerId for the
+ ApplicationMaster container, the application submission time and details
+ about the NodeManager host running the Application Master.
+ Ref ApplicationConstants for parameter names.
+
* All interactions with the ResourceManager require an ApplicationAttemptId
- (there can be multiple attempts per application in case of failures). When
- the ApplicationMaster starts up, the ApplicationAttemptId associated with
- this particular instance will be set in the environment. There are helper
- apis to convert the value obtained from the environment into an
- ApplicationAttemptId object.
+ (there can be multiple attempts per application in case of failures). The
+ ApplicationAttemptId can be obtained from the ApplicationMaster
+ containerId. There are helper apis to convert the value obtained from the
+ environment into objects.
+---+
Map<String, String> envs = System.getenv();
- ApplicationAttemptId appAttemptID =
- Records.newRecord(ApplicationAttemptId.class);
- if (!envs.containsKey(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV)) {
- // app attempt id should always be set in the env by the framework
+ String containerIdString =
+ envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV);
+ if (containerIdString == null) {
+ // container id should always be set in the env by the framework
throw new IllegalArgumentException(
- "ApplicationAttemptId not set in the environment");
- }
- appAttemptID =
- ConverterUtils.toApplicationAttemptId(
- envs.get(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV));
+ "ContainerId not set in the environment");
+ }
+ ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
+ ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
+---+
* After an ApplicationMaster has initialized itself completely, it needs to
@@ -402,7 +406,8 @@ Hadoop MapReduce Next Generation - Writi
* Resource capability: Currently, YARN only supports memory based resource
requirements so the request should define how much memory is needed. The
value is defined in MB and has to less than the max capability of the
- cluster and an exact multiple of the min capability.
+ cluster and an exact multiple of the min capability. Memory resources
+ correspond to physical memory limits imposed on the task containers.
* Priority: When asking for sets of containers, an ApplicationMaster may
define different priorities to each set. For example, the Map-Reduce
@@ -770,8 +775,9 @@ Hadoop MapReduce Next Generation - Writi
The two things you're interested in are physical memory and virtual memory.
If you have exceeded physical memory limits your app is using too much physical
memory. If you're running a Java app, you can use -hprof to look at what is
- taking up space in the heap. If you have exceeded virtual memory, things are
- slightly more complicated.
+ taking up space in the heap. If you have exceeded virtual memory, you may
+ need to increase the value of the the cluster-wide configuration variable
+ <<<yarn.nodemanager.vmem-pmem-ratio>>>.
* Useful Links
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm Wed Nov 2 05:34:31 2011
@@ -11,14 +11,12 @@
~~ limitations under the License. See accompanying LICENSE file.
---
- Hadoop MapReduce Next Generation ${project.version}
+ Apache Hadoop NextGen MapReduce
---
---
${maven.build.timestamp}
-
-Hadoop MapReduce Next Generation
-
-* Architecture
+
+MapReduce NextGen aka YARN aka MRv2
The new architecture introduced in hadoop-0.23, divides the two major
functions of the JobTracker: resource management and job life-cycle management
@@ -32,18 +30,21 @@ Hadoop MapReduce Next Generation
or a DAG of such jobs.
The ResourceManager and per-machine NodeManager daemon, which manages the
- user processes on that machine, form the computation fabric. The
- per-application ApplicationMaster is, in effect, a framework specific library
- and is tasked with negotiating resources from the ResourceManager and working
- with the NodeManager(s) to execute and monitor the tasks.
-
-* User Documentation
+ user processes on that machine, form the computation fabric.
+
+ The per-application ApplicationMaster is, in effect, a framework specific
+ library and is tasked with negotiating resources from the ResourceManager and
+ working with the NodeManager(s) to execute and monitor the tasks.
- * {{{./SingleCluster.html}Setting up a Single Node Cluster}}
+ More details are available in the {{{./YARN.html}YARN}}
+ document.
- * {{{./apidocs/index.html}JavaDocs}}
+* Documentation
+ * {{{./YARN.html}NextGen MapReduce}}
+
* {{{./WritingYarnApplications.html}Writing Yarn Applications}}
* {{{./CapacityScheduler.html}Capacity Scheduler}}
+
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/site.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/site.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/site.xml Wed Nov 2 05:34:31 2011
@@ -11,18 +11,12 @@
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
-<project name="Hadoop MapReduce Next Gen">
-
- <version position="right"/>
-
- <bannerLeft>
- <name> </name>
- </bannerLeft>
+<project name="Apache Hadoop 0.23">
<skin>
<groupId>org.apache.maven.skins</groupId>
<artifactId>maven-stylus-skin</artifactId>
- <version>1.1</version>
+ <version>1.2</version>
</skin>
<body>
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/pom.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/pom.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/pom.xml Wed Nov 2 05:34:31 2011
@@ -14,24 +14,37 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>0.24.0-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn</artifactId>
- <version>${yarn.version}</version>
+ <version>0.24.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hadoop-yarn</name>
- <url>http://hadoop.apache.org/mapreduce</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<test.logs>true</test.logs>
<test.timeout>600000</test.timeout>
- <hadoop-common.version>0.24.0-SNAPSHOT</hadoop-common.version>
- <hadoop-hdfs.version>0.24.0-SNAPSHOT</hadoop-hdfs.version>
- <yarn.version>0.24.0-SNAPSHOT</yarn.version>
- <install.pom>${project.build.directory}/saner-pom.xml</install.pom>
- <install.file>${install.pom}</install.file>
<yarn.basedir>${basedir}</yarn.basedir>
</properties>
+
+ <distributionManagement>
+ <repository>
+ <id>apache.releases.https</id>
+ <name>Apache Release Distribution Repository</name>
+ <url>https://repository.apache.org/service/local/staging/deploy/maven2</url>
+ </repository>
+ <snapshotRepository>
+ <id>apache.snapshots.https</id>
+ <name>Apache Development Snapshot Repository</name>
+ <url>https://repository.apache.org/content/repositories/snapshots</url>
+ </snapshotRepository>
+ </distributionManagement>
<repositories>
<repository>
@@ -54,6 +67,126 @@
<dependencies>
<dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.4.0a</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.5.3</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>ant</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>paranamer-ant</artifactId>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1-jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>hsqldb</groupId>
+ <artifactId>hsqldb</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.6.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>1.6.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.8.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <!-- needed for security and runtime -->
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ <version>2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.2.3.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>com.cenqua.clover</groupId>
+ <artifactId>clover</artifactId>
+ <version>3.0.2</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.5.3</version>
@@ -92,7 +225,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop-common.version}</version>
+ <version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.avro</groupId>
@@ -123,7 +256,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
- <version>${hadoop-common.version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -132,6 +265,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.1</version>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.8.5</version>
@@ -140,14 +278,14 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop-common.version}</version>
+ <version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop-hdfs.version}</version>
+ <version>${project.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
@@ -182,33 +320,38 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
- <version>${yarn.version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
- <version>${yarn.version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
- <version>${yarn.version}</version>
+ <version>${project.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
- <version>${yarn.version}</version>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>${yarn.version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <version>${yarn.version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
@@ -256,26 +399,6 @@
<version>2.4.1</version>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <!-- requires 2.5+ to make system properties work -->
- <!-- requires 2.7+ to avoid SUREFIRE-640 -->
- <version>2.7.2</version>
- <configuration>
- <failIfNoTests>false</failIfNoTests>
- <redirectTestOutputToFile>${test.logs}</redirectTestOutputToFile>
- <forkedProcessTimeoutInSeconds>${test.timeout}</forkedProcessTimeoutInSeconds>
- <environmentVariables>
- <JAVA_HOME>${java.home}</JAVA_HOME>
- </environmentVariables>
- <systemPropertyVariables>
- <build.dir>${project.build.directory}</build.dir>
- <build.output.dir>${project.build.outputDirectory}</build.output.dir>
- <log4j.configuration>file:///${yarn.basedir}/../src/test/log4j.properties</log4j.configuration>
- </systemPropertyVariables>
- </configuration>
- </plugin>
- <plugin>
<groupId>com.atlassian.maven.plugins</groupId>
<artifactId>maven-clover2-plugin</artifactId>
<version>3.0.2</version>
@@ -316,66 +439,6 @@
</plugins>
</pluginManagement>
<plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <tarLongFileMode>gnu</tarLongFileMode>
- <descriptors>
- <descriptor>assembly/all.xml</descriptor>
- </descriptors>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>santize-pom</id>
- <phase>package</phase>
- <configuration>
- <target>
- <echo message="project.build.directory: ${project.build.directory}"/>
- <copy file="pom.xml" tofile="${install.pom}">
- <filterchain>
- <!-- we'll have to wait for ant 1.8.3 for the following
- <expandproperties>
- <propertyset regex=".*version$">
- </propertyset>
- </expandproperties>
- until then an even uglier workaround: -->
- <tokenfilter>
- <replaceregex pattern="\$\{hadoop-common.version}"
- replace="${hadoop-common.version}" flags="g"/>
- <replaceregex pattern="\$\{hadoop-hdfs.version}"
- replace="${hadoop-hdfs.version}" flags="g"/>
- <replaceregex pattern="\$\{yarn.version}"
- replace="${yarn.version}" flags="g"/>
- </tokenfilter>
- </filterchain>
- </copy>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-install-plugin</artifactId>
- <executions>
- <execution>
- <id>install-sanitized-pom</id>
- <configuration>
- <file>${install.file}</file>
- <pomFile>${install.pom}</pomFile>
- </configuration>
- <phase>install</phase>
- <goals>
- <goal>install-file</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ivy.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ivy.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ivy.xml Wed Nov 2 05:34:31 2011
@@ -87,6 +87,8 @@
rev="${yarn.version}" conf="compile->default"/>
<dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core"
rev="${yarn.version}" conf="compile->default"/>
+ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common"
+ rev="${yarn.version}" conf="compile->default"/>
<dependency org="org.apache.hadoop" name="hadoop-yarn-common"
rev="${yarn.version}" conf="compile->default"/>
<dependency org="log4j" name="log4j" rev="${log4j.version}"
@@ -148,6 +150,7 @@
<exclude org="javax.jms"/>
<exclude org="javax.mail"/>
<exclude org="org.apache.hadoop" module="avro"/>
+ <exclude org="org.apache.commons" module="commons-daemon"/>
</dependencies>
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/pom.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/pom.xml Wed Nov 2 05:34:31 2011
@@ -15,9 +15,15 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>0.24.0-SNAPSHOT</version>
+ <relativePath>../hadoop-project</relativePath>
+ </parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce</artifactId>
- <version>${hadoop-mapreduce.version}</version>
+ <version>0.24.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hadoop-mapreduce</name>
<url>http://hadoop.apache.org/mapreduce/</url>
@@ -26,35 +32,15 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<test.logs>true</test.logs>
<test.timeout>600000</test.timeout>
- <hadoop-common.version>0.24.0-SNAPSHOT</hadoop-common.version>
- <hadoop-hdfs.version>0.24.0-SNAPSHOT</hadoop-hdfs.version>
- <hadoop-mapreduce.version>0.24.0-SNAPSHOT</hadoop-mapreduce.version>
- <yarn.version>0.24.0-SNAPSHOT</yarn.version>
- <install.pom>${project.build.directory}/saner-pom.xml</install.pom>
- <install.file>${install.pom}</install.file>
<fork.mode>once</fork.mode>
<mr.basedir>${basedir}</mr.basedir>
</properties>
-
- <repositories>
- <repository>
- <id>repository.jboss.org</id>
- <url>http://repository.jboss.org/nexus/content/groups/public/</url>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache.snapshots</id>
- <url>http://repository.apache.org/snapshots</url>
- <!-- until we move to hadoop-common/hdfs trunk and/or maven 3 -->
- <!-- cf. MNG-4326 -->
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
+
+ <modules>
+ <module>hadoop-yarn</module>
+ <module>hadoop-mapreduce-client</module>
+ </modules>
+
<dependencies>
<dependency>
<groupId>com.google.protobuf</groupId>
@@ -95,7 +81,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop-common.version}</version>
+ <version>${project.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -120,7 +106,7 @@
</exclusion>
</exclusions>
</dependency>
-
+
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@@ -134,7 +120,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
- <version>${hadoop-common.version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -145,7 +131,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop-common.version}</version>
+ <version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -153,7 +139,7 @@
<groupId>org.apache.hadoop</groupId>
<!-- needed for security and runtime -->
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop-hdfs.version}</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
@@ -171,13 +157,18 @@
<version>3.2.3.Final</version>
</dependency>
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.1</version>
+ </dependency>
+ <dependency>
<groupId>com.cenqua.clover</groupId>
<artifactId>clover</artifactId>
<version>3.0.2</version>
</dependency>
-
+
</dependencies>
-
+
<build>
<pluginManagement>
<plugins>
@@ -210,27 +201,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <!-- requires 2.5+ to make system properties work -->
- <!-- requires 2.7+ to avoid SUREFIRE-640 -->
- <version>2.7.2</version>
- <configuration>
- <failIfNoTests>false</failIfNoTests>
- <redirectTestOutputToFile>${test.logs}</redirectTestOutputToFile>
- <forkedProcessTimeoutInSeconds>${test.timeout}</forkedProcessTimeoutInSeconds>
- <forkMode>${fork.mode}</forkMode>
- <environmentVariables>
- <JAVA_HOME>${java.home}</JAVA_HOME>
- </environmentVariables>
- <systemPropertyVariables>
- <build.dir>${project.build.directory}</build.dir>
- <build.output.dir>${project.build.outputDirectory}</build.output.dir>
- <log4j.configuration>file:///${mr.basedir}/src/test/log4j.properties</log4j.configuration>
- </systemPropertyVariables>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.2.1</version>
</plugin>
@@ -271,64 +241,35 @@
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
- <id>santize-pom</id>
+ <id>tar</id>
<phase>package</phase>
- <configuration>
- <target>
- <copy file="pom.xml" tofile="${install.pom}">
- <filterchain>
- <!-- we'll have to wait for ant 1.8.3 for the following
- <expandproperties>
- <propertyset regex=".*version$">
- </propertyset>
- </expandproperties>
- until then an even uglier workaround: -->
- <tokenfilter>
- <replaceregex pattern="\$\{hadoop-common.version}"
- replace="${hadoop-common.version}" flags="g"/>
- <replaceregex pattern="\$\{hadoop-hdfs.version}"
- replace="${hadoop-hdfs.version}" flags="g"/>
- <replaceregex pattern="\$\{hadoop-mapreduce.version}"
- replace="${hadoop-mapreduce.version}" flags="g"/>
- <replaceregex pattern="\$\{yarn.version}"
- replace="${yarn.version}" flags="g"/>
- </tokenfilter>
- </filterchain>
- </copy>
- </target>
- </configuration>
<goals>
<goal>run</goal>
</goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-install-plugin</artifactId>
- <executions>
- <execution>
- <id>install-sanitized-pom</id>
<configuration>
- <file>${install.file}</file>
- <pomFile>${install.pom}</pomFile>
+ <!-- this is identical from hadoop-project-dist, eventually they must be unified -->
+ <target if="tar">
+ <!-- Using Unix script to preserve symlinks -->
+ <echo file="${project.build.directory}/dist-maketar.sh">
+
+ which cygpath 2> /dev/null
+ if [ $? = 1 ]; then
+ BUILD_DIR="${project.build.directory}"
+ else
+ BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+ fi
+ cd $BUILD_DIR
+ tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version}
+ </echo>
+ <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <arg line="./dist-maketar.sh"/>
+ </exec>
+ </target>
</configuration>
- <phase>install</phase>
- <goals>
- <goal>install-file</goal>
- </goals>
</execution>
</executions>
</plugin>
<plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <tarLongFileMode>gnu</tarLongFileMode>
- <descriptors>
- <descriptor>assembly/all.xml</descriptor>
- </descriptors>
- </configuration>
- </plugin>
- <plugin>
<groupId>com.atlassian.maven.plugins</groupId>
<artifactId>maven-clover2-plugin</artifactId>
<executions>
@@ -380,14 +321,49 @@
</executions>
</plugin>
</plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>dist</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <descriptorRefs>
+ <descriptorRef>hadoop-mapreduce-dist</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
</build>
</profile>
</profiles>
- <modules>
- <module>hadoop-yarn</module>
- <module>hadoop-mapreduce-client</module>
- </modules>
<reporting>
<plugins>
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1196451
/hadoop/core/branches/branch-0.19/mapred/src/c++:713112
/hadoop/core/trunk/src/c++:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1196451
/hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
/hadoop/core/trunk/src/contrib:784664-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1196451
/hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112
/hadoop/core/branches/branch-0.19/mapred/src/contrib/block_forensics:713112
/hadoop/core/trunk/src/contrib/block_forensics:784664-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1196451
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
/hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml Wed Nov 2 05:34:31 2011
@@ -60,13 +60,8 @@
<subant target="test">
<property name="continueOnFailure" value="true"/>
<fileset dir="." includes="streaming/build.xml"/>
- <fileset dir="." includes="fairscheduler/build.xml"/>
- <fileset dir="." includes="capacity-scheduler/build.xml"/>
- <fileset dir="." includes="dynamic-scheduler/build.xml"/>
<fileset dir="." includes="gridmix/build.xml"/>
<fileset dir="." includes="vertica/build.xml"/>
- <!-- mumak tests disabled due to timeouts. See MAPREDUCE-2348
- <fileset dir="." includes="mumak/build.xml"/> -->
<fileset dir="." includes="raid/build.xml"/>
</subant>
<available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
@@ -88,20 +83,12 @@
value="${hadoop.conf.dir.deployed}"/>
<fileset dir="." includes="hdfsproxy/build.xml"/>
<fileset dir="." includes="streaming/build.xml"/>
- <fileset dir="." includes="fairscheduler/build.xml"/>
- <fileset dir="." includes="capacity-scheduler/build.xml"/>
<fileset dir="." includes="gridmix/build.xml"/>
</subant>
<available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
<fail if="testsfailed">Tests failed!</fail>
</target>
- <target name="docs">
- <subant target="docs">
- <fileset dir="." includes="capacity-scheduler/build.xml"/>
- </subant>
- </target>
-
<!-- ====================================================== -->
<!-- Clean all the contribs. -->
<!-- ====================================================== -->
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1196451
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
/hadoop/core/trunk/src/contrib/build.xml:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1196451
/hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
/hadoop/core/trunk/src/contrib/data_join:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1196451
/hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
/hadoop/core/branches/branch-0.19/mapred/src/contrib/eclipse-plugin:713112
/hadoop/core/trunk/src/contrib/eclipse-plugin:776175-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java Wed Nov 2 05:34:31 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.Mappe
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.TaskInputOutputContext;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
@@ -72,7 +73,7 @@ class LoadJob extends GridmixJob {
job.setNumReduceTasks(jobdesc.getNumberReduces());
job.setMapOutputKeyClass(GridmixKey.class);
job.setMapOutputValueClass(GridmixRecord.class);
- job.setSortComparatorClass(GridmixKey.Comparator.class);
+ job.setSortComparatorClass(LoadSortComparator.class);
job.setGroupingComparatorClass(SpecGroupingComparator.class);
job.setInputFormatClass(LoadInputFormat.class);
job.setOutputFormatClass(RawBytesOutputFormat.class);
@@ -94,17 +95,84 @@ class LoadJob extends GridmixJob {
}
/**
+ * This is a load matching key comparator which will make sure that the
+ * resource usage load is matched even when the framework is in control.
+ */
+ public static class LoadSortComparator extends GridmixKey.Comparator {
+ private ResourceUsageMatcherRunner matcher = null;
+ private boolean isConfigured = false;
+
+ public LoadSortComparator() {
+ super();
+ }
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ configure();
+ int ret = super.compare(b1, s1, l1, b2, s2, l2);
+ if (matcher != null) {
+ try {
+ matcher.match(); // match the resource usage now
+ } catch (Exception e) {}
+ }
+ return ret;
+ }
+
+ //TODO Note that the sorter will be instantiated 2 times as follows
+ // 1. During the sort/spill in the map phase
+ // 2. During the merge in the sort phase
+ // We need the handle to the matcher thread only in (2).
+ // This logic can be relaxed to run only in (2).
+ private void configure() {
+ if (!isConfigured) {
+ ThreadGroup group = Thread.currentThread().getThreadGroup();
+ Thread[] threads = new Thread[group.activeCount() * 2];
+ group.enumerate(threads, true);
+ for (Thread t : threads) {
+ if (t != null && (t instanceof ResourceUsageMatcherRunner)) {
+ this.matcher = (ResourceUsageMatcherRunner) t;
+ isConfigured = true;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ /**
* This is a progress based resource usage matcher.
*/
@SuppressWarnings("unchecked")
- static class ResourceUsageMatcherRunner extends Thread {
+ static class ResourceUsageMatcherRunner extends Thread
+ implements Progressive {
private final ResourceUsageMatcher matcher;
- private final Progressive progress;
+ private final BoostingProgress progress;
private final long sleepTime;
private static final String SLEEP_CONFIG =
"gridmix.emulators.resource-usage.sleep-duration";
private static final long DEFAULT_SLEEP_TIME = 100; // 100ms
+ /**
+ * This is a progress bar that can be boosted for weaker use-cases.
+ */
+ private static class BoostingProgress implements Progressive {
+ private float boostValue = 0f;
+ TaskInputOutputContext context;
+
+ BoostingProgress(TaskInputOutputContext context) {
+ this.context = context;
+ }
+
+ void setBoostValue(float boostValue) {
+ this.boostValue = boostValue;
+ }
+
+ @Override
+ public float getProgress() {
+ return Math.min(1f, context.getProgress() + boostValue);
+ }
+ }
+
ResourceUsageMatcherRunner(final TaskInputOutputContext context,
ResourceUsageMetrics metrics) {
Configuration conf = context.getConfiguration();
@@ -118,19 +186,14 @@ class LoadJob extends GridmixJob {
// set the other parameters
this.sleepTime = conf.getLong(SLEEP_CONFIG, DEFAULT_SLEEP_TIME);
- progress = new Progressive() {
- @Override
- public float getProgress() {
- return context.getProgress();
- }
- };
+ progress = new BoostingProgress(context);
// instantiate a resource-usage-matcher
matcher = new ResourceUsageMatcher();
matcher.configure(conf, plugin, metrics, progress);
}
- protected void match() throws Exception {
+ protected void match() throws IOException, InterruptedException {
// match the resource usage
matcher.matchResourceUsage();
}
@@ -157,21 +220,34 @@ class LoadJob extends GridmixJob {
+ " thread! Exiting.", e);
}
}
+
+ @Override
+ public float getProgress() {
+ return matcher.getProgress();
+ }
+
+ // boost the progress bar as fasten up the emulation cycles.
+ void boost(float value) {
+ progress.setBoostValue(value);
+ }
}
// Makes sure that the TaskTracker doesn't kill the map/reduce tasks while
// they are emulating
private static class StatusReporter extends Thread {
- private TaskAttemptContext context;
- StatusReporter(TaskAttemptContext context) {
+ private final TaskAttemptContext context;
+ private final Progressive progress;
+
+ StatusReporter(TaskAttemptContext context, Progressive progress) {
this.context = context;
+ this.progress = progress;
}
@Override
public void run() {
LOG.info("Status reporter thread started.");
try {
- while (context.getProgress() < 1) {
+ while (!isInterrupted() && progress.getProgress() < 1) {
// report progress
context.progress();
@@ -275,9 +351,11 @@ class LoadJob extends GridmixJob {
matcher = new ResourceUsageMatcherRunner(ctxt,
split.getMapResourceUsageMetrics());
+ matcher.setDaemon(true);
// start the status reporter thread
- reporter = new StatusReporter(ctxt);
+ reporter = new StatusReporter(ctxt, matcher);
+ reporter.setDaemon(true);
reporter.start();
}
@@ -324,6 +402,17 @@ class LoadJob extends GridmixJob {
}
}
+ // check if the thread will get a chance to run or not
+ // check if there will be a sort&spill->merge phase or not
+ // check if the final sort&spill->merge phase is gonna happen or not
+ if (context.getNumReduceTasks() > 0
+ && context.getCounter(TaskCounter.SPILLED_RECORDS).getValue() == 0) {
+ LOG.info("Boosting the map phase progress.");
+ // add the sort phase progress to the map phase and emulate
+ matcher.boost(0.33f);
+ matcher.match();
+ }
+
// start the matcher thread since the map phase ends here
matcher.start();
}
@@ -392,7 +481,7 @@ class LoadJob extends GridmixJob {
matcher = new ResourceUsageMatcherRunner(context, metrics);
// start the status reporter thread
- reporter = new StatusReporter(context);
+ reporter = new StatusReporter(context, matcher);
reporter.start();
}
@Override
@@ -528,9 +617,13 @@ class LoadJob extends GridmixJob {
specRecords[j] = info.getOutputRecords();
metrics[j] = info.getResourceUsageMetrics();
if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("SPEC(%d) %d -> %d %d %d", id(), i,
+ LOG.debug(String.format("SPEC(%d) %d -> %d %d %d %d %d %d %d", id(), i,
i + j * maps, info.getOutputRecords(),
- info.getOutputBytes()));
+ info.getOutputBytes(),
+ info.getResourceUsageMetrics().getCumulativeCpuUsage(),
+ info.getResourceUsageMetrics().getPhysicalMemoryUsage(),
+ info.getResourceUsageMetrics().getVirtualMemoryUsage(),
+ info.getResourceUsageMetrics().getHeapUsage()));
}
}
final TaskInfo info = jobdesc.getTaskInfo(TaskType.MAP, i);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java Wed Nov 2 05:34:31 2011
@@ -67,7 +67,7 @@ implements ResourceUsageEmulatorPlugin {
private float emulationInterval; // emulation interval
private long targetCpuUsage = 0;
private float lastSeenProgress = 0;
- private long lastSeenCpuUsageCpuUsage = 0;
+ private long lastSeenCpuUsage = 0;
// Configuration parameters
public static final String CPU_EMULATION_PROGRESS_INTERVAL =
@@ -229,6 +229,15 @@ implements ResourceUsageEmulatorPlugin {
return progress * progress * progress * progress;
}
+ private synchronized long getCurrentCPUUsage() {
+ return monitor.getProcResourceValues().getCumulativeCpuTime();
+ }
+
+ @Override
+ public float getProgress() {
+ return Math.min(1f, ((float)getCurrentCPUUsage())/targetCpuUsage);
+ }
+
@Override
//TODO Multi-threading for speedup?
public void emulate() throws IOException, InterruptedException {
@@ -249,10 +258,9 @@ implements ResourceUsageEmulatorPlugin {
// Note that (Cc-Cl)/(Pc-Pl) is termed as 'rate' in the following
// section
- long currentCpuUsage =
- monitor.getProcResourceValues().getCumulativeCpuTime();
+ long currentCpuUsage = getCurrentCPUUsage();
// estimate the cpu usage rate
- float rate = (currentCpuUsage - lastSeenCpuUsageCpuUsage)
+ float rate = (currentCpuUsage - lastSeenCpuUsage)
/ (currentProgress - lastSeenProgress);
long projectedUsage =
currentCpuUsage + (long)((1 - currentProgress) * rate);
@@ -264,8 +272,7 @@ implements ResourceUsageEmulatorPlugin {
(long)(targetCpuUsage
* getWeightForProgressInterval(currentProgress));
- while (monitor.getProcResourceValues().getCumulativeCpuTime()
- < currentWeighedTarget) {
+ while (getCurrentCPUUsage() < currentWeighedTarget) {
emulatorCore.compute();
// sleep for 100ms
try {
@@ -281,8 +288,7 @@ implements ResourceUsageEmulatorPlugin {
// set the last seen progress
lastSeenProgress = progress.getProgress();
// set the last seen usage
- lastSeenCpuUsageCpuUsage =
- monitor.getProcResourceValues().getCumulativeCpuTime();
+ lastSeenCpuUsage = getCurrentCPUUsage();
}
}
}
@@ -310,6 +316,6 @@ implements ResourceUsageEmulatorPlugin {
// initialize the states
lastSeenProgress = 0;
- lastSeenCpuUsageCpuUsage = 0;
+ lastSeenCpuUsage = 0;
}
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java Wed Nov 2 05:34:31 2011
@@ -42,7 +42,7 @@ import org.apache.hadoop.conf.Configurat
* For configuring GridMix to load and and use a resource usage emulator,
* see {@link ResourceUsageMatcher}.
*/
-public interface ResourceUsageEmulatorPlugin {
+public interface ResourceUsageEmulatorPlugin extends Progressive {
/**
* Initialize the plugin. This might involve
* - initializing the variables
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java Wed Nov 2 05:34:31 2011
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.mapred.gridmix.emulators.resourceusage;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -35,7 +36,7 @@ import org.apache.hadoop.util.Reflection
* <p>Note that the order in which the emulators are invoked is same as the
* order in which they are configured.
*/
-public class ResourceUsageMatcher {
+public class ResourceUsageMatcher implements Progressive {
/**
* Configuration key to set resource usage emulators.
*/
@@ -80,10 +81,31 @@ public class ResourceUsageMatcher {
}
}
- public void matchResourceUsage() throws Exception {
+ public void matchResourceUsage() throws IOException, InterruptedException {
for (ResourceUsageEmulatorPlugin emulator : emulationPlugins) {
// match the resource usage
emulator.emulate();
}
}
+
+ /**
+ * Returns the average progress.
+ */
+ @Override
+ public float getProgress() {
+ if (emulationPlugins.size() > 0) {
+ // return the average progress
+ float progress = 0f;
+ for (ResourceUsageEmulatorPlugin emulator : emulationPlugins) {
+ // consider weighted progress of each emulator
+ progress += emulator.getProgress();
+ }
+
+ return progress / emulationPlugins.size();
+ }
+
+ // if no emulators are configured then return 1
+ return 1f;
+
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java Wed Nov 2 05:34:31 2011
@@ -187,6 +187,11 @@ implements ResourceUsageEmulatorPlugin {
}
@Override
+ public float getProgress() {
+ return Math.min(1f, ((float)getTotalHeapUsageInMB())/targetHeapUsageInMB);
+ }
+
+ @Override
public void emulate() throws IOException, InterruptedException {
if (enabled) {
float currentProgress = progress.getProgress();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java Wed Nov 2 05:34:31 2011
@@ -135,6 +135,14 @@ public class TestResourceUsageEmulators
? fs.getFileStatus(testPath).getModificationTime()
: 0;
}
+
+ @Override
+ public float getProgress() {
+ try {
+ return fs.exists(touchPath) ? 1.0f : 0f;
+ } catch (IOException ioe) {}
+ return 0f;
+ }
}
/**
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1196451
/hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
/hadoop/core/trunk/src/contrib/index:776175-786373
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/BlockFixer.java Wed Nov 2 05:34:31 2011
@@ -782,7 +782,7 @@ public abstract class BlockFixer extends
DatanodeInfo[] nodes = new DatanodeInfo[]{datanode};
new Sender(out).writeBlock(block.getBlock(), block.getBlockToken(), "",
nodes, null, BlockConstructionStage.PIPELINE_SETUP_CREATE,
- 1, 0L, blockSize, 0L);
+ 1, 0L, blockSize, 0L, DataChecksum.newDataChecksum(metadataIn));
blockSender.sendBlock(out, baseStream);
LOG.info("Sent block " + block.getBlock() + " to " + datanode.name);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java Wed Nov 2 05:34:31 2011
@@ -73,8 +73,6 @@ import org.apache.hadoop.mapreduce.lib.o
* raid.blockfix.filespertask - number of corrupt files to fix in a single
* map reduce task (i.e., at one mapper node)
*
- * raid.blockfix.fairscheduler.pool - the pool to use for block fixer jobs
- *
* raid.blockfix.maxpendingfiles - maximum number of files to fix
* simultaneously
*/
@@ -91,12 +89,6 @@ public class DistBlockFixer extends Bloc
"raid.blockfix.filespertask";
private static final String BLOCKFIX_MAX_PENDING_FILES =
"raid.blockfix.maxpendingfiles";
- private static final String BLOCKFIX_POOL =
- "raid.blockfix.fairscheduler.pool";
- // mapred.fairscheduler.pool is only used in the local configuration
- // passed to a block fixing job
- private static final String MAPRED_POOL =
- "mapred.fairscheduler.pool";
// default number of files to fix in a task
private static final long DEFAULT_BLOCKFIX_FILES_PER_TASK = 10L;
@@ -115,9 +107,6 @@ public class DistBlockFixer extends Bloc
// number of files being fixed right now
private long pendingFiles;
- // pool name to use (may be null, in which case no special pool is used)
- private String poolName;
-
private long lastCheckTime;
private final SimpleDateFormat dateFormat =
@@ -137,7 +126,6 @@ public class DistBlockFixer extends Bloc
filesPerTask = DistBlockFixer.filesPerTask(getConf());
maxPendingFiles = DistBlockFixer.maxPendingFiles(getConf());
pendingFiles = 0L;
- poolName = conf.get(BLOCKFIX_POOL);
// start off due for the first iteration
lastCheckTime = System.currentTimeMillis() - blockFixInterval;
@@ -365,9 +353,6 @@ public class DistBlockFixer extends Bloc
List<Path> filesInJob = createInputFile(jobName, inDir, corruptFiles);
Configuration jobConf = new Configuration(getConf());
- if (poolName != null) {
- jobConf.set(MAPRED_POOL, poolName);
- }
Job job = new Job(jobConf, jobName);
job.setJarByClass(getClass());
job.setMapperClass(DistBlockFixerMapper.class);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java Wed Nov 2 05:34:31 2011
@@ -693,14 +693,4 @@ public class TestRaidNode extends TestCa
}
LOG.info("Test testSuspendTraversal completed.");
}
-
- public void testSchedulerOption() throws IOException {
- Configuration conf = new Configuration();
- conf.set("raid.scheduleroption",
- "mapred.fairscheduler.pool:dummy,foo:bar");
- org.apache.hadoop.mapreduce.Job job = DistRaid.createJob(conf);
- Configuration jobConf = job.getConfiguration();
- assertEquals("dummy", jobConf.get("mapred.fairscheduler.pool"));
- assertEquals("bar", jobConf.get("foo"));
- }
}
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/streaming:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/streaming:1159757-1196451
/hadoop/core/branches/branch-0.19/mapred/src/contrib/streaming:713112
/hadoop/core/trunk/src/contrib/streaming:776175-786373
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java Wed Nov 2 05:34:31 2011
@@ -718,7 +718,7 @@ public class StreamJob implements Tool {
}
// general MapRed job properties
- jobConf_ = new JobConf(config_);
+ jobConf_ = new JobConf(config_, StreamJob.class);
// All streaming jobs get the task timeout value
// from the configuration settings.
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1196451
/hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
/hadoop/core/trunk/src/contrib/vaidya:776175-786373
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml Wed Nov 2 05:34:31 2011
@@ -1232,49 +1232,28 @@
</li>
<li>
- <code>mapreduce.{map|reduce}.ulimit</code>: The slaves where
- tasks are run could be configured with a ulimit value that
- applies a limit to every process that is launched on the slave.
- If the task, or any child that the task launches (like in
- streaming), requires more than the configured limit, this option
- must be used. The value is given in kilobytes. For example, to
- increase the ulimit to 1G, the option should be set to 1048576.
- Note that this value is a per process limit. Since it applies
- to the JVM as well, the heap space given to the JVM through
- the <code>mapreduce.{map|reduce}.java.opts</code> should be less
- than the value configured for the ulimit. Otherwise the JVM
- will not start.
- </li>
-
- <li>
- <code>mapreduce.{map|reduce}.memory.mb</code>: In some
- environments, administrators might have configured a total limit
- on the virtual memory used by the entire process tree for a task,
- including all processes launched recursively by the task or
- its children, like in streaming. More details about this can be
- found in the section on
- <a href="ext:cluster-setup/ConfiguringMemoryParameters">
- Monitoring Task Memory Usage</a> in the Cluster SetUp guide.
- If a task requires more virtual memory for its entire tree,
- this option
- must be used. The value is given in MB. For example, to set
+ <code>mapreduce.{map|reduce}.memory.mb</code>:
+ This parameter configures how many megabytes of physical memory
+ the job requires for its map and reduce tasks. It must be configured
+ to be slightly larger than the configured Java heap size above,
+ to account for the fact that the JVM uses physical memory beyond
+ just the heap. An overhead of 20% to 30% is usually sufficient.
+ <br/>
+ Note that, the smaller the amount of memory specified for tasks
+ in this configuration, the larger the number of tasks that
+ can be run in parallel.<br/>
+
+ The value is given in MB. For example, to set
the limit to 1G, the option should be set to 1024. Note that this
value does not automatically influence the per process ulimit or
heap space. Hence, you may need to set those parameters as well
(as described above) in order to give your tasks the right amount
of memory.
+ <br/>
+ If the amount of physical memory used by your task exceeds the
+ configured value, the NodeManager will automatically kill the task.
</li>
- <li>
- <code>mapreduce.{map|reduce}.memory.physical.mb</code>:
- This parameter is similar to
- <code>mapreduce.{map|reduce}.memory.mb</code>, except it specifies
- how much physical memory is required by a task for its entire
- tree of processes. The parameter is applicable if administrators
- have configured a total limit on the physical memory used by
- all MapReduce tasks.
- </li>
-
</ul>
<p>
@@ -1297,7 +1276,7 @@
<p>
Note: The memory related configuration options described above
are used only for configuring the launched child tasks from the
- tasktracker. Configuring the memory options for daemons is documented
+ NodeManager. Configuring the memory options for daemons is documented
under
<a href="ext:cluster-setup/ConfiguringEnvironmentHadoopDaemons">
Configuring the Environment of the Hadoop Daemons</a> (Cluster Setup).
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/site.xml?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/docs/src/documentation/content/xdocs/site.xml Wed Nov 2 05:34:31 2011
@@ -43,15 +43,9 @@ See http://forrest.apache.org/docs/linki
<vaidya label="Vaidya" href="vaidya.html"/>
<archives label="Hadoop Archives" href="hadoop_archives.html"/>
<gridmix label="Gridmix" href="gridmix.html"/>
- <Rumen label="Rumen" href="rumen.html"/>
+ <Rumen label="Rumen" href="rumen.html"/>
</docs>
- <docs label="Schedulers">
- <cap_scheduler label="Capacity Scheduler" href="capacity_scheduler.html"/>
- <fair_scheduler label="Fair Scheduler" href="fair_scheduler.html"/>
- <dyn_scheduler label="Dynamic Priority Scheduler" href="dynamic_scheduler.html"/>
- </docs>
-
<docs label="Miscellaneous">
<api label="API Docs" href="ext:api/index" />
<jdiff label="API Changes" href="ext:jdiff/changes" />
@@ -75,8 +69,6 @@ See http://forrest.apache.org/docs/linki
<hdfs-default href="http://hadoop.apache.org/hdfs/docs/current/hdfs-default.html" />
<mapred-default href="http://hadoop.apache.org/mapreduce/docs/current/mapred-default.html" />
<mapred-queues href="http://hadoop.apache.org/mapreduce/docs/current/mapred-queues.xml" />
- <mapred-queues-capacity-scheduler href="http://hadoop.apache.org/mapreduce/docs/current/mapred-queues-capacity-scheduler.xml" />
- <capacity-scheduler-conf href="http://hadoop.apache.org/mapreduce/docs/current/capacity-scheduler-conf.html" />
<single-node-setup href="http://hadoop.apache.org/common/docs/current/single_node_setup.html">
<PreReqs href="#PreReqs" />
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1196451
/hadoop/core/branches/branch-0.19/mapred/src/examples:713112
/hadoop/core/trunk/src/examples:776175-784663
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/org/apache/hadoop/examples/terasort/TeraOutputFormat.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/org/apache/hadoop/examples/terasort/TeraOutputFormat.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/org/apache/hadoop/examples/terasort/TeraOutputFormat.java Wed Nov 2 05:34:31 2011
@@ -29,7 +29,6 @@ import org.apache.hadoop.mapreduce.JobCo
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
@@ -100,28 +99,9 @@ public class TeraOutputFormat extends Fi
throws IOException {
if (committer == null) {
Path output = getOutputPath(context);
- committer = new TeraOutputCommitter(output, context);
+ committer = new FileOutputCommitter(output, context);
}
return committer;
}
- public static class TeraOutputCommitter extends FileOutputCommitter {
-
- public TeraOutputCommitter(Path outputPath, TaskAttemptContext context)
- throws IOException {
- super(outputPath, context);
- }
-
- @Override
- public void commitJob(JobContext jobContext) {
- }
-
- @Override
- public void setupJob(JobContext jobContext) {
- }
-
- @Override
- public void setupTask(TaskAttemptContext taskContext) {
- }
- }
}
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1196451
/hadoop/core/branches/branch-0.19/mapred/src/java:713112
/hadoop/core/trunk/src/mapred:776175-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider Wed Nov 2 05:34:31 2011
@@ -12,4 +12,3 @@
# limitations under the License.
#
org.apache.hadoop.mapred.JobTrackerClientProtocolProvider
-org.apache.hadoop.mapred.LocalClientProtocolProvider
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java Wed Nov 2 05:34:31 2011
@@ -2676,7 +2676,7 @@ public class JobInProgress {
TaskAttemptStartedEvent tse = new TaskAttemptStartedEvent(
status.getTaskID(), taskType, status.getStartTime(),
- status.getTaskTracker(), ttStatus.getHttpPort());
+ status.getTaskTracker(), ttStatus.getHttpPort(), -1);
jobHistory.logEvent(tse, status.getTaskID().getJobID());
TaskAttemptID statusAttemptID = status.getTaskID();
@@ -2685,7 +2685,7 @@ public class JobInProgress {
MapAttemptFinishedEvent mfe = new MapAttemptFinishedEvent(
statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(),
status.getMapFinishTime(),
- status.getFinishTime(), trackerHostname,
+ status.getFinishTime(), trackerHostname, "",
status.getStateString(),
new org.apache.hadoop.mapreduce.Counters(status.getCounters()),
tip.getSplits(statusAttemptID).burst()
@@ -2698,7 +2698,7 @@ public class JobInProgress {
statusAttemptID, taskType, TaskStatus.State.SUCCEEDED.toString(),
status.getShuffleFinishTime(),
status.getSortFinishTime(), status.getFinishTime(),
- trackerHostname, status.getStateString(),
+ trackerHostname, "", status.getStateString(),
new org.apache.hadoop.mapreduce.Counters(status.getCounters()),
tip.getSplits(statusAttemptID).burst()
);
@@ -3197,7 +3197,7 @@ public class JobInProgress {
StringUtils.arrayToString(taskDiagnosticInfo.toArray(new String[0]));
TaskType taskType = getTaskType(tip);
TaskAttemptStartedEvent tse = new TaskAttemptStartedEvent(
- taskid, taskType, startTime, taskTrackerName, taskTrackerPort);
+ taskid, taskType, startTime, taskTrackerName, taskTrackerPort, -1);
jobHistory.logEvent(tse, taskid.getJobID());
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobTracker.java Wed Nov 2 05:34:31 2011
@@ -95,6 +95,7 @@ import org.apache.hadoop.mapreduce.serve
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
import org.apache.hadoop.mapreduce.util.ConfigUtil;
import org.apache.hadoop.mapreduce.util.MRAsyncDiskService;
+import org.apache.hadoop.mapreduce.v2.LogParams;
import org.apache.hadoop.net.DNSToSwitchMapping;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NetworkTopology;
@@ -4829,6 +4830,13 @@ public class JobTracker implements MRCon
return secretManager.renewToken(token, user);
}
+ @Override
+ public LogParams getLogFileParams(org.apache.hadoop.mapreduce.JobID jobID,
+ org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID)
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException("Not supported by JobTracker");
+ }
+
JobACLsManager getJobACLsManager() {
return aclsManager.getJobACLsManager();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapReducePolicyProvider.java Wed Nov 2 05:34:31 2011
@@ -19,6 +19,7 @@ package org.apache.hadoop.mapred;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.authorize.PolicyProvider;
@@ -40,14 +41,17 @@ public class MapReducePolicyProvider ext
ClientProtocol.class),
new Service("security.task.umbilical.protocol.acl",
TaskUmbilicalProtocol.class),
- new Service("security.refresh.policy.protocol.acl",
- RefreshAuthorizationPolicyProtocol.class),
- new Service("security.refresh.user.mappings.protocol.acl",
- RefreshUserMappingsProtocol.class),
+ new Service(
+ CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_POLICY,
+ RefreshAuthorizationPolicyProtocol.class),
+ new Service(
+ CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_USER_MAPPINGS,
+ RefreshUserMappingsProtocol.class),
new Service("security.admin.operations.protocol.acl",
AdminOperationsProtocol.class),
- new Service("security.get.user.mappings.protocol.acl",
- GetUserMappingsProtocol.class)
+ new Service(
+ CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_GET_USER_MAPPINGS,
+ GetUserMappingsProtocol.class)
};
@Override
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/
('svn:externals' removed)
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/log4j.properties?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/log4j.properties (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/log4j.properties Wed Nov 2 05:34:31 2011
@@ -16,4 +16,4 @@ log4j.rootLogger=info,stdout
log4j.threshhold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1196451
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
/hadoop/core/trunk/src/test/mapred:776175-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/cli/TestMRCLI.java Wed Nov 2 05:34:31 2011
@@ -30,6 +30,7 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.xml.sax.SAXException;
@@ -113,6 +114,7 @@ public class TestMRCLI extends TestHDFSC
}
@Test
+ @Ignore
@Override
public void testAll () {
super.testAll();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/conf/TestNoDefaultsJobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/conf/TestNoDefaultsJobConf.java?rev=1196458&r1=1196457&r2=1196458&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/conf/TestNoDefaultsJobConf.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/conf/TestNoDefaultsJobConf.java Wed Nov 2 05:34:31 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.conf;
import junit.framework.Assert;
import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileUtil;
@@ -59,7 +60,18 @@ public class TestNoDefaultsJobConf exten
JobConf conf = new JobConf(false);
//seeding JT and NN info into non-defaults (empty jobconf)
- conf.set(JTConfig.JT_IPC_ADDRESS, createJobConf().get(JTConfig.JT_IPC_ADDRESS));
+ String jobTrackerAddress = createJobConf().get(JTConfig.JT_IPC_ADDRESS);
+ if (jobTrackerAddress == null) {
+ jobTrackerAddress = "local";
+ }
+ conf.set(JTConfig.JT_IPC_ADDRESS, jobTrackerAddress);
+ if (jobTrackerAddress == "local") {
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
+ }
+ else {
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
+ }
+
conf.set("fs.default.name", createJobConf().get("fs.default.name"));
conf.setJobName("mr");
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 2 05:34:31 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1179483
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1196451
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:817878-835934