You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by tu...@apache.org on 2011/10/12 02:27:33 UTC
svn commit: r1182151 - in /hadoop/common/trunk/hadoop-mapreduce-project:
INSTALL assembly/all.xml hadoop-yarn/pom.xml pom.xml
Author: tucu
Date: Wed Oct 12 00:27:32 2011
New Revision: 1182151
URL: http://svn.apache.org/viewvc?rev=1182151&view=rev
Log:
HADOOP-7642. create hadoop-dist module where TAR stitching would happen. Contributed by Thomas White.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/INSTALL
hadoop/common/trunk/hadoop-mapreduce-project/assembly/all.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml
hadoop/common/trunk/hadoop-mapreduce-project/pom.xml
Modified: hadoop/common/trunk/hadoop-mapreduce-project/INSTALL
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/INSTALL?rev=1182151&r1=1182150&r2=1182151&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/INSTALL (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/INSTALL Wed Oct 12 00:27:32 2011
@@ -2,49 +2,31 @@ To compile Hadoop Mapreduce next follow
Step 1) Install dependencies for yarn
-See http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-mapreduce/hadoop-yarn/README
+See http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-mapreduce-porject/hadoop-yarn/README
Make sure protbuf library is in your library path or set: export LD_LIBRARY_PATH=/usr/local/lib
Step 2) Checkout
svn checkout http://svn.apache.org/repos/asf/hadoop/common/trunk
-Step 3) Build common
+Step 3) Build
-Go to common directory - choose your regular common build command
-Example: mvn clean install package -Pbintar -DskipTests
+Go to common directory - choose your regular common build command. For example:
-Step 4) Build HDFS
-
-Go to hdfs directory
-ant veryclean mvn-install -Dresolvers=internal
-
-Step 5) Build yarn and mapreduce
-
-Go to mapreduce directory
export MAVEN_OPTS=-Xmx512m
+mvn clean package -Pdist -Dtar -DskipTests -Pnative
-mvn clean install assembly:assembly -DskipTests
-
-Copy in build.properties if appropriate - make sure eclipse.home not set
-ant veryclean tar -Dresolvers=internal
+You can omit -Pnative it you don't want to build native packages.
-You will see a tarball in
-ls target/hadoop-mapreduce-0.24.0-SNAPSHOT-all.tar.gz
+Step 4) Untar the tarball from hadoop-dist/target/ into a clean and different
+directory, say YARN_HOME.
-Step 6) Untar the tarball in a clean and different directory.
-say YARN_HOME.
-
-Make sure you aren't picking up avro-1.3.2.jar, remove:
- $HADOOP_COMMON_HOME/share/hadoop/common/lib/avro-1.3.2.jar
- $YARN_HOME/lib/avro-1.3.2.jar
-
-Step 7)
-Install hdfs/common and start hdfs
+Step 5)
+Start hdfs
To run Hadoop Mapreduce next applications:
-Step 8) export the following variables to where you have things installed:
+Step 6) export the following variables to where you have things installed:
You probably want to export these in hadoop-env.sh and yarn-env.sh also.
export HADOOP_MAPRED_HOME=<mapred loc>
@@ -54,7 +36,7 @@ export YARN_HOME=directory where you unt
export HADOOP_CONF_DIR=<conf loc>
export YARN_CONF_DIR=$HADOOP_CONF_DIR
-Step 9) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager.
+Step 7) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager.
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce.shuffle</value>
@@ -65,31 +47,21 @@ Step 9) Setup config: for running mapred
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
-Step 10) Modify mapred-site.xml to use yarn framework
+Step 8) Modify mapred-site.xml to use yarn framework
<property>
<name> mapreduce.framework.name</name>
<value>yarn</value>
</property>
-Step 11) Create the following symlinks in $HADOOP_COMMON_HOME/share/hadoop/common/lib
-
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-app-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-yarn-api-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-common-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-yarn-common-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-core-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-yarn-server-common-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-0.24.0-SNAPSHOT.jar .
-
-Step 12) cd $YARN_HOME
+Step 9) cd $YARN_HOME
-Step 13) bin/yarn-daemon.sh start resourcemanager
+Step 10) bin/yarn-daemon.sh start resourcemanager
-Step 14) bin/yarn-daemon.sh start nodemanager
+Step 11) bin/yarn-daemon.sh start nodemanager
-Step 15) bin/yarn-daemon.sh start historyserver
+Step 12) bin/yarn-daemon.sh start historyserver
-Step 16) You are all set, an example on how to run a mapreduce job is:
+Step 13) You are all set, an example on how to run a mapreduce job is:
cd $HADOOP_MAPRED_HOME
ant examples -Dresolvers=internal
$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-0.24.0-SNAPSHOT.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-0.24.0-SNAPSHOT.jar output
Modified: hadoop/common/trunk/hadoop-mapreduce-project/assembly/all.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/assembly/all.xml?rev=1182151&r1=1182150&r2=1182151&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/assembly/all.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/assembly/all.xml Wed Oct 12 00:27:32 2011
@@ -1,101 +0,0 @@
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>all</id>
- <formats>
- <format>tar.gz</format>
- </formats>
- <includeBaseDirectory>true</includeBaseDirectory>
- <!-- TODO: this layout is wrong. We need module specific bin files in module specific dirs -->
- <fileSets>
- <fileSet>
- <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/c/container-executor</directory>
- <outputDirectory>bin</outputDirectory>
- <includes>
- <include>container-executor</include>
- </includes>
- <fileMode>0755</fileMode>
- </fileSet>
- <fileSet>
- <directory>hadoop-yarn/bin</directory>
- <outputDirectory>bin</outputDirectory>
- <includes>
- <include>*</include>
- </includes>
- <fileMode>0755</fileMode>
- </fileSet>
- <fileSet>
- <directory>bin</directory>
- <outputDirectory>bin</outputDirectory>
- <includes>
- <include>*</include>
- </includes>
- <fileMode>0755</fileMode>
- </fileSet>
- <fileSet>
- <directory>hadoop-yarn/conf</directory>
- <outputDirectory>conf</outputDirectory>
- <includes>
- <include>**/*</include>
- </includes>
- </fileSet>
- <fileSet>
- <outputDirectory>sources</outputDirectory>
- <excludes>
- <exclude>**/*.jar</exclude>
- <exclude>**/target/**</exclude>
- <!-- scripts to include later for setting fileMode -->
- <exclude>**/bin/*</exclude>
- <exclude>**/scripts/*</exclude>
- <!-- images that we don't need (and cause problems for our tools) -->
- <exclude>**/dt-*/images/**</exclude>
- <!-- until the code that does this is fixed -->
- <exclude>**/file:/**</exclude>
- <exclude>**/SecurityAuth.audit*</exclude>
- </excludes>
- <includes>
- <include>assembly/**</include>
- <include>pom.xml</include>
- <include>build*.xml</include>
- <include>ivy.xml</include>
- <include>ivy/**</include>
- <include>INSTALL</include>
- <include>LICENSE.txt</include>
- <include>mr-client/**</include>
- <include>hadoop-yarn/**</include>
- <include>src/**</include>
- </includes>
- </fileSet>
- <fileSet>
- <outputDirectory>sources</outputDirectory>
- <includes>
- <include>**/bin/*</include>
- <include>**/scripts/*</include>
- </includes>
- <fileMode>0755</fileMode>
- </fileSet>
- </fileSets>
- <moduleSets>
- <moduleSet>
- <excludes>
- <exclude>org.apache.hadoop:hadoop-yarn-server-tests</exclude>
- </excludes>
- <binaries>
- <outputDirectory>modules</outputDirectory>
- <includeDependencies>false</includeDependencies>
- <unpack>false</unpack>
- </binaries>
- </moduleSet>
- </moduleSets>
- <dependencySets>
- <dependencySet>
- <useProjectArtifact>false</useProjectArtifact>
- <outputDirectory>/lib</outputDirectory>
- <!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
- <excludes>
- <exclude>org.apache.hadoop:hadoop-common</exclude>
- <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
- </excludes>
- </dependencySet>
- </dependencySets>
-</assembly>
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml?rev=1182151&r1=1182150&r2=1182151&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml Wed Oct 12 00:27:32 2011
@@ -25,6 +25,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<test.logs>true</test.logs>
<test.timeout>600000</test.timeout>
+ <hadoop.assemblies.version>${project.version}</hadoop.assemblies.version>
<hadoop-common.version>0.24.0-SNAPSHOT</hadoop-common.version>
<hadoop-hdfs.version>0.24.0-SNAPSHOT</hadoop-hdfs.version>
<yarn.version>0.24.0-SNAPSHOT</yarn.version>
@@ -317,13 +318,33 @@
</pluginManagement>
<plugins>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${hadoop.assemblies.version}</version>
+ </dependency>
+ </dependencies>
<configuration>
<tarLongFileMode>gnu</tarLongFileMode>
- <descriptors>
- <descriptor>assembly/all.xml</descriptor>
- </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <descriptorRefs>
+ <descriptorRef>hadoop-mapreduce-dist</descriptorRef>
+ </descriptorRefs>
</configuration>
+ <executions>
+ <execution>
+ <id>tar</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
Modified: hadoop/common/trunk/hadoop-mapreduce-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/pom.xml?rev=1182151&r1=1182150&r2=1182151&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/pom.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/pom.xml Wed Oct 12 00:27:32 2011
@@ -26,6 +26,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<test.logs>true</test.logs>
<test.timeout>600000</test.timeout>
+ <hadoop.assemblies.version>${project.version}</hadoop.assemblies.version>
<hadoop-common.version>0.24.0-SNAPSHOT</hadoop-common.version>
<hadoop-hdfs.version>0.24.0-SNAPSHOT</hadoop-hdfs.version>
<hadoop-mapreduce.version>0.24.0-SNAPSHOT</hadoop-mapreduce.version>
@@ -320,13 +321,33 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${hadoop.assemblies.version}</version>
+ </dependency>
+ </dependencies>
<configuration>
<tarLongFileMode>gnu</tarLongFileMode>
- <descriptors>
- <descriptor>assembly/all.xml</descriptor>
- </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <descriptorRefs>
+ <descriptorRef>hadoop-mapreduce-dist</descriptorRef>
+ </descriptorRefs>
</configuration>
+ <executions>
+ <execution>
+ <id>tar</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
</plugin>
<plugin>
<groupId>com.atlassian.maven.plugins</groupId>