You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sh...@apache.org on 2009/09/22 00:33:12 UTC
svn commit: r817449 [1/8] - in /hadoop/hdfs/branches/HDFS-265: ./
.eclipse.templates/.launches/ lib/ src/contrib/block_forensics/
src/contrib/block_forensics/client/ src/contrib/block_forensics/ivy/
src/contrib/block_forensics/src/java/org/apache/hadoo...
Author: shv
Date: Mon Sep 21 22:33:09 2009
New Revision: 817449
URL: http://svn.apache.org/viewvc?rev=817449&view=rev
Log:
Revert merge to trunk 817119. Should be a merge to branch 0.21.
Added:
hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-0.21.0-dev.jar (with props)
hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-test-0.21.0-dev.jar (with props)
hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-0.21.0-dev.jar (with props)
hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-examples-0.21.0-dev.jar (with props)
hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-test-0.21.0-dev.jar (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/cluster_setup.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/commands_manual.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/distcp.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/fair_scheduler.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/hadoop_archives.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/hdfs_shell.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/hod_admin_guide.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/hod_config_guide.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/hod_user_guide.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/native_libraries.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/quickstart.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/service_level_auth.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/streaming.xml (with props)
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/vaidya.xml (with props)
Removed:
hadoop/hdfs/branches/HDFS-265/.eclipse.templates/.launches/AllTests.launch
hadoop/hdfs/branches/HDFS-265/.eclipse.templates/.launches/DataNode.launch
hadoop/hdfs/branches/HDFS-265/.eclipse.templates/.launches/NameNode.launch
hadoop/hdfs/branches/HDFS-265/.eclipse.templates/.launches/SpecificTestTemplate.launch
hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-0.22.0-dev.jar
hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-test-0.22.0-dev.jar
hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-0.22.0-dev.jar
hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-examples-0.22.0-dev.jar
hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-test-0.22.0-dev.jar
hadoop/hdfs/branches/HDFS-265/src/contrib/block_forensics/README
hadoop/hdfs/branches/HDFS-265/src/contrib/block_forensics/build.xml
hadoop/hdfs/branches/HDFS-265/src/contrib/block_forensics/client/BlockForensics.java
hadoop/hdfs/branches/HDFS-265/src/contrib/block_forensics/ivy.xml
hadoop/hdfs/branches/HDFS-265/src/contrib/block_forensics/ivy/libraries.properties
hadoop/hdfs/branches/HDFS-265/src/contrib/block_forensics/src/java/org/apache/hadoop/block_forensics/BlockSearch.java
hadoop/hdfs/branches/HDFS-265/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
Modified:
hadoop/hdfs/branches/HDFS-265/.gitignore
hadoop/hdfs/branches/HDFS-265/CHANGES.txt
hadoop/hdfs/branches/HDFS-265/build.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/faultinject_framework.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/hdfs_design.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/hdfs_permissions_guide.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/hdfs_quota_admin_guide.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/index.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/libhdfs.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/site.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/tabs.xml
hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/skinconf.xml
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicy.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockPlacementPolicyDefault.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Modified: hadoop/hdfs/branches/HDFS-265/.gitignore
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/.gitignore?rev=817449&r1=817448&r2=817449&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/.gitignore (original)
+++ hadoop/hdfs/branches/HDFS-265/.gitignore Mon Sep 21 22:33:09 2009
@@ -1,35 +1,34 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*~
-.classpath
-.project
-.launches/
-.settings
-.svn
-build/
-build-fi/
-conf/masters
-conf/slaves
-conf/hadoop-env.sh
-conf/hadoop-site.xml
-conf/hdfs-site.xml
-conf/hadoop-policy.xml
-docs/api/
-logs/
-src/docs/build
-src/docs/cn/build
-src/docs/cn/src/documentation/sitemap.xmap
-src/docs/cn/uming.conf
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*~
+.classpath
+.project
+.settings
+.svn
+build/
+build-fi/
+conf/masters
+conf/slaves
+conf/hadoop-env.sh
+conf/hadoop-site.xml
+conf/hdfs-site.xml
+conf/hadoop-policy.xml
+docs/api/
+logs/
+src/docs/build
+src/docs/cn/build
+src/docs/cn/src/documentation/sitemap.xmap
+src/docs/cn/uming.conf
Modified: hadoop/hdfs/branches/HDFS-265/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/CHANGES.txt?rev=817449&r1=817448&r2=817449&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-265/CHANGES.txt Mon Sep 21 22:33:09 2009
@@ -63,22 +63,6 @@
INCOMPATIBLE CHANGES
- NEW FEATURES
-
- IMPROVEMENTS
-
- OPTIMIZATIONS
-
- BUG FIXES
-
- HDFS-629. Remove ReplicationTargetChooser.java along with fixing
- import warnings generated by Eclipse. (dhruba)
-
-
-Release 0.21.0 - Unreleased
-
- INCOMPATIBLE CHANGES
-
HDFS-538. Per the contract elucidated in HADOOP-6201, throw
FileNotFoundException from FileSystem::listStatus rather than returning
null. (Jakob Homan via cdouglas)
@@ -115,15 +99,6 @@
HDFS-385. Add support for an experimental API that allows a module external
to HDFS to specify how HDFS blocks should be placed. (dhruba)
- HADOOP-4952. Update hadoop-core and test jars to propagate new FileContext
- file system application interface. (Sanjay Radia via suresh).
-
- HDFS-567. Add block forensics contrib tool to print history of corrupt and
- missing blocks from the HDFS logs.
- (Bill Zeller, Jithendra Pandey via suresh).
-
- HDFS-610. Support o.a.h.fs.FileContext. (Sanjay Radia via szetszwo)
-
IMPROVEMENTS
HDFS-381. Remove blocks from DataNode maps when corresponding file
@@ -243,11 +218,6 @@
HDFS-618. Support non-recursive mkdir(). (Kan Zhang via szetszwo)
- HDFS-574. Split the documentation between the subprojects.
- (Corinne Chandel via omalley)
-
- HDFS-598. Eclipse launch task for HDFS. (Eli Collins via tomwhite)
-
BUG FIXES
HDFS-76. Better error message to users when commands fail because of
@@ -338,7 +308,7 @@
HDFS-622. checkMinReplication should count live nodes only. (shv)
-Release 0.20.1 - 2009-09-01
+Release 0.20.1 - Unreleased
IMPROVEMENTS
Modified: hadoop/hdfs/branches/HDFS-265/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/build.xml?rev=817449&r1=817448&r2=817449&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/build.xml (original)
+++ hadoop/hdfs/branches/HDFS-265/build.xml Mon Sep 21 22:33:09 2009
@@ -27,9 +27,9 @@
<property name="Name" value="Hadoop-Hdfs"/>
<property name="name" value="hadoop-hdfs"/>
- <property name="version" value="0.22.0-dev"/>
- <property name="hadoop-core.version" value="0.22.0-dev"/>
- <property name="hadoop-mr.version" value="0.22.0-dev"/>
+ <property name="version" value="0.21.0-dev"/>
+ <property name="hadoop-core.version" value="0.21.0-dev"/>
+ <property name="hadoop-mr.version" value="0.21.0-dev"/>
<property name="final.name" value="${name}-${version}"/>
<property name="test.hdfs.final.name" value="${name}-test-${version}"/>
<property name="test.hdfswithmr.final.name" value="${name}-hdfswithmr-test-${version}"/>
@@ -43,6 +43,7 @@
<property name="conf.dir" value="${basedir}/conf"/>
<property name="contrib.dir" value="${basedir}/src/contrib"/>
<property name="docs.src" value="${basedir}/src/docs"/>
+ <property name="src.docs.cn" value="${basedir}/src/docs/cn"/>
<property name="changes.src" value="${docs.src}/changes"/>
<property name="build.dir" value="${basedir}/build"/>
@@ -62,6 +63,7 @@
value="${sun.arch.data.model}"/>
<property name="build.docs" value="${build.dir}/docs"/>
+ <property name="build.docs.cn" value="${build.dir}/docs/cn"/>
<property name="build.javadoc" value="${build.docs}/api"/>
<property name="build.javadoc.timestamp" value="${build.javadoc}/index.html" />
<property name="build.javadoc.dev" value="${build.docs}/dev-api"/>
@@ -819,6 +821,22 @@
<style basedir="${hdfs.src.dir}" destdir="${build.docs}"
includes="hdfs-default.xml" style="conf/configuration.xsl"/>
<antcall target="changes-to-html"/>
+ <antcall target="cn-docs"/>
+ </target>
+
+ <target name="cn-docs" depends="forrest.check, init"
+ description="Generate forrest-based Chinese documentation. To use, specify -Dforrest.home=<base of Apache Forrest installation> on the command line."
+ if="forrest.home">
+ <exec dir="${src.docs.cn}" executable="${forrest.home}/bin/forrest" failonerror="true">
+ <env key="LANG" value="en_US.utf8"/>
+ <env key="JAVA_HOME" value="${java5.home}"/>
+ </exec>
+ <copy todir="${build.docs.cn}">
+ <fileset dir="${src.docs.cn}/build/site/" />
+ </copy>
+ <style basedir="${hdfs.src.dir}" destdir="${build.docs.cn}"
+ includes="hdfs-default.xml" style="conf/configuration.xsl"/>
+ <antcall target="changes-to-html"/>
</target>
<target name="forrest.check" unless="forrest.home" depends="java5.check">
@@ -1132,6 +1150,7 @@
<delete dir="${build.dir}"/>
<delete dir="${build-fi.dir}"/>
<delete dir="${docs.src}/build"/>
+ <delete dir="${src.docs.cn}/build"/>
</target>
<!-- ================================================================== -->
Added: hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-0.21.0-dev.jar
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-0.21.0-dev.jar?rev=817449&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-0.21.0-dev.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-test-0.21.0-dev.jar
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-test-0.21.0-dev.jar?rev=817449&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/hdfs/branches/HDFS-265/lib/hadoop-core-test-0.21.0-dev.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-0.21.0-dev.jar
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-0.21.0-dev.jar?rev=817449&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-0.21.0-dev.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-examples-0.21.0-dev.jar
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-examples-0.21.0-dev.jar?rev=817449&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-examples-0.21.0-dev.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-test-0.21.0-dev.jar
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-test-0.21.0-dev.jar?rev=817449&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/hdfs/branches/HDFS-265/lib/hadoop-mapred-test-0.21.0-dev.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml?rev=817449&r1=817448&r2=817449&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml (original)
+++ hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/SLG_user_guide.xml Mon Sep 21 22:33:09 2009
@@ -18,12 +18,12 @@
<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
<document>
<header>
- <title>Synthetic Load Generator Guide </title>
+ <title> HDFS Synthetic Load Generator Guide </title>
</header>
<body>
- <section>
- <title>Overview</title>
- <p>
+ <section>
+ <title> Description </title>
+ <p>
The synthetic load generator (SLG) is a tool for testing NameNode behavior
under different client loads. The user can generate different mixes
of read, write, and list requests by specifying the probabilities of
@@ -33,121 +33,91 @@
monitor the running of the NameNode. When a load generator exits, it
prints some NameNode statistics like the average execution time of each
kind of operation and the NameNode throughput.
- </p>
- </section>
-
- <section>
- <title> Synopsis </title>
- <p>
- The synopsis of the command is:
- </p>
- <source>java LoadGenerator [options]</source>
- <p> Options include:</p>
-
- <ul>
- <li>
- <code>-readProbability <read probability></code><br/>
- The probability of the read operation; default is 0.3333.
- </li>
-
- <li>
- <code>-writeProbability <write probability></code><br/>
- The probability of the write operations; default is 0.3333.
- </li>
-
- <li>
- <code>-root <test space root></code><br/>
- The root of the test space; default is /testLoadSpace.
- </li>
-
- <li>
- <code>-maxDelayBetweenOps <maxDelayBetweenOpsInMillis></code><br/>
- The maximum delay between two consecutive operations in a thread; default is 0 indicating no delay.
- </li>
-
- <li>
- <code>-numOfThreads <numOfThreads></code><br/>
- The number of threads to spawn; default is 200.
- </li>
-
- <li>
- <code>-elapsedTime <elapsedTimeInSecs></code><br/>
- The number of seconds that the program
- will run; A value of zero indicates that the program runs
- forever. The default value is 0.
- </li>
-
- <li>
- <code>-startTime <startTimeInMillis></code><br/>
- The time that all worker threads
+ </p>
+ </section>
+ <section>
+ <title> Synopsis </title>
+ <p>
+ <code>java LoadGenerator [options]</code><br/>
+ </p>
+ <p>
+ Options include:<br/>
+ <code> -readProbability <read probability></code><br/>
+ <code> the probability of the read operation;
+ default is 0.3333. </code><br/>
+ <code> -writeProbability <write probability></code><br/>
+ <code> the probability of the write
+ operations; default is 0.3333.</code><br/>
+ <code> -root <test space root></code><br/>
+ <code> the root of the test space;
+ default is /testLoadSpace.</code><br/>
+ <code> -maxDelayBetweenOps
+ <maxDelayBetweenOpsInMillis></code><br/>
+ <code> the maximum delay between two consecutive
+ operations in a thread; default is 0 indicating no delay.
+ </code><br/>
+ <code> -numOfThreads <numOfThreads></code><br/>
+ <code> the number of threads to spawn;
+ default is 200.</code><br/>
+ <code> -elapsedTime <elapsedTimeInSecs></code><br/>
+ <code> the number of seconds that the program
+ will run; A value of zero indicates that the program runs
+ forever. The default value is 0.</code><br/>
+ <code> -startTime <startTimeInMillis></code><br/>
+ <code> the time that all worker threads
start to run. By default it is 10 seconds after the main
program starts running.This creates a barrier if more than
one load generator is running.
- </li>
-
- <li>
- <code>-seed <seed></code><br/>
- The random generator seed for repeating
+ </code><br/>
+ <code> -seed <seed></code><br/>
+ <code> the random generator seed for repeating
requests to NameNode when running with a single thread;
- default is the current time.
- </li>
-
- </ul>
-
- <p>
+ default is the current time.</code><br/>
+ </p>
+ <p>
After command line argument parsing, the load generator traverses
the test space and builds a table of all directories and another table
of all files in the test space. It then waits until the start time to
- spawn the number of worker threads as specified by the user.
-
- Each thread sends a stream of requests to NameNode. At each iteration,
+ spawn the number of worker threads as specified by the user. Each
+ thread sends a stream of requests to NameNode. At each iteration,
it first decides if it is going to read a file, create a file, or
list a directory following the read and write probabilities specified
by the user. The listing probability is equal to
<em>1-read probability-write probability</em>. When reading,
it randomly picks a file in the test space and reads the entire file.
When writing, it randomly picks a directory in the test space and
- creates a file there.
- </p>
- <p>
- To avoid two threads with the same load
- generator or from two different load generators creating the same
+ creates a file there. To avoid two threads with the same load
+ generator or from two different load generators create the same
file, the file name consists of the current machine's host name
and the thread id. The length of the file follows Gaussian
distribution with an average size of 2 blocks and the standard
- deviation of 1. The new file is filled with byte 'a'. To avoid the test
- space growing indefinitely, the file is deleted immediately
- after the file creation completes. While listing, it randomly picks
- a directory in the test space and lists its content.
- </p>
- <p>
+ deviation of 1. The new file is filled with byte 'a'. To avoid
+ the test space to grow indefinitely, the file is deleted immediately
+ after the file creation completes. While listing, it randomly
+ picks a directory in the test space and lists its content.
After an operation completes, the thread pauses for a random
amount of time in the range of [0, maxDelayBetweenOps] if the
specified maximum delay is not zero. All threads are stopped when
the specified elapsed time is passed. Before exiting, the program
prints the average execution for each kind of NameNode operations,
and the number of requests served by the NameNode per second.
- </p>
-
- </section>
-
- <section>
- <title> Test Space Population </title>
- <p>
- The user needs to populate a test space before running a
+ </p>
+ </section>
+ <section>
+ <title> Test Space Population </title>
+ <p>
+ The user needs to populate a test space before she runs a
load generator. The structure generator generates a random
test space structure and the data generator creates the files
and directories of the test space in Hadoop distributed file system.
- </p>
-
- <section>
- <title> Structure Generator </title>
- <p>
+ </p>
+ <section>
+ <title> Structure Generator </title>
+ <p>
This tool generates a random namespace structure with the
following constraints:
- </p>
-
- <ol>
+ </p>
+ <ol>
<li>The number of subdirectories that a directory can have is
a random number in [minWidth, maxWidth].</li>
<li>The maximum depth of each subdirectory is a random number
@@ -155,83 +125,69 @@
<li>Files are randomly placed in leaf directories. The size of
each file follows Gaussian distribution with an average size
of 1 block and a standard deviation of 1.</li>
- </ol>
- <p>
+ </ol>
+ <p>
The generated namespace structure is described by two files in
the output directory. Each line of the first file contains the
full name of a leaf directory. Each line of the second file
contains the full name of a file and its size, separated by a blank.
- </p>
- <p>
- The synopsis of the command is:
- </p>
- <source>java StructureGenerator [options]</source>
-
- <p>Options include:</p>
- <ul>
- <li>
- <code>-maxDepth <maxDepth></code><br/>
- Maximum depth of the directory tree; default is 5.
- </li>
-
- <li>
- <code>-minWidth <minWidth></code><br/>
- Minimum number of subdirectories per directories; default is 1.
- </li>
-
- <li>
- <code>-maxWidth <maxWidth></code><br/>
- Maximum number of subdirectories per directories; default is 5.
- </li>
-
- <li>
- <code>-numOfFiles <#OfFiles></code><br/>
- The total number of files in the test space; default is 10.
- </li>
-
- <li>
- <code>-avgFileSize <avgFileSizeInBlocks></code><br/>
- Average size of blocks; default is 1.
- </li>
-
- <li>
- <code>-outDir <outDir></code><br/>
- Output directory; default is the current directory.
- </li>
-
- <li>
- <code>-seed <seed></code><br/>
- Random number generator seed; default is the current time.
- </li>
- </ul>
- </section>
-
- <section>
- <title>Data Generator </title>
- <p>
+ </p>
+ <p>
+ The synopsis of the command is
+ </p>
+ <p>
+ <code>java StructureGenerator [options]</code>
+ </p>
+ <p>
+ Options include:<br/>
+ <code> -maxDepth <maxDepth></code><br/>
+ <code> maximum depth of the directory tree;
+ default is 5.</code><br/>
+ <code> -minWidth <minWidth></code><br/>
+ <code> minimum number of subdirectories per
+ directories; default is 1.</code><br/>
+ <code> -maxWidth <maxWidth></code><br/>
+ <code> maximum number of subdirectories per
+ directories; default is 5.</code><br/>
+ <code> -numOfFiles <#OfFiles></code><br/>
+ <code> the total number of files in the test
+ space; default is 10.</code><br/>
+ <code> -avgFileSize <avgFileSizeInBlocks></code><br/>
+ <code> average size of blocks; default is 1.
+ </code><br/>
+ <code> -outDir <outDir></code><br/>
+ <code> output directory; default is the
+ current directory. </code><br/>
+ <code> -seed <seed></code><br/>
+ <code> random number generator seed;
+ default is the current time.</code><br/>
+ </p>
+ </section>
+ <section>
+ <title> Test Space Generator </title>
+ <p>
This tool reads the directory structure and file structure from
the input directory and creates the namespace in Hadoop distributed
file system. All files are filled with byte 'a'.
- </p>
- <p>
- The synopsis of the command is:
- </p>
- <source>java DataGenerator [options]</source>
- <p>Options include:</p>
- <ul>
- <li>
- <code>-inDir <inDir></code><br/>
- Input directory name where directory/file
- structures are stored; default is the current directory.
- </li>
- <li>
- <code>-root <test space root></code><br/>
- The name of the root directory which the
- new namespace is going to be placed under;
- default is "/testLoadSpace".
- </li>
- </ul>
- </section>
- </section>
+ </p>
+ <p>
+ The synopsis of the command is
+ </p>
+ <p>
+ <code>java DataGenerator [options]</code>
+ </p>
+ <p>
+ Options include:<br/>
+ <code> -inDir <inDir></code><br/>
+ <code> input directory name where directory/file
+ structures are stored; default is the current directory.
+ </code><br/>
+ <code> -root <test space root></code><br/>
+ <code> the name of the root directory which the
+ new namespace is going to be placed under;
+ default is "/testLoadSpace".</code><br/>
+ </p>
+ </section>
+ </section>
</body>
</document>
Added: hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml?rev=817449&view=auto
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml (added)
+++ hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml Mon Sep 21 22:33:09 2009
@@ -0,0 +1,356 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+
+ <header>
+ <title>Capacity Scheduler Guide</title>
+ </header>
+
+ <body>
+
+ <section>
+ <title>Purpose</title>
+
+ <p>This document describes the Capacity Scheduler, a pluggable
+ Map/Reduce scheduler for Hadoop which provides a way to share
+ large clusters.</p>
+ </section>
+
+ <section>
+ <title>Features</title>
+
+ <p>The Capacity Scheduler supports the following features:</p>
+ <ul>
+ <li>
+ Support for multiple queues, where a job is submitted to a queue.
+ </li>
+ <li>
+ Queues are allocated a fraction of the capacity of the grid in the
+ sense that a certain capacity of resources will be at their
+ disposal. All jobs submitted to a queue will have access to the
+ capacity allocated to the queue.
+ </li>
+ <li>
+ Free resources can be allocated to any queue beyond it's capacity.
+ When there is demand for these resources from queues running below
+ capacity at a future point in time, as tasks scheduled on these
+ resources complete, they will be assigned to jobs on queues
+ running below the capacity.
+ </li>
+ <li>
+ Queues optionally support job priorities (disabled by default).
+ </li>
+ <li>
+ Within a queue, jobs with higher priority will have access to the
+ queue's resources before jobs with lower priority. However, once a
+ job is running, it will not be preempted for a higher priority job,
+ though new tasks from the higher priority job will be
+ preferentially scheduled.
+ </li>
+ <li>
+ In order to prevent one or more users from monopolizing its
+ resources, each queue enforces a limit on the percentage of
+ resources allocated to a user at any given time, if there is
+ competition for them.
+ </li>
+ <li>
+ Support for memory-intensive jobs, wherein a job can optionally
+ specify higher memory-requirements than the default, and the tasks
+ of the job will only be run on TaskTrackers that have enough memory
+ to spare.
+ </li>
+ </ul>
+ </section>
+
+ <section>
+ <title>Picking a task to run</title>
+
+ <p>Note that many of these steps can be, and will be, enhanced over time
+ to provide better algorithms.</p>
+
+ <p>Whenever a TaskTracker is free, the Capacity Scheduler picks
+ a queue which has most free space (whose ratio of # of running slots to
+ capacity is the lowest).</p>
+
+ <p>Once a queue is selected, the Scheduler picks a job in the queue. Jobs
+ are sorted based on when they're submitted and their priorities (if the
+ queue supports priorities). Jobs are considered in order, and a job is
+ selected if its user is within the user-quota for the queue, i.e., the
+ user is not already using queue resources above his/her limit. The
+ Scheduler also makes sure that there is enough free memory in the
+ TaskTracker to tun the job's task, in case the job has special memory
+ requirements.</p>
+
+ <p>Once a job is selected, the Scheduler picks a task to run. This logic
+ to pick a task remains unchanged from earlier versions.</p>
+
+ </section>
+
+ <section>
+ <title>Installation</title>
+
+ <p>The Capacity Scheduler is available as a JAR file in the Hadoop
+ tarball under the <em>contrib/capacity-scheduler</em> directory. The name of
+ the JAR file would be on the lines of hadoop-*-capacity-scheduler.jar.</p>
+ <p>You can also build the Scheduler from source by executing
+ <em>ant package</em>, in which case it would be available under
+ <em>build/contrib/capacity-scheduler</em>.</p>
+ <p>To run the Capacity Scheduler in your Hadoop installation, you need
+ to put it on the <em>CLASSPATH</em>. The easiest way is to copy the
+ <code>hadoop-*-capacity-scheduler.jar</code> from
+ to <code>HADOOP_HOME/lib</code>. Alternatively, you can modify
+ <em>HADOOP_CLASSPATH</em> to include this jar, in
+ <code>conf/hadoop-env.sh</code>.</p>
+ </section>
+
+ <section>
+ <title>Configuration</title>
+
+ <section>
+ <title>Using the Capacity Scheduler</title>
+ <p>
+ To make the Hadoop framework use the Capacity Scheduler, set up
+ the following property in the site configuration:</p>
+ <table>
+ <tr>
+ <td>Property</td>
+ <td>Value</td>
+ </tr>
+ <tr>
+ <td>mapred.jobtracker.taskScheduler</td>
+ <td>org.apache.hadoop.mapred.CapacityTaskScheduler</td>
+ </tr>
+ </table>
+ </section>
+
+ <section>
+ <title>Setting up queues</title>
+ <p>
+ You can define multiple queues to which users can submit jobs with
+ the Capacity Scheduler. To define multiple queues, you should edit
+ the site configuration for Hadoop and modify the
+ <em>mapred.queue.names</em> property.
+ </p>
+ <p>
+ You can also configure ACLs for controlling which users or groups
+ have access to the queues.
+ </p>
+ <p>
+ For more details, refer to
+ <a href="cluster_setup.html#Configuring+the+Hadoop+Daemons">Cluster
+ Setup</a> documentation.
+ </p>
+ </section>
+
+ <section>
+ <title>Configuring properties for queues</title>
+
+ <p>The Capacity Scheduler can be configured with several properties
+ for each queue that control the behavior of the Scheduler. This
+ configuration is in the <em>conf/capacity-scheduler.xml</em>. By
+ default, the configuration is set up for one queue, named
+ <em>default</em>.</p>
+ <p>To specify a property for a queue that is defined in the site
+ configuration, you should use the property name as
+ <em>mapred.capacity-scheduler.queue.<queue-name>.<property-name></em>.
+ </p>
+ <p>For example, to define the property <em>capacity</em>
+ for queue named <em>research</em>, you should specify the property
+ name as
+ <em>mapred.capacity-scheduler.queue.research.capacity</em>.
+ </p>
+
+ <p>The properties defined for queues and their descriptions are
+ listed in the table below:</p>
+
+ <table>
+ <tr><th>Name</th><th>Description</th></tr>
+ <tr><td>mapred.capacity-scheduler.queue.<queue-name>.capacity</td>
+ <td>Percentage of the number of slots in the cluster that are made
+ to be available for jobs in this queue. The sum of capacities
+ for all queues should be less than or equal 100.</td>
+ </tr>
+ <tr><td>mapred.capacity-scheduler.queue.<queue-name>.supports-priority</td>
+ <td>If true, priorities of jobs will be taken into account in scheduling
+ decisions.</td>
+ </tr>
+ <tr><td>mapred.capacity-scheduler.queue.<queue-name>.minimum-user-limit-percent</td>
+ <td>Each queue enforces a limit on the percentage of resources
+ allocated to a user at any given time, if there is competition
+ for them. This user limit can vary between a minimum and maximum
+ value. The former depends on the number of users who have submitted
+ jobs, and the latter is set to this property value. For example,
+ suppose the value of this property is 25. If two users have
+ submitted jobs to a queue, no single user can use more than 50%
+ of the queue resources. If a third user submits a job, no single
+ user can use more than 33% of the queue resources. With 4 or more
+ users, no user can use more than 25% of the queue's resources. A
+ value of 100 implies no user limits are imposed.</td>
+ </tr>
+ </table>
+ </section>
+
+ <section>
+ <title>Memory management</title>
+
+ <p>The Capacity Scheduler supports scheduling of tasks on a
+ <code>TaskTracker</code>(TT) based on a job's memory requirements
+ and the availability of RAM and Virtual Memory (VMEM) on the TT node.
+ See the <a href="mapred_tutorial.html#Memory+monitoring">Hadoop
+ Map/Reduce tutorial</a> for details on how the TT monitors
+ memory usage.</p>
+ <p>Currently the memory based scheduling is only supported
+ in Linux platform.</p>
+ <p>Memory-based scheduling works as follows:</p>
+ <ol>
+ <li>The absence of any one or more of three config parameters
+ or -1 being set as value of any of the parameters,
+ <code>mapred.tasktracker.vmem.reserved</code>,
+ <code>mapred.task.default.maxvmem</code>, or
+ <code>mapred.task.limit.maxvmem</code>, disables memory-based
+ scheduling, just as it disables memory monitoring for a TT. These
+ config parameters are described in the
+ <a href="mapred_tutorial.html#Memory+monitoring">Hadoop Map/Reduce
+ tutorial</a>. The value of
+ <code>mapred.tasktracker.vmem.reserved</code> is
+ obtained from the TT via its heartbeat.
+ </li>
+ <li>If all the three mandatory parameters are set, the Scheduler
+ enables VMEM-based scheduling. First, the Scheduler computes the free
+ VMEM on the TT. This is the difference between the available VMEM on the
+ TT (the node's total VMEM minus the offset, both of which are sent by
+ the TT on each heartbeat)and the sum of VMs already allocated to
+ running tasks (i.e., sum of the VMEM task-limits). Next, the Scheduler
+ looks at the VMEM requirements for the job that's first in line to
+ run. If the job's VMEM requirements are less than the available VMEM on
+ the node, the job's task can be scheduled. If not, the Scheduler
+ ensures that the TT does not get a task to run (provided the job
+ has tasks to run). This way, the Scheduler ensures that jobs with
+ high memory requirements are not starved, as eventually, the TT
+ will have enough VMEM available. If the high-mem job does not have
+ any task to run, the Scheduler moves on to the next job.
+ </li>
+ <li>In addition to VMEM, the Capacity Scheduler can also consider
+ RAM on the TT node. RAM is considered the same way as VMEM. TTs report
+ the total RAM available on their node, and an offset. If both are
+ set, the Scheduler computes the available RAM on the node. Next,
+ the Scheduler figures out the RAM requirements of the job, if any.
+ As with VMEM, users can optionally specify a RAM limit for their job
+ (<code>mapred.task.maxpmem</code>, described in the Map/Reduce
+ tutorial). The Scheduler also maintains a limit for this value
+ (<code>mapred.capacity-scheduler.task.default-pmem-percentage-in-vmem</code>,
+ described below). All these three values must be set for the
+ Scheduler to schedule tasks based on RAM constraints.
+ </li>
+ <li>The Scheduler ensures that jobs cannot ask for RAM or VMEM higher
+ than configured limits. If this happens, the job is failed when it
+ is submitted.
+ </li>
+ </ol>
+
+ <p>As described above, the additional scheduler-based config
+ parameters are as follows:</p>
+
+ <table>
+ <tr><th>Name</th><th>Description</th></tr>
+ <tr><td>mapred.capacity-scheduler.task.default-pmem-percentage-in-vmem</td>
+ <td>A percentage of the default VMEM limit for jobs
+ (<code>mapred.task.default.maxvmem</code>). This is the default
+ RAM task-limit associated with a task. Unless overridden by a
+ job's setting, this number defines the RAM task-limit.</td>
+ </tr>
+ <tr><td>mapred.capacity-scheduler.task.limit.maxpmem</td>
+ <td>Configuration which provides an upper limit to maximum physical
+ memory which can be specified by a job. If a job requires more
+ physical memory than what is specified in this limit then the same
+ is rejected.</td>
+ </tr>
+ </table>
+ </section>
+ <section>
+ <title>Job Initialization Parameters</title>
+ <p>Capacity scheduler lazily initializes the jobs before they are
+ scheduled, for reducing the memory footprint on jobtracker.
+ Following are the parameters, by which you can control the laziness
+ of the job initialization. The following parameters can be
+ configured in capacity-scheduler.xml
+ </p>
+
+ <table>
+ <tr><th>Name</th><th>Description</th></tr>
+ <tr>
+ <td>
+ mapred.capacity-scheduler.queue.<queue-name>.maximum-initialized-jobs-per-user
+ </td>
+ <td>
+ Maximum number of jobs which are allowed to be pre-initialized for
+ a particular user in the queue. Once a job is scheduled, i.e.
+ it starts running, then that job is not considered
+ while scheduler computes the maximum job a user is allowed to
+ initialize.
+ </td>
+ </tr>
+ <tr>
+ <td>
+ mapred.capacity-scheduler.init-poll-interval
+ </td>
+ <td>
+ Amount of time in miliseconds which is used to poll the scheduler
+ job queue to look for jobs to be initialized.
+ </td>
+ </tr>
+ <tr>
+ <td>
+ mapred.capacity-scheduler.init-worker-threads
+ </td>
+ <td>
+ Number of worker threads which would be used by Initialization
+ poller to initialize jobs in a set of queue. If number mentioned
+ in property is equal to number of job queues then a thread is
+ assigned jobs from one queue. If the number configured is lesser than
+ number of queues, then a thread can get jobs from more than one queue
+ which it initializes in a round robin fashion. If the number configured
+ is greater than number of queues, then number of threads spawned
+ would be equal to number of job queues.
+ </td>
+ </tr>
+ </table>
+ </section>
+ <section>
+ <title>Reviewing the configuration of the Capacity Scheduler</title>
+ <p>
+ Once the installation and configuration is completed, you can review
+ it after starting the Map/Reduce cluster from the admin UI.
+ </p>
+ <ul>
+ <li>Start the Map/Reduce cluster as usual.</li>
+ <li>Open the JobTracker web UI.</li>
+ <li>The queues you have configured should be listed under the <em>Scheduling
+ Information</em> section of the page.</li>
+ <li>The properties for the queues should be visible in the <em>Scheduling
+ Information</em> column against each queue.</li>
+ </ul>
+ </section>
+
+ </section>
+ </body>
+
+</document>
Propchange: hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/cluster_setup.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/cluster_setup.xml?rev=817449&view=auto
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/cluster_setup.xml (added)
+++ hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/cluster_setup.xml Mon Sep 21 22:33:09 2009
@@ -0,0 +1,879 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+
+ <header>
+ <title>Cluster Setup</title>
+ </header>
+
+ <body>
+
+ <section>
+ <title>Purpose</title>
+
+ <p>This document describes how to install, configure and manage non-trivial
+ Hadoop clusters ranging from a few nodes to extremely large clusters with
+ thousands of nodes.</p>
+ <p>
+ To play with Hadoop, you may first want to install Hadoop on a single machine (see <a href="quickstart.html"> Hadoop Quick Start</a>).
+ </p>
+ </section>
+
+ <section>
+ <title>Pre-requisites</title>
+
+ <ol>
+ <li>
+ Make sure all <a href="quickstart.html#PreReqs">requisite</a> software
+ is installed on all nodes in your cluster.
+ </li>
+ <li>
+ <a href="quickstart.html#Download">Get</a> the Hadoop software.
+ </li>
+ </ol>
+ </section>
+
+ <section>
+ <title>Installation</title>
+
+ <p>Installing a Hadoop cluster typically involves unpacking the software
+ on all the machines in the cluster.</p>
+
+ <p>Typically one machine in the cluster is designated as the
+ <code>NameNode</code> and another machine the as <code>JobTracker</code>,
+ exclusively. These are the <em>masters</em>. The rest of the machines in
+ the cluster act as both <code>DataNode</code> <em>and</em>
+ <code>TaskTracker</code>. These are the <em>slaves</em>.</p>
+
+ <p>The root of the distribution is referred to as
+ <code>HADOOP_HOME</code>. All machines in the cluster usually have the same
+ <code>HADOOP_HOME</code> path.</p>
+ </section>
+
+ <section>
+ <title>Configuration</title>
+
+ <p>The following sections describe how to configure a Hadoop cluster.</p>
+
+ <section>
+ <title>Configuration Files</title>
+
+ <p>Hadoop configuration is driven by two types of important
+ configuration files:</p>
+ <ol>
+ <li>
+ Read-only default configuration -
+ <a href="ext:core-default">src/core/core-default.xml</a>,
+ <a href="ext:hdfs-default">src/hdfs/hdfs-default.xml</a> and
+ <a href="ext:mapred-default">src/mapred/mapred-default.xml</a>.
+ </li>
+ <li>
+ Site-specific configuration -
+ <em>conf/core-site.xml</em>,
+ <em>conf/hdfs-site.xml</em> and
+ <em>conf/mapred-site.xml</em>.
+ </li>
+ </ol>
+
+ <p>To learn more about how the Hadoop framework is controlled by these
+ configuration files, look
+ <a href="ext:api/org/apache/hadoop/conf/configuration">here</a>.</p>
+
+ <p>Additionally, you can control the Hadoop scripts found in the
+ <code>bin/</code> directory of the distribution, by setting site-specific
+ values via the <code>conf/hadoop-env.sh</code>.</p>
+ </section>
+
+ <section>
+ <title>Site Configuration</title>
+
+ <p>To configure the Hadoop cluster you will need to configure the
+ <em>environment</em> in which the Hadoop daemons execute as well as
+ the <em>configuration parameters</em> for the Hadoop daemons.</p>
+
+ <p>The Hadoop daemons are <code>NameNode</code>/<code>DataNode</code>
+ and <code>JobTracker</code>/<code>TaskTracker</code>.</p>
+
+ <section>
+ <title>Configuring the Environment of the Hadoop Daemons</title>
+
+ <p>Administrators should use the <code>conf/hadoop-env.sh</code> script
+ to do site-specific customization of the Hadoop daemons' process
+ environment.</p>
+
+ <p>At the very least you should specify the
+ <code>JAVA_HOME</code> so that it is correctly defined on each
+ remote node.</p>
+
+ <p>Administrators can configure individual daemons using the
+ configuration options <code>HADOOP_*_OPTS</code>. Various options
+ available are shown below in the table. </p>
+ <table>
+ <tr><th>Daemon</th><th>Configure Options</th></tr>
+ <tr><td>NameNode</td><td>HADOOP_NAMENODE_OPTS</td></tr>
+ <tr><td>DataNode</td><td>HADOOP_DATANODE_OPTS</td></tr>
+ <tr><td>SecondaryNamenode</td>
+ <td>HADOOP_SECONDARYNAMENODE_OPTS</td></tr>
+ <tr><td>JobTracker</td><td>HADOOP_JOBTRACKER_OPTS</td></tr>
+ <tr><td>TaskTracker</td><td>HADOOP_TASKTRACKER_OPTS</td></tr>
+ </table>
+
+ <p> For example, To configure Namenode to use parallelGC, the
+ following statement should be added in <code>hadoop-env.sh</code> :
+ <br/><code>
+ export HADOOP_NAMENODE_OPTS="-XX:+UseParallelGC ${HADOOP_NAMENODE_OPTS}"
+ </code><br/></p>
+
+ <p>Other useful configuration parameters that you can customize
+ include:</p>
+ <ul>
+ <li>
+ <code>HADOOP_LOG_DIR</code> - The directory where the daemons'
+ log files are stored. They are automatically created if they don't
+ exist.
+ </li>
+ <li>
+ <code>HADOOP_HEAPSIZE</code> - The maximum amount of heapsize
+ to use, in MB e.g. <code>1000MB</code>. This is used to
+ configure the heap size for the hadoop daemon. By default,
+ the value is <code>1000MB</code>.
+ </li>
+ </ul>
+ </section>
+
+ <section>
+ <title>Configuring the Hadoop Daemons</title>
+
+ <p>This section deals with important parameters to be specified in the
+ following:
+ <br/>
+ <code>conf/core-site.xml</code>:</p>
+
+ <table>
+ <tr>
+ <th>Parameter</th>
+ <th>Value</th>
+ <th>Notes</th>
+ </tr>
+ <tr>
+ <td>fs.default.name</td>
+ <td>URI of <code>NameNode</code>.</td>
+ <td><em>hdfs://hostname/</em></td>
+ </tr>
+ </table>
+
+ <p><br/><code>conf/hdfs-site.xml</code>:</p>
+
+ <table>
+ <tr>
+ <th>Parameter</th>
+ <th>Value</th>
+ <th>Notes</th>
+ </tr>
+ <tr>
+ <td>dfs.name.dir</td>
+ <td>
+ Path on the local filesystem where the <code>NameNode</code>
+ stores the namespace and transactions logs persistently.</td>
+ <td>
+ If this is a comma-delimited list of directories then the name
+ table is replicated in all of the directories, for redundancy.
+ </td>
+ </tr>
+ <tr>
+ <td>dfs.data.dir</td>
+ <td>
+ Comma separated list of paths on the local filesystem of a
+ <code>DataNode</code> where it should store its blocks.
+ </td>
+ <td>
+ If this is a comma-delimited list of directories, then data will
+ be stored in all named directories, typically on different
+ devices.
+ </td>
+ </tr>
+ </table>
+
+ <p><br/><code>conf/mapred-site.xml</code>:</p>
+
+ <table>
+ <tr>
+ <th>Parameter</th>
+ <th>Value</th>
+ <th>Notes</th>
+ </tr>
+ <tr>
+ <td>mapred.job.tracker</td>
+ <td>Host or IP and port of <code>JobTracker</code>.</td>
+ <td><em>host:port</em> pair.</td>
+ </tr>
+ <tr>
+ <td>mapred.system.dir</td>
+ <td>
+ Path on the HDFS where where the Map/Reduce framework stores
+ system files e.g. <code>/hadoop/mapred/system/</code>.
+ </td>
+ <td>
+ This is in the default filesystem (HDFS) and must be accessible
+ from both the server and client machines.
+ </td>
+ </tr>
+ <tr>
+ <td>mapred.local.dir</td>
+ <td>
+ Comma-separated list of paths on the local filesystem where
+ temporary Map/Reduce data is written.
+ </td>
+ <td>Multiple paths help spread disk i/o.</td>
+ </tr>
+ <tr>
+ <td>mapred.tasktracker.{map|reduce}.tasks.maximum</td>
+ <td>
+ The maximum number of Map/Reduce tasks, which are run
+ simultaneously on a given <code>TaskTracker</code>, individually.
+ </td>
+ <td>
+ Defaults to 2 (2 maps and 2 reduces), but vary it depending on
+ your hardware.
+ </td>
+ </tr>
+ <tr>
+ <td>dfs.hosts/dfs.hosts.exclude</td>
+ <td>List of permitted/excluded DataNodes.</td>
+ <td>
+ If necessary, use these files to control the list of allowable
+ datanodes.
+ </td>
+ </tr>
+ <tr>
+ <td>mapred.hosts/mapred.hosts.exclude</td>
+ <td>List of permitted/excluded TaskTrackers.</td>
+ <td>
+ If necessary, use these files to control the list of allowable
+ TaskTrackers.
+ </td>
+ </tr>
+ <tr>
+ <td>mapred.queue.names</td>
+ <td>Comma separated list of queues to which jobs can be submitted.</td>
+ <td>
+ The Map/Reduce system always supports atleast one queue
+ with the name as <em>default</em>. Hence, this parameter's
+ value should always contain the string <em>default</em>.
+ Some job schedulers supported in Hadoop, like the
+ <a href="capacity_scheduler.html">Capacity
+ Scheduler</a>, support multiple queues. If such a scheduler is
+ being used, the list of configured queue names must be
+ specified here. Once queues are defined, users can submit
+ jobs to a queue using the property name
+ <em>mapred.job.queue.name</em> in the job configuration.
+ There could be a separate
+ configuration file for configuring properties of these
+ queues that is managed by the scheduler.
+ Refer to the documentation of the scheduler for information on
+ the same.
+ </td>
+ </tr>
+ <tr>
+ <td>mapred.acls.enabled</td>
+ <td>Specifies whether ACLs are supported for controlling job
+ submission and administration</td>
+ <td>
+ If <em>true</em>, ACLs would be checked while submitting
+ and administering jobs. ACLs can be specified using the
+ configuration parameters of the form
+ <em>mapred.queue.queue-name.acl-name</em>, defined below.
+ </td>
+ </tr>
+ </table>
+
+ <p><br/><code> conf/mapred-queue-acls.xml</code></p>
+
+ <table>
+ <tr>
+ <th>Parameter</th>
+ <th>Value</th>
+ <th>Notes</th>
+ </tr>
+ <tr>
+ <td>mapred.queue.<em>queue-name</em>.acl-submit-job</td>
+ <td>List of users and groups that can submit jobs to the
+ specified <em>queue-name</em>.</td>
+ <td>
+ The list of users and groups are both comma separated
+ list of names. The two lists are separated by a blank.
+ Example: <em>user1,user2 group1,group2</em>.
+ If you wish to define only a list of groups, provide
+ a blank at the beginning of the value.
+ </td>
+ </tr>
+ <tr>
+ <td>mapred.queue.<em>queue-name</em>.acl-administer-job</td>
+ <td>List of users and groups that can change the priority
+ or kill jobs that have been submitted to the
+ specified <em>queue-name</em>.</td>
+ <td>
+ The list of users and groups are both comma separated
+ list of names. The two lists are separated by a blank.
+ Example: <em>user1,user2 group1,group2</em>.
+ If you wish to define only a list of groups, provide
+ a blank at the beginning of the value. Note that an
+ owner of a job can always change the priority or kill
+ his/her own job, irrespective of the ACLs.
+ </td>
+ </tr>
+ </table>
+
+
+ <p>Typically all the above parameters are marked as
+ <a href="ext:api/org/apache/hadoop/conf/configuration/final_parameters">
+ final</a> to ensure that they cannot be overriden by user-applications.
+ </p>
+
+ <section>
+ <title>Real-World Cluster Configurations</title>
+
+ <p>This section lists some non-default configuration parameters which
+ have been used to run the <em>sort</em> benchmark on very large
+ clusters.</p>
+
+ <ul>
+ <li>
+ <p>Some non-default configuration values used to run sort900,
+ that is 9TB of data sorted on a cluster with 900 nodes:</p>
+ <table>
+ <tr>
+ <th>Configuration File</th>
+ <th>Parameter</th>
+ <th>Value</th>
+ <th>Notes</th>
+ </tr>
+ <tr>
+ <td>conf/hdfs-site.xml</td>
+ <td>dfs.block.size</td>
+ <td>134217728</td>
+ <td>HDFS blocksize of 128MB for large file-systems.</td>
+ </tr>
+ <tr>
+ <td>conf/hdfs-site.xml</td>
+ <td>dfs.namenode.handler.count</td>
+ <td>40</td>
+ <td>
+ More NameNode server threads to handle RPCs from large
+ number of DataNodes.
+ </td>
+ </tr>
+ <tr>
+ <td>conf/mapred-site.xml</td>
+ <td>mapred.reduce.parallel.copies</td>
+ <td>20</td>
+ <td>
+ Higher number of parallel copies run by reduces to fetch
+ outputs from very large number of maps.
+ </td>
+ </tr>
+ <tr>
+ <td>conf/mapred-site.xml</td>
+ <td>mapred.child.java.opts</td>
+ <td>-Xmx512M</td>
+ <td>
+ Larger heap-size for child jvms of maps/reduces.
+ </td>
+ </tr>
+ <tr>
+ <td>conf/core-site.xml</td>
+ <td>fs.inmemory.size.mb</td>
+ <td>200</td>
+ <td>
+ Larger amount of memory allocated for the in-memory
+ file-system used to merge map-outputs at the reduces.
+ </td>
+ </tr>
+ <tr>
+ <td>conf/core-site.xml</td>
+ <td>io.sort.factor</td>
+ <td>100</td>
+ <td>More streams merged at once while sorting files.</td>
+ </tr>
+ <tr>
+ <td>conf/core-site.xml</td>
+ <td>io.sort.mb</td>
+ <td>200</td>
+ <td>Higher memory-limit while sorting data.</td>
+ </tr>
+ <tr>
+ <td>conf/core-site.xml</td>
+ <td>io.file.buffer.size</td>
+ <td>131072</td>
+ <td>Size of read/write buffer used in SequenceFiles.</td>
+ </tr>
+ </table>
+ </li>
+ <li>
+ <p>Updates to some configuration values to run sort1400 and
+ sort2000, that is 14TB of data sorted on 1400 nodes and 20TB of
+ data sorted on 2000 nodes:</p>
+ <table>
+ <tr>
+ <th>Configuration File</th>
+ <th>Parameter</th>
+ <th>Value</th>
+ <th>Notes</th>
+ </tr>
+ <tr>
+ <td>conf/mapred-site.xml</td>
+ <td>mapred.job.tracker.handler.count</td>
+ <td>60</td>
+ <td>
+ More JobTracker server threads to handle RPCs from large
+ number of TaskTrackers.
+ </td>
+ </tr>
+ <tr>
+ <td>conf/mapred-site.xml</td>
+ <td>mapred.reduce.parallel.copies</td>
+ <td>50</td>
+ <td></td>
+ </tr>
+ <tr>
+ <td>conf/mapred-site.xml</td>
+ <td>tasktracker.http.threads</td>
+ <td>50</td>
+ <td>
+ More worker threads for the TaskTracker's http server. The
+ http server is used by reduces to fetch intermediate
+ map-outputs.
+ </td>
+ </tr>
+ <tr>
+ <td>conf/mapred-site.xml</td>
+ <td>mapred.child.java.opts</td>
+ <td>-Xmx1024M</td>
+ <td>Larger heap-size for child jvms of maps/reduces.</td>
+ </tr>
+ </table>
+ </li>
+ </ul>
+ </section>
+ <section>
+ <title> Memory management</title>
+ <p>Users/admins can also specify the maximum virtual memory
+ of the launched child-task, and any sub-process it launches
+ recursively, using <code>mapred.child.ulimit</code>. Note that
+ the value set here is a per process limit.
+ The value for <code>mapred.child.ulimit</code> should be specified
+ in kilo bytes (KB). And also the value must be greater than
+ or equal to the -Xmx passed to JavaVM, else the VM might not start.
+ </p>
+
+ <p>Note: <code>mapred.child.java.opts</code> are used only for
+ configuring the launched child tasks from task tracker. Configuring
+ the memory options for daemons is documented in
+ <a href="cluster_setup.html#Configuring+the+Environment+of+the+Hadoop+Daemons">
+ cluster_setup.html </a></p>
+
+ <p>The memory available to some parts of the framework is also
+ configurable. In map and reduce tasks, performance may be influenced
+ by adjusting parameters influencing the concurrency of operations and
+ the frequency with which data will hit disk. Monitoring the filesystem
+ counters for a job- particularly relative to byte counts from the map
+ and into the reduce- is invaluable to the tuning of these
+ parameters.</p>
+ </section>
+
+ <section>
+ <title> Memory monitoring</title>
+ <p>A <code>TaskTracker</code>(TT) can be configured to monitor memory
+ usage of tasks it spawns, so that badly-behaved jobs do not bring
+ down a machine due to excess memory consumption. With monitoring
+ enabled, every task is assigned a task-limit for virtual memory (VMEM).
+ In addition, every node is assigned a node-limit for VMEM usage.
+ A TT ensures that a task is killed if it, and
+ its descendants, use VMEM over the task's per-task limit. It also
+ ensures that one or more tasks are killed if the sum total of VMEM
+ usage by all tasks, and their descendents, cross the node-limit.</p>
+
+ <p>Users can, optionally, specify the VMEM task-limit per job. If no
+ such limit is provided, a default limit is used. A node-limit can be
+ set per node.</p>
+ <p>Currently the memory monitoring and management is only supported
+ in Linux platform.</p>
+ <p>To enable monitoring for a TT, the
+ following parameters all need to be set:</p>
+
+ <table>
+ <tr><th>Name</th><th>Type</th><th>Description</th></tr>
+ <tr><td>mapred.tasktracker.vmem.reserved</td><td>long</td>
+ <td>A number, in bytes, that represents an offset. The total VMEM on
+ the machine, minus this offset, is the VMEM node-limit for all
+ tasks, and their descendants, spawned by the TT.
+ </td></tr>
+ <tr><td>mapred.task.default.maxvmem</td><td>long</td>
+ <td>A number, in bytes, that represents the default VMEM task-limit
+ associated with a task. Unless overridden by a job's setting,
+ this number defines the VMEM task-limit.
+ </td></tr>
+ <tr><td>mapred.task.limit.maxvmem</td><td>long</td>
+ <td>A number, in bytes, that represents the upper VMEM task-limit
+ associated with a task. Users, when specifying a VMEM task-limit
+ for their tasks, should not specify a limit which exceeds this amount.
+ </td></tr>
+ </table>
+
+ <p>In addition, the following parameters can also be configured.</p>
+
+ <table>
+ <tr><th>Name</th><th>Type</th><th>Description</th></tr>
+ <tr><td>mapred.tasktracker.taskmemorymanager.monitoring-interval</td>
+ <td>long</td>
+ <td>The time interval, in milliseconds, between which the TT
+ checks for any memory violation. The default value is 5000 msec
+ (5 seconds).
+ </td></tr>
+ </table>
+
+ <p>Here's how the memory monitoring works for a TT.</p>
+ <ol>
+ <li>If one or more of the configuration parameters described
+ above are missing or -1 is specified , memory monitoring is
+ disabled for the TT.
+ </li>
+ <li>In addition, monitoring is disabled if
+ <code>mapred.task.default.maxvmem</code> is greater than
+ <code>mapred.task.limit.maxvmem</code>.
+ </li>
+ <li>If a TT receives a task whose task-limit is set by the user
+ to a value larger than <code>mapred.task.limit.maxvmem</code>, it
+ logs a warning but executes the task.
+ </li>
+ <li>Periodically, the TT checks the following:
+ <ul>
+ <li>If any task's current VMEM usage is greater than that task's
+ VMEM task-limit, the task is killed and reason for killing
+ the task is logged in task diagonistics . Such a task is considered
+ failed, i.e., the killing counts towards the task's failure count.
+ </li>
+ <li>If the sum total of VMEM used by all tasks and descendants is
+ greater than the node-limit, the TT kills enough tasks, in the
+ order of least progress made, till the overall VMEM usage falls
+ below the node-limt. Such killed tasks are not considered failed
+ and their killing does not count towards the tasks' failure counts.
+ </li>
+ </ul>
+ </li>
+ </ol>
+
+ <p>Schedulers can choose to ease the monitoring pressure on the TT by
+ preventing too many tasks from running on a node and by scheduling
+ tasks only if the TT has enough VMEM free. In addition, Schedulers may
+ choose to consider the physical memory (RAM) available on the node
+ as well. To enable Scheduler support, TTs report their memory settings
+ to the JobTracker in every heartbeat. Before getting into details,
+ consider the following additional memory-related parameters than can be
+ configured to enable better scheduling:</p>
+
+ <table>
+ <tr><th>Name</th><th>Type</th><th>Description</th></tr>
+ <tr><td>mapred.tasktracker.pmem.reserved</td><td>int</td>
+ <td>A number, in bytes, that represents an offset. The total
+ physical memory (RAM) on the machine, minus this offset, is the
+ recommended RAM node-limit. The RAM node-limit is a hint to a
+ Scheduler to scheduler only so many tasks such that the sum
+ total of their RAM requirements does not exceed this limit.
+ RAM usage is not monitored by a TT.
+ </td></tr>
+ </table>
+
+ <p>A TT reports the following memory-related numbers in every
+ heartbeat:</p>
+ <ul>
+ <li>The total VMEM available on the node.</li>
+ <li>The value of <code>mapred.tasktracker.vmem.reserved</code>,
+ if set.</li>
+ <li>The total RAM available on the node.</li>
+ <li>The value of <code>mapred.tasktracker.pmem.reserved</code>,
+ if set.</li>
+ </ul>
+ </section>
+
+ <section>
+ <title>Task Controllers</title>
+ <p>Task controllers are classes in the Hadoop Map/Reduce
+ framework that define how user's map and reduce tasks
+ are launched and controlled. They can
+ be used in clusters that require some customization in
+ the process of launching or controlling the user tasks.
+ For example, in some
+ clusters, there may be a requirement to run tasks as
+ the user who submitted the job, instead of as the task
+ tracker user, which is how tasks are launched by default.
+ This section describes how to configure and use
+ task controllers.</p>
+ <p>The following task controllers are the available in
+ Hadoop.
+ </p>
+ <table>
+ <tr><th>Name</th><th>Class Name</th><th>Description</th></tr>
+ <tr>
+ <td>DefaultTaskController</td>
+ <td>org.apache.hadoop.mapred.DefaultTaskController</td>
+ <td> The default task controller which Hadoop uses to manage task
+ execution. The tasks run as the task tracker user.</td>
+ </tr>
+ <tr>
+ <td>LinuxTaskController</td>
+ <td>org.apache.hadoop.mapred.LinuxTaskController</td>
+ <td>This task controller, which is supported only on Linux,
+ runs the tasks as the user who submitted the job. It requires
+ these user accounts to be created on the cluster nodes
+ where the tasks are launched. It
+ uses a setuid executable that is included in the Hadoop
+ distribution. The task tracker uses this executable to
+ launch and kill tasks. The setuid executable switches to
+ the user who has submitted the job and launches or kills
+ the tasks. Currently, this task controller
+ opens up permissions to local files and directories used
+ by the tasks such as the job jar files, distributed archive
+ files, intermediate files and task log files. In future,
+ it is expected that stricter file permissions are used.
+ </td>
+ </tr>
+ </table>
+ <section>
+ <title>Configuring Task Controllers</title>
+ <p>The task controller to be used can be configured by setting the
+ value of the following key in mapred-site.xml</p>
+ <table>
+ <tr>
+ <th>Property</th><th>Value</th><th>Notes</th>
+ </tr>
+ <tr>
+ <td>mapred.task.tracker.task-controller</td>
+ <td>Fully qualified class name of the task controller class</td>
+ <td>Currently there are two implementations of task controller
+ in the Hadoop system, DefaultTaskController and LinuxTaskController.
+ Refer to the class names mentioned above to determine the value
+ to set for the class of choice.
+ </td>
+ </tr>
+ </table>
+ </section>
+ <section>
+ <title>Using the LinuxTaskController</title>
+ <p>This section of the document describes the steps required to
+ use the LinuxTaskController.</p>
+
+ <p>In order to use the LinuxTaskController, a setuid executable
+ should be built and deployed on the compute nodes. The
+ executable is named task-controller. To build the executable,
+ execute
+ <em>ant task-controller -Dhadoop.conf.dir=/path/to/conf/dir.
+ </em>
+ The path passed in <em>-Dhadoop.conf.dir</em> should be the path
+ on the cluster nodes where a configuration file for the setuid
+ executable would be located. The executable would be built to
+ <em>build.dir/dist.dir/bin</em> and should be installed to
+ <em>$HADOOP_HOME/bin</em>.
+ </p>
+
+ <p>
+ The executable must be deployed as a setuid executable, by changing
+ the ownership to <em>root</em> and giving it permissions <em>4755</em>.
+ </p>
+
+ <p>The executable requires a configuration file called
+ <em>taskcontroller.cfg</em> to be
+ present in the configuration directory passed to the ant target
+ mentioned above. If the binary was not built with a specific
+ conf directory, the path defaults to <em>/path-to-binary/../conf</em>.
+ </p>
+
+ <p>The executable requires following configuration items to be
+ present in the <em>taskcontroller.cfg</em> file. The items should
+ be mentioned as simple <em>key=value</em> pairs.
+ </p>
+ <table><tr><th>Name</th><th>Description</th></tr>
+ <tr>
+ <td>mapred.local.dir</td>
+ <td>Path to mapred local directories. Should be same as the value
+ which was provided to key in mapred-site.xml. This is required to
+ validate paths passed to the setuid executable in order to prevent
+ arbitrary paths being passed to it.</td>
+ </tr>
+ </table>
+
+ <p>
+ The LinuxTaskController requires that paths leading up to
+ the directories specified in
+ <em>mapred.local.dir</em> and <em>hadoop.log.dir</em> to be 755
+ and directories themselves having 777 permissions.
+ </p>
+ </section>
+
+ </section>
+
+ </section>
+
+ <section>
+ <title>Slaves</title>
+
+ <p>Typically you choose one machine in the cluster to act as the
+ <code>NameNode</code> and one machine as to act as the
+ <code>JobTracker</code>, exclusively. The rest of the machines act as
+ both a <code>DataNode</code> and <code>TaskTracker</code> and are
+ referred to as <em>slaves</em>.</p>
+
+ <p>List all slave hostnames or IP addresses in your
+ <code>conf/slaves</code> file, one per line.</p>
+ </section>
+
+ <section>
+ <title>Logging</title>
+
+ <p>Hadoop uses the <a href="http://logging.apache.org/log4j/">Apache
+ log4j</a> via the <a href="http://commons.apache.org/logging/">Apache
+ Commons Logging</a> framework for logging. Edit the
+ <code>conf/log4j.properties</code> file to customize the Hadoop
+ daemons' logging configuration (log-formats and so on).</p>
+
+ <section>
+ <title>History Logging</title>
+
+ <p> The job history files are stored in central location
+ <code> hadoop.job.history.location </code> which can be on DFS also,
+ whose default value is <code>${HADOOP_LOG_DIR}/history</code>.
+ The history web UI is accessible from job tracker web UI.</p>
+
+ <p> The history files are also logged to user specified directory
+ <code>hadoop.job.history.user.location</code>
+ which defaults to job output directory. The files are stored in
+ "_logs/history/" in the specified directory. Hence, by default
+ they will be in "mapred.output.dir/_logs/history/". User can stop
+ logging by giving the value <code>none</code> for
+ <code>hadoop.job.history.user.location</code> </p>
+
+ <p> User can view the history logs summary in specified directory
+ using the following command <br/>
+ <code>$ bin/hadoop job -history output-dir</code><br/>
+ This command will print job details, failed and killed tip
+ details. <br/>
+ More details about the job such as successful tasks and
+ task attempts made for each task can be viewed using the
+ following command <br/>
+ <code>$ bin/hadoop job -history all output-dir</code><br/></p>
+ </section>
+ </section>
+ </section>
+
+ <p>Once all the necessary configuration is complete, distribute the files
+ to the <code>HADOOP_CONF_DIR</code> directory on all the machines,
+ typically <code>${HADOOP_HOME}/conf</code>.</p>
+ </section>
+ <section>
+ <title>Cluster Restartability</title>
+ <section>
+ <title>Map/Reduce</title>
+ <p>The job tracker restart can recover running jobs if
+ <code>mapred.jobtracker.restart.recover</code> is set true and
+ <a href="#Logging">JobHistory logging</a> is enabled. Also
+ <code>mapred.jobtracker.job.history.block.size</code> value should be
+ set to an optimal value to dump job history to disk as soon as
+ possible, the typical value is 3145728(3MB).</p>
+ </section>
+ </section>
+
+ <section>
+ <title>Hadoop Rack Awareness</title>
+ <p>The HDFS and the Map/Reduce components are rack-aware.</p>
+ <p>The <code>NameNode</code> and the <code>JobTracker</code> obtains the
+ <code>rack id</code> of the slaves in the cluster by invoking an API
+ <a href="ext:api/org/apache/hadoop/net/dnstoswitchmapping/resolve
+ ">resolve</a> in an administrator configured
+ module. The API resolves the slave's DNS name (also IP address) to a
+ rack id. What module to use can be configured using the configuration
+ item <code>topology.node.switch.mapping.impl</code>. The default
+ implementation of the same runs a script/command configured using
+ <code>topology.script.file.name</code>. If topology.script.file.name is
+ not set, the rack id <code>/default-rack</code> is returned for any
+ passed IP address. The additional configuration in the Map/Reduce
+ part is <code>mapred.cache.task.levels</code> which determines the number
+ of levels (in the network topology) of caches. So, for example, if it is
+ the default value of 2, two levels of caches will be constructed -
+ one for hosts (host -> task mapping) and another for racks
+ (rack -> task mapping).
+ </p>
+ </section>
+
+ <section>
+ <title>Hadoop Startup</title>
+
+ <p>To start a Hadoop cluster you will need to start both the HDFS and
+ Map/Reduce cluster.</p>
+
+ <p>
+ Format a new distributed filesystem:<br/>
+ <code>$ bin/hadoop namenode -format</code>
+ </p>
+
+ <p>
+ Start the HDFS with the following command, run on the designated
+ <code>NameNode</code>:<br/>
+ <code>$ bin/start-dfs.sh</code>
+ </p>
+ <p>The <code>bin/start-dfs.sh</code> script also consults the
+ <code>${HADOOP_CONF_DIR}/slaves</code> file on the <code>NameNode</code>
+ and starts the <code>DataNode</code> daemon on all the listed slaves.</p>
+
+ <p>
+ Start Map-Reduce with the following command, run on the designated
+ <code>JobTracker</code>:<br/>
+ <code>$ bin/start-mapred.sh</code>
+ </p>
+ <p>The <code>bin/start-mapred.sh</code> script also consults the
+ <code>${HADOOP_CONF_DIR}/slaves</code> file on the <code>JobTracker</code>
+ and starts the <code>TaskTracker</code> daemon on all the listed slaves.
+ </p>
+ </section>
+
+ <section>
+ <title>Hadoop Shutdown</title>
+
+ <p>
+ Stop HDFS with the following command, run on the designated
+ <code>NameNode</code>:<br/>
+ <code>$ bin/stop-dfs.sh</code>
+ </p>
+ <p>The <code>bin/stop-dfs.sh</code> script also consults the
+ <code>${HADOOP_CONF_DIR}/slaves</code> file on the <code>NameNode</code>
+ and stops the <code>DataNode</code> daemon on all the listed slaves.</p>
+
+ <p>
+ Stop Map/Reduce with the following command, run on the designated
+ the designated <code>JobTracker</code>:<br/>
+ <code>$ bin/stop-mapred.sh</code><br/>
+ </p>
+ <p>The <code>bin/stop-mapred.sh</code> script also consults the
+ <code>${HADOOP_CONF_DIR}/slaves</code> file on the <code>JobTracker</code>
+ and stops the <code>TaskTracker</code> daemon on all the listed slaves.</p>
+ </section>
+ </body>
+
+</document>
Propchange: hadoop/hdfs/branches/HDFS-265/src/docs/src/documentation/content/xdocs/cluster_setup.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain