You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/05/27 18:49:57 UTC
svn commit: r1128390 [1/2] - in
/hadoop/common/branches/branch-0.20-security-204: ./ bin/ conf/ ivy/
src/docs/src/documentation/content/xdocs/ src/packages/ src/packages/deb/
src/packages/deb/hadoop.control/ src/packages/deb/init.d/ src/packages/rpm/
s...
Author: omalley
Date: Fri May 27 16:49:55 2011
New Revision: 1128390
URL: http://svn.apache.org/viewvc?rev=1128390&view=rev
Log:
HADOOP-6255. Create RPM and Debian packages for common. Changes deployment
layout to be consistent across the binary tgz, rpm, and deb. Adds setup
scripts for easy one node cluster configuration and user creation.
(Eric Yang via omalley)
Added:
hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/deployment_layout.xml
hadoop/common/branches/branch-0.20-security-204/src/packages/
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/conffile
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/control
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postinst
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postrm
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/preinst
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/prerm
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-datanode
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-jobtracker
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-namenode
hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-tasktracker
hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-create-user.sh
hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-setup-conf.sh
hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-setup-hdfs.sh
hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-setup-single-node.sh
hadoop/common/branches/branch-0.20-security-204/src/packages/rpm/
hadoop/common/branches/branch-0.20-security-204/src/packages/rpm/init.d/
hadoop/common/branches/branch-0.20-security-204/src/packages/rpm/init.d/hadoop-datanode
hadoop/common/branches/branch-0.20-security-204/src/packages/rpm/init.d/hadoop-jobtracker
hadoop/common/branches/branch-0.20-security-204/src/packages/rpm/init.d/hadoop-namenode
hadoop/common/branches/branch-0.20-security-204/src/packages/rpm/init.d/hadoop-tasktracker
hadoop/common/branches/branch-0.20-security-204/src/packages/rpm/spec/
hadoop/common/branches/branch-0.20-security-204/src/packages/rpm/spec/hadoop.spec
hadoop/common/branches/branch-0.20-security-204/src/packages/templates/
hadoop/common/branches/branch-0.20-security-204/src/packages/templates/conf/
hadoop/common/branches/branch-0.20-security-204/src/packages/templates/conf/core-site.xml
hadoop/common/branches/branch-0.20-security-204/src/packages/templates/conf/hadoop-env.sh
hadoop/common/branches/branch-0.20-security-204/src/packages/templates/conf/hdfs-site.xml
hadoop/common/branches/branch-0.20-security-204/src/packages/templates/conf/mapred-site.xml
hadoop/common/branches/branch-0.20-security-204/src/packages/update-hadoop-env.sh
Modified:
hadoop/common/branches/branch-0.20-security-204/CHANGES.txt
hadoop/common/branches/branch-0.20-security-204/bin/hadoop
hadoop/common/branches/branch-0.20-security-204/bin/hadoop-config.sh
hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemon.sh
hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemons.sh
hadoop/common/branches/branch-0.20-security-204/bin/rcc
hadoop/common/branches/branch-0.20-security-204/bin/slaves.sh
hadoop/common/branches/branch-0.20-security-204/bin/start-all.sh
hadoop/common/branches/branch-0.20-security-204/bin/start-balancer.sh
hadoop/common/branches/branch-0.20-security-204/bin/start-dfs.sh
hadoop/common/branches/branch-0.20-security-204/bin/start-jobhistoryserver.sh
hadoop/common/branches/branch-0.20-security-204/bin/start-mapred.sh
hadoop/common/branches/branch-0.20-security-204/bin/stop-all.sh
hadoop/common/branches/branch-0.20-security-204/bin/stop-balancer.sh
hadoop/common/branches/branch-0.20-security-204/bin/stop-dfs.sh
hadoop/common/branches/branch-0.20-security-204/bin/stop-jobhistoryserver.sh
hadoop/common/branches/branch-0.20-security-204/bin/stop-mapred.sh
hadoop/common/branches/branch-0.20-security-204/build.xml
hadoop/common/branches/branch-0.20-security-204/conf/log4j.properties
hadoop/common/branches/branch-0.20-security-204/ivy.xml
hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties
hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/site.xml
Modified: hadoop/common/branches/branch-0.20-security-204/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/CHANGES.txt?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security-204/CHANGES.txt Fri May 27 16:49:55 2011
@@ -6,6 +6,13 @@ Release 0.20.205.0 - unreleased
Release 0.20.204.0 - unreleased
+ NEW FEATURES
+
+ HADOOP-6255. Create RPM and Debian packages for common. Changes deployment
+ layout to be consistent across the binary tgz, rpm, and deb. Adds setup
+ scripts for easy one node cluster configuration and user creation.
+ (Eric Yang via omalley)
+
BUG FIXES
MAPREDUCE-2447. Fix Child.java to set Task.jvmContext sooner to avoid
Modified: hadoop/common/branches/branch-0.20-security-204/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/hadoop?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/hadoop (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/hadoop Fri May 27 16:49:55 2011
@@ -50,7 +50,12 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+if [ "$HADOOP_HOME" != "" ]; then
+ echo "Warning: \$HADOOP_HOME is deprecated."
+ echo
+fi
+
+. "$bin"/../libexec/hadoop-config.sh
cygwin=false
case "`uname`" in
@@ -308,6 +313,10 @@ if [ -d "${HADOOP_HOME}/build/native" -o
fi
fi
fi
+if [ -e "${HADOOP_PREFIX}/lib/libhadoop.a" ]; then
+ JAVA_LIBRARY_PATH=${HADOOP_PREFIX}/lib
+fi
+
# cygwin path translation
if $cygwin; then
@@ -319,6 +328,13 @@ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.f
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_HOME"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+
+if [ $COMMAND = "namenode" ]; then
+ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,DRFAS}"
+else
+ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
+fi
+
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
fi
@@ -332,14 +348,14 @@ if [ "$starting_secure_dn" = "true" ]; t
HADOOP_SECURE_DN_PID="$HADOOP_PID_DIR/hadoop_secure_dn.pid"
fi
- exec "$HADOOP_HOME/bin/jsvc" -Dproc_$COMMAND -outfile "$HADOOP_LOG_DIR/jsvc.out" \
- -errfile "$HADOOP_LOG_DIR/jsvc.err" \
- -pidfile "$HADOOP_SECURE_DN_PID" \
- -nodetach \
- -user "$HADOOP_SECURE_DN_USER" \
- -cp "$CLASSPATH" \
- $JAVA_HEAP_MAX $HADOOP_OPTS \
- org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter "$@"
+ exec "$HADOOP_HOME/libexec/jsvc" -Dproc_$COMMAND -outfile "$HADOOP_LOG_DIR/jsvc.out" \
+ -errfile "$HADOOP_LOG_DIR/jsvc.err" \
+ -pidfile "$HADOOP_SECURE_DN_PID" \
+ -nodetach \
+ -user "$HADOOP_SECURE_DN_USER" \
+ -cp "$CLASSPATH" \
+ $JAVA_HEAP_MAX $HADOOP_OPTS \
+ org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter "$@"
else
# run it
exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@"
Modified: hadoop/common/branches/branch-0.20-security-204/bin/hadoop-config.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/hadoop-config.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/hadoop-config.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/hadoop-config.sh Fri May 27 16:49:55 2011
@@ -19,16 +19,10 @@
# resolve links - $0 may be a softlink
-this="$0"
-while [ -h "$this" ]; do
- ls=`ls -ld "$this"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '.*/.*' > /dev/null; then
- this="$link"
- else
- this=`dirname "$this"`/"$link"
- fi
-done
+this="${BASH_SOURCE-$0}"
+common_bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
+script="$(basename -- "$this")"
+this="$common_bin/$script"
# convert relative path to absolute path
bin=`dirname "$this"`
@@ -37,7 +31,8 @@ bin=`cd "$bin"; pwd`
this="$bin/$script"
# the root of the Hadoop installation
-export HADOOP_HOME=`dirname "$this"`/..
+export HADOOP_PREFIX=`dirname "$this"`/..
+export HADOOP_HOME=${HADOOP_PREFIX}/share/hadoop
#check to see if the conf dir is given as an optional argument
if [ $# -gt 1 ]
@@ -52,7 +47,12 @@ then
fi
# Allow alternate conf dir location.
-HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_HOME/conf}"
+if [ -e "${HADOOP_PREFIX}/conf/hadoop-env.sh" ]; then
+ DEFAULT_CONF_DIR="conf"
+else
+ DEFAULT_CONF_DIR="etc/hadoop"
+fi
+HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_PREFIX/$DEFAULT_CONF_DIR}"
#check to see it is specified whether to use the slaves or the
# masters file
Modified: hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemon.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemon.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemon.sh Fri May 27 16:49:55 2011
@@ -20,7 +20,7 @@
#
# Environment Variables
#
-# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf.
+# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_PREFIX}/conf.
# HADOOP_LOG_DIR Where log files are stored. PWD by default.
# HADOOP_MASTER host:path where hadoop code should be rsync'd from
# HADOOP_PID_DIR The pid files are stored. /tmp by default.
@@ -39,7 +39,7 @@ fi
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
# get arguments
startStop=$1
@@ -121,8 +121,8 @@ case $startStop in
hadoop_rotate_log $log
echo starting $command, logging to $log
- cd "$HADOOP_HOME"
- nohup nice -n $HADOOP_NICENESS "$HADOOP_HOME"/bin/hadoop --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null &
+ cd "$HADOOP_PREFIX"
+ nohup nice -n $HADOOP_NICENESS "$HADOOP_PREFIX"/bin/hadoop --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null &
echo $! > $pid
sleep 1; head "$log"
;;
Modified: hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemons.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemons.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemons.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/hadoop-daemons.sh Fri May 27 16:49:55 2011
@@ -29,6 +29,6 @@ fi
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. $bin/hadoop-config.sh
+. $bin/../libexec/hadoop-config.sh
exec "$bin/slaves.sh" --config $HADOOP_CONF_DIR cd "$HADOOP_HOME" \; "$bin/hadoop-daemon.sh" --config $HADOOP_CONF_DIR "$@"
Modified: hadoop/common/branches/branch-0.20-security-204/bin/rcc
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/rcc?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/rcc (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/rcc Fri May 27 16:49:55 2011
@@ -30,7 +30,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
. "${HADOOP_CONF_DIR}/hadoop-env.sh"
Modified: hadoop/common/branches/branch-0.20-security-204/bin/slaves.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/slaves.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/slaves.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/slaves.sh Fri May 27 16:49:55 2011
@@ -38,7 +38,7 @@ fi
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
# If the slaves file is specified in the command line,
# then it takes precedence over the definition in
Modified: hadoop/common/branches/branch-0.20-security-204/bin/start-all.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/start-all.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/start-all.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/start-all.sh Fri May 27 16:49:55 2011
@@ -21,7 +21,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
# start dfs daemons
"$bin"/start-dfs.sh --config $HADOOP_CONF_DIR
Modified: hadoop/common/branches/branch-0.20-security-204/bin/start-balancer.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/start-balancer.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/start-balancer.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/start-balancer.sh Fri May 27 16:49:55 2011
@@ -18,7 +18,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
# Start balancer daemon.
Modified: hadoop/common/branches/branch-0.20-security-204/bin/start-dfs.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/start-dfs.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/start-dfs.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/start-dfs.sh Fri May 27 16:49:55 2011
@@ -25,7 +25,7 @@ usage="Usage: start-dfs.sh [-upgrade|-ro
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
# get arguments
if [ $# -ge 1 ]; then
Modified: hadoop/common/branches/branch-0.20-security-204/bin/start-jobhistoryserver.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/start-jobhistoryserver.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/start-jobhistoryserver.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/start-jobhistoryserver.sh Fri May 27 16:49:55 2011
@@ -21,7 +21,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
# start daemon
"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start historyserver
Modified: hadoop/common/branches/branch-0.20-security-204/bin/start-mapred.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/start-mapred.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/start-mapred.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/start-mapred.sh Fri May 27 16:49:55 2011
@@ -21,7 +21,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
# start mapred daemons
# start jobtracker first to minimize connection errors at startup
Modified: hadoop/common/branches/branch-0.20-security-204/bin/stop-all.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/stop-all.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/stop-all.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/stop-all.sh Fri May 27 16:49:55 2011
@@ -21,7 +21,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
"$bin"/stop-mapred.sh --config $HADOOP_CONF_DIR
"$bin"/stop-dfs.sh --config $HADOOP_CONF_DIR
Modified: hadoop/common/branches/branch-0.20-security-204/bin/stop-balancer.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/stop-balancer.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/stop-balancer.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/stop-balancer.sh Fri May 27 16:49:55 2011
@@ -18,7 +18,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
# Stop balancer daemon.
# Run this on the machine where the balancer is running
Modified: hadoop/common/branches/branch-0.20-security-204/bin/stop-dfs.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/stop-dfs.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/stop-dfs.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/stop-dfs.sh Fri May 27 16:49:55 2011
@@ -21,7 +21,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop namenode
"$bin"/hadoop-daemons.sh --config $HADOOP_CONF_DIR stop datanode
Modified: hadoop/common/branches/branch-0.20-security-204/bin/stop-jobhistoryserver.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/stop-jobhistoryserver.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/stop-jobhistoryserver.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/stop-jobhistoryserver.sh Fri May 27 16:49:55 2011
@@ -21,7 +21,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop historyserver
Modified: hadoop/common/branches/branch-0.20-security-204/bin/stop-mapred.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/bin/stop-mapred.sh?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/bin/stop-mapred.sh (original)
+++ hadoop/common/branches/branch-0.20-security-204/bin/stop-mapred.sh Fri May 27 16:49:55 2011
@@ -21,7 +21,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-config.sh
+. "$bin"/../libexec/hadoop-config.sh
"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop jobtracker
"$bin"/hadoop-daemons.sh --config $HADOOP_CONF_DIR stop tasktracker
Modified: hadoop/common/branches/branch-0.20-security-204/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/build.xml?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/build.xml (original)
+++ hadoop/common/branches/branch-0.20-security-204/build.xml Fri May 27 16:49:55 2011
@@ -32,6 +32,7 @@
<property name="final.name" value="${name}-${version}"/>
<property name="test.final.name" value="${name}-test-${version}"/>
<property name="year" value="2009"/>
+ <property name="package.release" value="1"/>
<property name="core.final.name" value="${name}-core-${version}"/>
<property name="test.final.name" value="${name}-test-${version}"/>
@@ -61,6 +62,11 @@
<property name="librecordio.src" value="${c++.src}/librecordio"/>
<property name="tools.src" value="${basedir}/src/tools"/>
+ <property name="package.prefix" value="/usr"/>
+ <property name="package.conf.dir" value="/etc/hadoop"/>
+ <property name="package.log.dir" value="/var/log/hadoop"/>
+ <property name="package.pid.dir" value="/var/run/hadoop"/>
+
<property name="xercescroot" value=""/>
<property name="build.dir" value="${basedir}/build"/>
<property name="build.classes" value="${build.dir}/classes"/>
@@ -163,7 +169,7 @@
<property name="make.cmd" value="make"/>
<property name="jsvc.build.dir" value="${build.dir}/jsvc" />
- <property name="jsvc.install.dir" value="${dist.dir}/bin" />
+ <property name="jsvc.install.dir" value="${dist.dir}/libexec" />
<property name="jsvc.location" value="http://archive.apache.org/dist/commons/daemon/binaries/1.0.2/linux/commons-daemon-1.0.2-bin-linux-i386.tar.gz" />
<property name="jsvc.dest.name" value="jsvc.tar.gz" />
@@ -183,6 +189,9 @@
<!-- end of task-controller properties -->
+ <property name="package.buildroot" value="/tmp/hadoop_package_build_${user.name}"/>
+ <property name="package.build.dir" value="/tmp/hadoop_package_build_${user.name}/BUILD"/>
+
<!-- IVY properteis set here -->
<property name="ivy.dir" location="ivy" />
<loadproperties srcfile="${ivy.dir}/libraries.properties"/>
@@ -1375,11 +1384,13 @@
description="Build distribution">
<mkdir dir="${dist.dir}"/>
<mkdir dir="${dist.dir}/lib"/>
+ <mkdir dir="${dist.dir}/libexec"/>
<mkdir dir="${dist.dir}/contrib"/>
<mkdir dir="${dist.dir}/bin"/>
<mkdir dir="${dist.dir}/docs"/>
<mkdir dir="${dist.dir}/docs/api"/>
<mkdir dir="${dist.dir}/docs/jdiff"/>
+ <mkdir dir="${dist.dir}/sbin"/>
<copy todir="${dist.dir}/lib" includeEmptyDirs="false" flatten="true">
<fileset dir="${common.ivy.lib.dir}"/>
@@ -1418,6 +1429,12 @@
<fileset dir="bin"/>
</copy>
+ <copy todir="${dist.dir}/libexec">
+ <fileset dir="bin">
+ <include name="hadoop-config.sh"/>
+ </fileset>
+ </copy>
+
<copy todir="${dist.dir}/conf">
<fileset dir="${conf.dir}" excludes="**/*.template"/>
</copy>
@@ -1438,6 +1455,12 @@
</fileset>
</copy>
+ <copy todir="${dist.dir}/sbin">
+ <fileset dir="${basedir}/src/packages">
+ <include name="*.sh" />
+ </fileset>
+ </copy>
+
<copy todir="${dist.dir}/src" includeEmptyDirs="true">
<fileset dir="src" excludes="**/*.template **/docs/build/**/*"/>
</copy>
@@ -1450,6 +1473,7 @@
<chmod perm="ugo+x" type="file" parallel="false">
<fileset dir="${dist.dir}/bin"/>
+ <fileset dir="${dist.dir}/sbin"/>
<fileset dir="${dist.dir}/src/contrib/">
<include name="*/bin/*" />
</fileset>
@@ -1487,71 +1511,109 @@
<target name="bin-package" depends="compile, jar, examples, tools-jar, jar-test, ant-tasks, package-librecordio, jsvc"
description="assembles artifacts for binary target">
<mkdir dir="${dist.dir}"/>
- <mkdir dir="${dist.dir}/lib"/>
- <mkdir dir="${dist.dir}/contrib"/>
<mkdir dir="${dist.dir}/bin"/>
+ <mkdir dir="${dist.dir}/etc/hadoop"/>
+ <mkdir dir="${dist.dir}/lib"/>
+ <mkdir dir="${dist.dir}/libexec"/>
+ <mkdir dir="${dist.dir}/sbin"/>
+ <mkdir dir="${dist.dir}/share/${name}/contrib"/>
+ <mkdir dir="${dist.dir}/share/${name}/webapps"/>
+ <mkdir dir="${dist.dir}/share/${name}/templates/conf"/>
+
+ <copy todir="${dist.dir}/share/${name}/templates/conf" includeEmptyDirs="false">
+ <fileset dir="${basedir}/src/packages/templates/conf">
+ <include name="*"/>
+ </fileset>
+ </copy>
- <copy todir="${dist.dir}/lib" includeEmptyDirs="false" flatten="true">
+ <copy todir="${dist.dir}/share/${name}/lib" includeEmptyDirs="false" flatten="true">
<fileset dir="${common.ivy.lib.dir}"/>
</copy>
- <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
+ <copy todir="${dist.dir}/share/${name}/lib" includeEmptyDirs="false">
<fileset dir="lib">
<exclude name="**/native/**"/>
</fileset>
</copy>
- <exec dir="${dist.dir}" executable="sh" failonerror="true">
- <env key="BASE_NATIVE_LIB_DIR" value="${lib.dir}/native"/>
- <env key="BUILD_NATIVE_DIR" value="${build.dir}/native"/>
- <env key="DIST_LIB_DIR" value="${dist.dir}/lib/native"/>
- <arg line="${native.src.dir}/packageNativeHadoop.sh"/>
+ <exec dir="${dist.dir}" executable="sh" failonerror="true">
+ <env key="BASE_NATIVE_LIB_DIR" value="${lib.dir}/native"/>
+ <env key="BUILD_NATIVE_DIR" value="${build.dir}/native"/>
+ <env key="DIST_LIB_DIR" value="${dist.dir}/native"/>
+ <arg line="${native.src.dir}/packageNativeHadoop.sh"/>
</exec>
+ <move todir="${dist.dir}/lib" flatten="true">
+ <fileset dir="${dist.dir}/native">
+ <include name="*/*"/>
+ </fileset>
+ </move>
+ <delete dir="${dist.dir}/native"/>
+
<subant target="package">
<!--Pass down the version in case its needed again and the target
distribution directory so contribs know where to install to.-->
<property name="version" value="${version}"/>
- <property name="dist.dir" value="${dist.dir}"/>
+ <property name="dist.dir" value="${dist.dir}/share/${name}"/>
<fileset file="${contrib.dir}/build.xml"/>
</subant>
- <copy todir="${dist.dir}/webapps">
+ <copy todir="${dist.dir}/share/${name}/webapps">
<fileset dir="${build.webapps}"/>
</copy>
- <copy todir="${dist.dir}">
+ <copy todir="${dist.dir}/share/${name}">
<fileset file="${build.dir}/${name}-*-${version}.jar"/>
</copy>
- <copy todir="${dist.dir}/bin">
- <fileset dir="bin"/>
- </copy>
+ <copy file="bin/hadoop" todir="${dist.dir}/bin"/>
- <copy todir="${dist.dir}/conf">
- <fileset dir="${conf.dir}" excludes="**/*.template"/>
+ <copy todir="${dist.dir}/sbin">
+ <fileset dir="bin">
+ <include name="*"/>
+ <exclude name="hadoop"/>
+ <exclude name="hadoop-config.sh"/>
+ </fileset>
</copy>
- <copy file="ivy.xml" tofile="${dist.dir}/ivy.xml"/>
+ <copy todir="${dist.dir}/libexec">
+ <fileset dir="bin">
+ <include name="hadoop-config.sh"/>
+ </fileset>
+ </copy>
- <copy todir="${dist.dir}/ivy">
- <fileset dir="ivy"/>
+ <copy todir="${dist.dir}/etc/hadoop">
+ <fileset dir="${conf.dir}" excludes="**/*.template"/>
</copy>
- <copy todir="${dist.dir}">
+ <copy todir="${dist.dir}/share/doc/${name}">
<fileset dir=".">
<include name="*.txt" />
</fileset>
</copy>
- <copy todir="${dist.dir}/c++" includeEmptyDirs="false">
- <fileset dir="${build.dir}/c++"/>
+ <copy todir="${dist.dir}/sbin">
+ <fileset dir="${basedir}/src/packages">
+ <include name="*.sh" />
+ </fileset>
</copy>
- <copy todir="${dist.dir}/" file="build.xml"/>
+ <copy todir="${dist.dir}/include/${name}" includeEmptyDirs="false" flatten="true">
+ <fileset dir="${build.dir}/c++">
+ <include name="**/include/${name}/*"/>
+ </fileset>
+ </copy>
+
+ <copy todir="${dist.dir}/lib" includeEmptyDirs="false" flatten="true">
+ <fileset dir="${build.dir}/c++">
+ <include name="**/lib/*"/>
+ </fileset>
+ </copy>
<chmod perm="ugo+x" type="file" parallel="false">
<fileset dir="${dist.dir}/bin"/>
+ <fileset dir="${dist.dir}/libexec"/>
+ <fileset dir="${dist.dir}/sbin"/>
</chmod>
</target>
@@ -1597,6 +1659,121 @@
</macro_tar>
</target>
+ <target name="rpm" depends="binary" description="Make rpm package">
+ <mkdir dir="${package.buildroot}/BUILD" />
+ <mkdir dir="${package.buildroot}/RPMS" />
+ <mkdir dir="${package.buildroot}/SRPMS" />
+ <mkdir dir="${package.buildroot}/SOURCES" />
+ <mkdir dir="${package.buildroot}/SPECS" />
+ <macro_tar param.destfile="${package.buildroot}/SOURCES/${final.name}-script.tar.gz">
+ <param.listofitems>
+ <tarfileset dir="${basedir}/src/packages/rpm/init.d" mode="755">
+ <include name="*" />
+ </tarfileset>
+ </param.listofitems>
+ </macro_tar>
+ <copy todir="${package.buildroot}/SOURCES">
+ <fileset dir="${build.dir}">
+ <include name="${final.name}-bin.tar.gz" />
+ </fileset>
+ </copy>
+ <copy file="${src.dir}/packages/rpm/spec/hadoop.spec" todir="${package.buildroot}/SPECS">
+ <filterchain>
+ <replacetokens>
+ <token key="final.name" value="${final.name}" />
+ <token key="version" value="${hadoop.version}" />
+ <token key="package.release" value="${package.release}" />
+ <token key="package.build.dir" value="${package.build.dir}" />
+ <token key="package.prefix" value="${package.prefix}" />
+ <token key="package.conf.dir" value="${package.conf.dir}" />
+ <token key="package.log.dir" value="${package.log.dir}" />
+ <token key="package.pid.dir" value="${package.pid.dir}" />
+ </replacetokens>
+ </filterchain>
+ </copy>
+ <rpm specFile="hadoop.spec" command="-bb --target ${os.arch}" topDir="${package.buildroot}" cleanBuildDir="true" failOnError="true"/>
+ <copy todir="${build.dir}/" flatten="true">
+ <fileset dir="${package.buildroot}/RPMS">
+ <include name="**/${name}*.rpm" />
+ </fileset>
+ <fileset dir="${package.buildroot}/SRPMS">
+ <include name="**/${name}*.rpm" />
+ </fileset>
+ </copy>
+ <delete dir="${package.buildroot}" quiet="true" verbose="false"/>
+ </target>
+
+ <target name="deb" depends="ant-tasks, binary" description="Make deb package">
+ <taskdef name="deb"
+ classname="org.vafer.jdeb.ant.DebAntTask">
+ <classpath refid="classpath" />
+ </taskdef>
+
+ <mkdir dir="${package.build.dir}/hadoop.control" />
+ <mkdir dir="${package.buildroot}/${package.prefix}/share/hadoop" />
+ <copy todir="${package.buildroot}/${package.prefix}">
+ <fileset dir="${build.dir}/${final.name}">
+ <include name="**" />
+ </fileset>
+ </copy>
+ <copy todir="${package.build.dir}/hadoop.control">
+ <fileset dir="${src.dir}/packages/deb/hadoop.control">
+ <exclude name="control" />
+ </fileset>
+ </copy>
+ <copy file="${src.dir}/packages/deb/hadoop.control/control" todir="${package.build.dir}/hadoop.control">
+ <filterchain>
+ <replacetokens>
+ <token key="final.name" value="${final.name}" />
+ <token key="version" value="${hadoop.version}" />
+ <token key="package.release" value="${package.release}" />
+ <token key="package.build.dir" value="${package.build.dir}" />
+ <token key="package.prefix" value="${package.prefix}" />
+ <token key="package.conf.dir" value="${package.conf.dir}" />
+ <token key="package.log.dir" value="${package.log.dir}" />
+ <token key="package.pid.dir" value="${package.pid.dir}" />
+ </replacetokens>
+ </filterchain>
+ </copy>
+ <deb destfile="${package.buildroot}/${name}_${hadoop.version}-${package.release}_${os.arch}.deb" control="${package.build.dir}/hadoop.control">
+ <tarfileset dir="${build.dir}/${final.name}" filemode="644" prefix="${package.prefix}">
+ <exclude name="bin/*" />
+ <exclude name="sbin/*" />
+ <exclude name="libexec/*" />
+ <exclude name="contrib/*/bin/*" />
+ <exclude name="src/contrib/ec2/bin/*" />
+ <exclude name="src/contrib/ec2/bin/image/*" />
+ <exclude name="etc" />
+ <exclude name="etc/**" />
+ <include name="**" />
+ </tarfileset>
+ <tarfileset dir="${build.dir}/${final.name}" filemode="755" prefix="${package.prefix}">
+ <include name="bin/*" />
+ <include name="sbin/*" />
+ <include name="libexec/*" />
+ <include name="contrib/*/bin/*" />
+ <include name="src/contrib/ec2/bin/*" />
+ <include name="src/contrib/ec2/bin/image/*" />
+ </tarfileset>
+ <tarfileset dir="${build.dir}/${final.name}/etc/hadoop" filemode="644" prefix="${package.conf.dir}">
+ <exclude name="core-site.xml" />
+ <exclude name="hdfs-site.xml" />
+ <exclude name="mapred-site.xml" />
+ <include name="**" />
+ </tarfileset>
+ <tarfileset dir="${basedir}/src/packages/deb/init.d" filemode="755" prefix="/etc/init.d">
+ <include name="**" />
+ </tarfileset>
+ </deb>
+
+ <copy todir="${build.dir}/" flatten="true">
+ <fileset dir="${package.buildroot}">
+ <include name="**/${name}*.deb" />
+ </fileset>
+ </copy>
+ <delete dir="${package.buildroot}" quiet="true" verbose="false"/>
+ </target>
+
<!-- ================================================================== -->
<!-- Perform audit activities for the release -->
<!-- ================================================================== -->
@@ -1615,6 +1792,7 @@
<delete dir="${build.dir}"/>
<delete dir="${docs.src}/build"/>
<delete dir="${src.docs.cn}/build"/>
+ <delete dir="${package.buildroot}"/>
<delete file="${basedir}/ivy/hadoop-core-pom.xml"/>
<delete file="${basedir}/ivy/hadoop-test-pom.xml"/>
<delete file="${basedir}/ivy/hadoop-examples-pom.xml"/>
@@ -1690,8 +1868,8 @@
</target>
<target name="package-librecordio" depends="compile-librecordio" if="librecordio">
- <mkdir dir="${dist.dir}/librecordio"/>
- <copy todir="${dist.dir}/librecordio">
+ <mkdir dir="${dist.dir}/lib/librecordio"/>
+ <copy todir="${dist.dir}/lib/librecordio">
<fileset dir="${build.librecordio}" casesensitive="yes" followsymlinks="false">
<exclude name="**/tests/**"/>
<exclude name="*.so"/>
@@ -1699,7 +1877,7 @@
</fileset>
</copy>
<chmod perm="ugo+x" type="file">
- <fileset dir="${dist.dir}/librecordio"/>
+ <fileset dir="${dist.dir}/lib/librecordio"/>
</chmod>
</target>
Modified: hadoop/common/branches/branch-0.20-security-204/conf/log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/conf/log4j.properties?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/conf/log4j.properties (original)
+++ hadoop/common/branches/branch-0.20-security-204/conf/log4j.properties Fri May 27 16:49:55 2011
@@ -115,6 +115,12 @@ log4j.logger.org.apache.hadoop.metrics2=
log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
#
+# Null Appender
+# Trap security logger on the hadoop client side
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
# Event Counter Appender
# Sends counts of logging messages at different severity levels to Hadoop Metrics.
#
Modified: hadoop/common/branches/branch-0.20-security-204/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/ivy.xml?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/ivy.xml (original)
+++ hadoop/common/branches/branch-0.20-security-204/ivy.xml Fri May 27 16:49:55 2011
@@ -70,7 +70,7 @@
<conf name="lucene" visibility="private"/>
<conf name="jdiff" visibility="private" extends="log4j,s3-client,jetty,server"/>
<conf name="checkstyle" visibility="private"/>
-
+ <conf name="packaging" visibility="private"/>
</configurations>
<publications>
@@ -291,5 +291,10 @@
rev="${aspectj.version}"
conf="common->default">
</dependency>
+ <dependency org="org.vafer"
+ name="jdeb"
+ rev="${jdeb.version}"
+ conf="common->master">
+ </dependency>
</dependencies>
</ivy-module>
Modified: hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties (original)
+++ hadoop/common/branches/branch-0.20-security-204/ivy/libraries.properties Fri May 27 16:49:55 2011
@@ -14,7 +14,7 @@
#It drives ivy and the generation of a maven POM
# This is the version of hadoop we are generating
-hadoop.version=0.20.203.0
+hadoop.version=0.20.204.0
hadoop-gpl-compression.version=0.1.0
#These are the versions of our dependencies (in alphabetical order)
@@ -57,6 +57,7 @@ jets3t.version=0.6.1
jetty.version=6.1.26
jetty-util.version=6.1.26
junit.version=4.5
+jdeb.version=0.8
jdiff.version=1.0.9
json.version=1.0
Added: hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/deployment_layout.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/deployment_layout.xml?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/deployment_layout.xml (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/deployment_layout.xml Fri May 27 16:49:55 2011
@@ -0,0 +1,147 @@
+<?xml version="1.0"?>
+<!--
+ Copyright 2002-2004 The Apache Software Foundation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+ "http://forrest.apache.org/dtd/document-v20.dtd">
+
+
+<document>
+
+ <header>
+ <title>
+ Hadoop Deployment Layout
+ </title>
+ </header>
+
+ <body>
+ <section>
+ <title> Introduction </title>
+ <p>
+ This document describes the standard deployment layout for Hadoop. With increased complexity and evolving Hadoop ecosystem, having standard deployment layout ensures better integration between Hadoop sub-projects. By making the installation process easier, we can lower the barrier to entry and increase Hadoop adoption.
+ </p>
+ </section>
+
+ <section>
+ <title> Packages </title>
+ <p>
+ We need to divide Hadoop up into packages that can be independently upgraded. The list of packages should include:
+ </p>
+ <ul>
+ <li>Hadoop Common - Common including the native code and required jar files.</li>
+ <li>HDFS Client - HDFS jars, scripts, and shared libraries.</li>
+ <li>HDFS Server - jsvc executable</li>
+ <li>Yarn Client - Yarn client jars and scripts</li>
+ <li>Yarn Server - Yarn server jars and scripts</li>
+ <li>MapReduce - MapReduce jars, scripts, and shared libraries</li>
+ <li>LZO - LZ0 codec from github.com/omally/hadoop-gpl-compression</li>
+ <li>Metrics - Plugins for Chukwa and Ganglia</li>
+ </ul>
+ <p>Packages from other teams will include:</p>
+ <ul>
+ <li>Pig</li>
+ <li>Hive</li>
+ <li>Oozie client</li>
+ <li>Oozie server</li>
+ <li>Howl client</li>
+ <li>Howl server</li>
+ </ul>
+ <p>These packages should be deployable with RPM on RedHat. We also need a package that depends on a version of each of these packages. In general, we can generate tarballs in the new deployment layout.</p>
+ <p>Note that some packages, like Pig, which are user facing, will have 2 versions installed in a given deployment. This will be accomplished by modifying the package name and the associated binaries to include the version number.</p>
+ <p>All of the following paths are based on a prefix directory that is the root of the installation. Our packages must support having multiple Hadoop stack installation on a computer at the same time. For RPMs, this means that the packages must be relocatable and honor the --prefix option.</p>
+ </section>
+
+
+ <section>
+ <title> Deployment </title>
+ <p>It is important to have a standard deployment that results from installing the packages regardless of the package manager. Here are the top level directories and a sample of what would be under each. Note that all of the packages are installed "flattened" into the prefix directory. For compatibility reasons, we should create "share/hadoop" that matches the old HADOOP_HOME and set the HADOOP_HOME variable to that.</p>
+ <source>
+ $PREFIX/ bin / hadoop
+ | | mapred
+ | | pig -> pig7
+ | | pig6
+ | + pig7
+ |
+ + etc / hadoop / core-site.xml
+ | | hdfs-site.xml
+ | + mapred-site.xml
+ |
+ + include / hadoop / Pipes.hh
+ | | + TemplateFactory.hh
+ | + hdfs.h
+ |
+ + lib / jni / hadoop-common / libhadoop.so.0.20.0
+ | |
+ | | libhdfs.so -> libhdfs.so.0.20.0
+ | + libhdfs.so.0.20.0
+ |
+ + libexec / task-controller
+ |
+ + man / man1 / hadoop.1
+ | | mapred.1
+ | | pig6.1
+ | + pig7.1
+ |
+ + share / hadoop-common
+ | | hadoop-hdfs
+ | | hadoop-mapreduce
+ | | pig6
+ | + pig7
+ |
+ + sbin / hdfs-admin
+ | | mapred-admin
+ |
+ + src / hadoop-common
+ | | hadoop-hdfs
+ | + hadoop-mapreduce
+ |
+ + var / lib / data-node
+ | + task-tracker
+ |
+ | log / hadoop-datanode
+ | + hadoop-tasktracker
+ |
+ + run / hadoop-datanode.pid
+ + hadoop-tasktracker.pid
+ </source>
+ <p>Note that we must continue to honor HADOOP_CONF_DIR to override the configuration location, but that it should default to $prefix/etc. User facing binaries and scripts go into bin. Configuration files go into etc with multiple configuration files having a directory. JNI shared libraries go into lib/jni/$tool since Java does not allow to specify the version of the library to load. Libraries that aren't loaded via System.loadLibrary are placed directly under lib. 64 bit versions of the libraries for platforms that support them should be placed in lib64. All of the architecture-independent pieces, including the jars for each tool will be placed in share/$tool. The default location for all the run time information will be in var. The storage will be in var/lib, the logs in var/log and the pid files in var/run.</p>
+ </section>
+
+ <section>
+ <title> Path Configurations </title>
+ <p>Path can be configured at compile phase or installation phase. For RPM, it takes advantage of the --relocate directive to allow path reconfiguration at install phase. For Debian package, path is configured at compile phase.
+ </p>
+ <p>Build phase parameter:</p>
+ <ul>
+ <li>package.prefix - Location of package prefix (Default /usr)</li>
+ <li>package.conf.dir - Location of configuration directory (Default /etc/hadoop)</li>
+ <li>package.log.dir - Location of log directory (Default /var/log/hadoop)</li>
+ <li>package.pid.dir - Location of pid directory (Default /var/run/hadoop)</li>
+ </ul>
+
+ <p>Install phase parameter:</p>
+ <source>
+ rpm -i hadoop-[version]-[rev].[arch].rpm \
+ --relocate /usr=/usr/local/hadoop \
+ --relocate /etc/hadoop=/usr/local/etc/hadoop \
+ --relocate /var/log/hadoop=/opt/logs/hadoop \
+ --relocate /var/run/hadoop=/opt/run/hadoop
+ </source>
+ </section>
+
+ </body>
+</document>
+
Modified: hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/site.xml?rev=1128390&r1=1128389&r2=1128390&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/common/branches/branch-0.20-security-204/src/docs/src/documentation/content/xdocs/site.xml Fri May 27 16:49:55 2011
@@ -59,7 +59,8 @@ See http://forrest.apache.org/docs/linki
<hdfs_libhdfs label="C API libhdfs" href="libhdfs.html" />
</docs>
- <docs label="Common">
+ <docs label="Common">
+ <deployment label="Deployment Layout" href="deployment_layout.html" />
<fsshell label="File System Shell" href="file_system_shell.html" />
<SLA label="Service Level Authorization" href="service_level_auth.html"/>
<native_lib label="Native Libraries" href="native_libraries.html" />
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/conffile
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/conffile?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/conffile (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/conffile Fri May 27 16:49:55 2011
@@ -0,0 +1 @@
+/usr/share/hadoop/conf
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/control
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/control?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/control (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/control Fri May 27 16:49:55 2011
@@ -0,0 +1,9 @@
+Package: hadoop
+Version: @version@
+Section: misc
+Priority: optional
+Provides: hadoop
+Architecture: all
+Maintainer: Apache Software Foundation <ge...@hadoop.apache.org>
+Description: The Apache Hadoop project develops open-source software for reliable, scalable, distributed computing.
+Distribution: development
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postinst
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postinst?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postinst (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postinst Fri May 27 16:49:55 2011
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bash /usr/sbin/update-hadoop-env.sh \
+ --prefix=/usr \
+ --bin-dir=/usr/bin \
+ --sbin-dir=/usr/sbin \
+ --conf-dir=/etc/hadoop \
+ --log-dir=/var/log/hadoop \
+ --pid-dir=/var/run/hadoop
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postrm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postrm?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postrm (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/postrm Fri May 27 16:49:55 2011
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+/usr/sbin/userdel hdfs 2> /dev/null >/dev/null
+/usr/sbin/userdel mapred 2> /dev/null >/dev/null
+/usr/sbin/groupdel hadoop 2> /dev/null >dev/null
+exit 0
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/preinst
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/preinst?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/preinst (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/preinst Fri May 27 16:49:55 2011
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop
+
+/usr/sbin/useradd --comment "Hadoop MapReduce" --shell /bin/bash -M -r --groups hadoop --home /var/lib/hadoop/mapred mapred 2> /dev/null || :
+/usr/sbin/useradd --comment "Hadoop HDFS" --shell /bin/bash -M -r --groups hadoop --home /var/lib/hadoop/hdfs hdfs 2> /dev/null || :
+
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/prerm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/prerm?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/prerm (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/hadoop.control/prerm Fri May 27 16:49:55 2011
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+/etc/init.d/hadoop-namenode stop 2>/dev/null >/dev/null
+/etc/init.d/hadoop-datanode stop 2>/dev/null >/dev/null
+/etc/init.d/hadoop-jobtracker stop 2>/dev/null >/dev/null
+/etc/init.d/hadoop-tasktracker stop 2>/dev/null >/dev/null
+bash /usr/sbin/update-hadoop-env.sh \
+ --prefix=/usr \
+ --bin-dir=/usr/bin \
+ --sbin-dir=/usr/sbin \
+ --conf-dir=/etc/hadoop \
+ --log-dir=/var/log/hadoop \
+ --pid-dir=/var/run/hadoop \
+ --uninstal
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-datanode
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-datanode?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-datanode (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-datanode Fri May 27 16:49:55 2011
@@ -0,0 +1,142 @@
+#! /bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### BEGIN INIT INFO
+# Provides: hadoop-datanode
+# Required-Start: $remote_fs $syslog
+# Required-Stop: $remote_fs $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop:
+# Short-Description: Apache Hadoop Name Node server
+### END INIT INFO
+
+set -e
+
+# /etc/init.d/hadoop-datanode: start and stop the Apache Hadoop Data Node daemon
+
+test -x /usr/bin/hadoop || exit 0
+( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
+
+umask 022
+
+if test -f /etc/default/hadoop-env.sh; then
+ . /etc/default/hadoop-env.sh
+fi
+
+. /lib/lsb/init-functions
+
+# Are we running from init?
+run_by_init() {
+ ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
+}
+
+check_for_no_start() {
+ # forget it if we're trying to start, and /etc/hadoop/hadoop-datanode_not_to_be_run exists
+ if [ -e /etc/hadoop/hadoop-datanode_not_to_be_run ]; then
+ if [ "$1" = log_end_msg ]; then
+ log_end_msg 0
+ fi
+ if ! run_by_init; then
+ log_action_msg "Apache Hadoop Data Node server not in use (/etc/hadoop/hadoop-datanode_not_to_be_run)"
+ fi
+ exit 0
+ fi
+}
+
+check_privsep_dir() {
+ # Create the PrivSep empty dir if necessary
+ if [ ! -d ${HADOOP_PID_DIR} ]; then
+ mkdir -p ${HADOOP_PID_DIR}
+ chown root:hadoop ${HADOOP_PID_DIR}
+ chmod 0775 ${HADOOP_PID_DIR}
+ fi
+}
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
+
+case "$1" in
+ start)
+ check_privsep_dir
+ check_for_no_start
+ log_daemon_msg "Starting Apache Hadoop Data Node server" "hadoop-datanode"
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ stop)
+ log_daemon_msg "Stopping Apache Hadoop Data Node server" "hadoop-datanode"
+ if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+
+ restart)
+ check_privsep_dir
+ log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
+ start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid
+ check_for_no_start log_end_msg
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+
+ try-restart)
+ check_privsep_dir
+ log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
+ set +e
+ start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid
+ RET="$?"
+ set -e
+ case $RET in
+ 0)
+ # old daemon stopped
+ check_for_no_start log_end_msg
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ 1)
+ # daemon not running
+ log_progress_msg "(not running)"
+ log_end_msg 0
+ ;;
+ *)
+ # failed to stop
+ log_progress_msg "(failed to stop)"
+ log_end_msg 1
+ ;;
+ esac
+ ;;
+
+ status)
+ status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid ${JAVA_HOME}/bin/java hadoop-datanode && exit 0 || exit $?
+ ;;
+
+ *)
+ log_action_msg "Usage: /etc/init.d/hadoop-datanode {start|stop|restart|try-restart|status}"
+ exit 1
+esac
+
+exit 0
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-jobtracker
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-jobtracker?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-jobtracker (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-jobtracker Fri May 27 16:49:55 2011
@@ -0,0 +1,142 @@
+#! /bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### BEGIN INIT INFO
+# Provides: hadoop-jobtracker
+# Required-Start: $remote_fs $syslog
+# Required-Stop: $remote_fs $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop:
+# Short-Description: Apache Hadoop Job Tracker server
+### END INIT INFO
+
+set -e
+
+# /etc/init.d/hadoop-jobtracker: start and stop the Apache Hadoop Job Tracker daemon
+
+test -x /usr/bin/hadoop || exit 0
+( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
+
+umask 022
+
+if test -f /etc/default/hadoop-env.sh; then
+ . /etc/default/hadoop-env.sh
+fi
+
+. /lib/lsb/init-functions
+
+# Are we running from init?
+run_by_init() {
+ ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
+}
+
+check_for_no_start() {
+ # forget it if we're trying to start, and /etc/hadoop/hadoop-jobtracker_not_to_be_run exists
+ if [ -e /etc/hadoop/hadoop-jobtracker_not_to_be_run ]; then
+ if [ "$1" = log_end_msg ]; then
+ log_end_msg 0
+ fi
+ if ! run_by_init; then
+ log_action_msg "Apache Hadoop Job Tracker server not in use (/etc/hadoop/hadoop-jobtracker_not_to_be_run)"
+ fi
+ exit 0
+ fi
+}
+
+check_privsep_dir() {
+ # Create the PrivSep empty dir if necessary
+ if [ ! -d ${HADOOP_PID_DIR} ]; then
+ mkdir -p ${HADOOP_PID_DIR}
+ chown root:hadoop ${HADOOP_PID_DIR}
+ chmod 0775 ${HADOOP_PID_DIR}
+ fi
+}
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
+
+case "$1" in
+ start)
+ check_privsep_dir
+ check_for_no_start
+ log_daemon_msg "Starting Apache Hadoop Job Tracker server" "hadoop-jobtracker"
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ stop)
+ log_daemon_msg "Stopping Apache Hadoop Job Tracker server" "hadoop-jobtracker"
+ if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+
+ restart)
+ check_privsep_dir
+ log_daemon_msg "Restarting Apache Hadoop Job Tracker server" "hadoop-jobtracker"
+ start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid
+ check_for_no_start log_end_msg
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+
+ try-restart)
+ check_privsep_dir
+ log_daemon_msg "Restarting Apache Hadoop Job Tracker server" "hadoop-jobtracker"
+ set +e
+ start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid
+ RET="$?"
+ set -e
+ case $RET in
+ 0)
+ # old daemon stopped
+ check_for_no_start log_end_msg
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ 1)
+ # daemon not running
+ log_progress_msg "(not running)"
+ log_end_msg 0
+ ;;
+ *)
+ # failed to stop
+ log_progress_msg "(failed to stop)"
+ log_end_msg 1
+ ;;
+ esac
+ ;;
+
+ status)
+ status_of_proc -p ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid ${JAVA_HOME}/bin/java hadoop-jobtracker && exit 0 || exit $?
+ ;;
+
+ *)
+ log_action_msg "Usage: /etc/init.d/hadoop-jobtracker {start|stop|restart|try-restart|status}"
+ exit 1
+esac
+
+exit 0
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-namenode
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-namenode?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-namenode (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-namenode Fri May 27 16:49:55 2011
@@ -0,0 +1,154 @@
+#! /bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### BEGIN INIT INFO
+# Provides: hadoop-namenode
+# Required-Start: $remote_fs $syslog
+# Required-Stop: $remote_fs $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop:
+# Short-Description: Apache Hadoop Name Node server
+### END INIT INFO
+
+set -e
+
+# /etc/init.d/hadoop-namenode: start and stop the Apache Hadoop Name Node daemon
+
+test -x /usr/bin/hadoop || exit 0
+( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
+
+umask 022
+
+if test -f /etc/default/hadoop-env.sh; then
+ . /etc/default/hadoop-env.sh
+fi
+
+. /lib/lsb/init-functions
+
+# Are we running from init?
+run_by_init() {
+ ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
+}
+
+check_for_no_start() {
+ # forget it if we're trying to start, and /etc/hadoop/hadoop-namenode_not_to_be_run exists
+ if [ -e /etc/hadoop/hadoop-namenode_not_to_be_run ]; then
+ if [ "$1" = log_end_msg ]; then
+ log_end_msg 0
+ fi
+ if ! run_by_init; then
+ log_action_msg "Apache Hadoop Name Node server not in use (/etc/hadoop/hadoop-namenode_not_to_be_run)"
+ fi
+ exit 0
+ fi
+}
+
+check_privsep_dir() {
+ # Create the PrivSep empty dir if necessary
+ if [ ! -d ${HADOOP_PID_DIR} ]; then
+ mkdir -p ${HADOOP_PID_DIR}
+ chown root:hadoop ${HADOOP_PID_DIR}
+ chmod 0775 ${HADOOP_PID_DIR}
+ fi
+}
+
+format() {
+ sudo -u hdfs ${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format
+}
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
+
+case "$1" in
+ start)
+ check_privsep_dir
+ check_for_no_start
+ log_daemon_msg "Starting Apache Hadoop Name Node server" "hadoop-namenode"
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ stop)
+ log_daemon_msg "Stopping Apache Hadoop Name Node server" "hadoop-namenode"
+ if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ format)
+ log_daemon_msg "Formatting Apache Hadoop Name Node" "hadoop-namenode"
+ format
+ if [ $? -eq 0 ]; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ restart)
+ check_privsep_dir
+ log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode"
+ start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid
+ check_for_no_start log_end_msg
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+
+ try-restart)
+ check_privsep_dir
+ log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode"
+ set +e
+ start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid
+ RET="$?"
+ set -e
+ case $RET in
+ 0)
+ # old daemon stopped
+ check_for_no_start log_end_msg
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ 1)
+ # daemon not running
+ log_progress_msg "(not running)"
+ log_end_msg 0
+ ;;
+ *)
+ # failed to stop
+ log_progress_msg "(failed to stop)"
+ log_end_msg 1
+ ;;
+ esac
+ ;;
+
+ status)
+ status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid ${JAVA_HOME}/bin/java hadoop-namenode && exit 0 || exit $?
+ ;;
+
+ *)
+ log_action_msg "Usage: /etc/init.d/hadoop-namenode {start|stop|restart|try-restart|status}"
+ exit 1
+esac
+
+exit 0
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-tasktracker
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-tasktracker?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-tasktracker (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/deb/init.d/hadoop-tasktracker Fri May 27 16:49:55 2011
@@ -0,0 +1,142 @@
+#! /bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### BEGIN INIT INFO
+# Provides: hadoop-tasktracker
+# Required-Start: $remote_fs $syslog
+# Required-Stop: $remote_fs $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop:
+# Short-Description: Apache Hadoop Task Tracker server
+### END INIT INFO
+
+set -e
+
+# /etc/init.d/hadoop-tasktracker: start and stop the Apache Hadoop Task Tracker daemon
+
+test -x /usr/bin/hadoop || exit 0
+( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
+
+umask 022
+
+if test -f /etc/default/hadoop-env.sh; then
+ . /etc/default/hadoop-env.sh
+fi
+
+. /lib/lsb/init-functions
+
+# Are we running from init?
+run_by_init() {
+ ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
+}
+
+check_for_no_start() {
+ # forget it if we're trying to start, and /etc/hadoop/hadoop-tasktracker_not_to_be_run exists
+ if [ -e /etc/hadoop/hadoop-tasktracker_not_to_be_run ]; then
+ if [ "$1" = log_end_msg ]; then
+ log_end_msg 0
+ fi
+ if ! run_by_init; then
+ log_action_msg "Apache Hadoop Task Tracker server not in use (/etc/hadoop/hadoop-tasktracker_not_to_be_run)"
+ fi
+ exit 0
+ fi
+}
+
+check_privsep_dir() {
+ # Create the PrivSep empty dir if necessary
+ if [ ! -d ${HADOOP_PID_DIR} ]; then
+ mkdir -p ${HADOOP_PID_DIR}
+ chown root:hadoop ${HADOOP_PID_DIR}
+ chmod 0775 ${HADOOP_PID_DIR}
+ fi
+}
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
+
+case "$1" in
+ start)
+ check_privsep_dir
+ check_for_no_start
+ log_daemon_msg "Starting Apache Hadoop Task Tracker server" "hadoop-tasktracker"
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ stop)
+ log_daemon_msg "Stopping Apache Hadoop Task Tracker server" "hadoop-tasktracker"
+ if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+
+ restart)
+ check_privsep_dir
+ log_daemon_msg "Restarting Apache Hadoop Task Tracker server" "hadoop-tasktracker"
+ start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid
+ check_for_no_start log_end_msg
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+
+ try-restart)
+ check_privsep_dir
+ log_daemon_msg "Restarting Apache Hadoop Task Tracker server" "hadoop-tasktracker"
+ set +e
+ start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid
+ RET="$?"
+ set -e
+ case $RET in
+ 0)
+ # old daemon stopped
+ check_for_no_start log_end_msg
+ if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then
+ log_end_msg 0
+ else
+ log_end_msg 1
+ fi
+ ;;
+ 1)
+ # daemon not running
+ log_progress_msg "(not running)"
+ log_end_msg 0
+ ;;
+ *)
+ # failed to stop
+ log_progress_msg "(failed to stop)"
+ log_end_msg 1
+ ;;
+ esac
+ ;;
+
+ status)
+ status_of_proc -p ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid ${JAVA_HOME}/bin/java hadoop-tasktracker && exit 0 || exit $?
+ ;;
+
+ *)
+ log_action_msg "Usage: /etc/init.d/hadoop-tasktracker {start|stop|restart|try-restart|status}"
+ exit 1
+esac
+
+exit 0
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-create-user.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-create-user.sh?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-create-user.sh (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-create-user.sh Fri May 27 16:49:55 2011
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
+script="$(basename -- "$this")"
+this="$bin/$script"
+
+if [ "$HADOOP_HOME" != "" ]; then
+ echo "Warning: \$HADOOP_HOME is deprecated."
+ echo
+fi
+
+. "$bin"/../libexec/hadoop-config.sh
+
+usage() {
+ echo "
+usage: $0 <parameters>
+ Require parameter:
+ -u <username> Create user on HDFS
+ Optional parameters:
+ -h Display this message
+ "
+ exit 1
+}
+
+if [ $# != 2 ] ; then
+ usage
+ exit 1
+fi
+
+while getopts "hu:" OPTION
+do
+ case $OPTION in
+ u)
+ SETUP_USER=$2; shift 2
+ ;;
+ h)
+ usage
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+export HADOOP_PREFIX
+export HADOOP_CONF_DIR
+export JAVA_HOME
+export SETUP_USER=${SETUP_USER}
+export SETUP_PATH=/user/${SETUP_USER}
+
+su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir ${SETUP_PATH}' hdfs
+su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown ${SETUP_USER}:${SETUP_USER} ${SETUP_PATH}' hdfs
+
+if [ "$?" == "0" ]; then
+ echo "User directory has been setup: ${SETUP_PATH}"
+fi
Added: hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-setup-conf.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-setup-conf.sh?rev=1128390&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-setup-conf.sh (added)
+++ hadoop/common/branches/branch-0.20-security-204/src/packages/hadoop-setup-conf.sh Fri May 27 16:49:55 2011
@@ -0,0 +1,279 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+if [ "$HADOOP_HOME" != "" ]; then
+ echo "Warning: \$HADOOP_HOME is deprecated."
+ echo
+fi
+
+. "$bin"/../libexec/hadoop-config.sh
+
+usage() {
+ echo "
+usage: $0 <parameters>
+
+ Optional parameters:
+ --auto Setup path and configuration automatically
+ --default Setup configuration as default
+ --conf-dir=/etc/hadoop Set configuration directory
+ --datanode-dir=/var/lib/hadoop/hdfs/datanode Set datanode directory
+ -h Display this message
+ --jobtracker-url=hostname:9001 Set jobtracker url
+ --log-dir=/var/log/hadoop Set log directory
+ --pid-dir=/var/run/hadoop Set pid directory
+ --hdfs-dir=/var/lib/hadoop/hdfs Set hdfs directory
+ --mapred-dir=/var/lib/hadoop/mapred Set mapreduce directory
+ --namenode-dir=/var/lib/hadoop/hdfs/namenode Set namenode directory
+ --namenode-url=hdfs://hostname:9000/ Set namenode url
+ --replication=3 Set replication factor
+ --taskscheduler=org.apache.hadoop.mapred.JobQueueTaskScheduler Set task scheduler
+ "
+ exit 1
+}
+
+template_generator() {
+ REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
+ cat $1 |
+ while read line ; do
+ while [[ "$line" =~ $REGEX ]] ; do
+ LHS=${BASH_REMATCH[1]}
+ RHS="$(eval echo "\"$LHS\"")"
+ line=${line//$LHS/$RHS}
+ done
+ echo $line >> $2
+ done
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'auto' \
+ -l 'conf-dir:' \
+ -l 'default' \
+ -l 'hdfs-dir:' \
+ -l 'namenode-dir:' \
+ -l 'datanode-dir:' \
+ -l 'mapred-dir:' \
+ -l 'namenode-url:' \
+ -l 'jobtracker-url:' \
+ -l 'log-dir:' \
+ -l 'pid-dir:' \
+ -l 'replication:' \
+ -l 'taskscheduler:' \
+ -o 'h' \
+ -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+# Make sure the HADOOP_LOG_DIR is not picked up from user environment.
+unset HADOOP_LOG_DIR
+
+eval set -- "${OPTS}"
+while true ; do
+ case "$1" in
+ --auto)
+ AUTOSETUP=1
+ AUTOMATED=1
+ shift
+ ;;
+ --conf-dir)
+ HADOOP_CONF_DIR=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --default)
+ AUTOMATED=1; shift
+ ;;
+ -h)
+ usage
+ ;;
+ --hdfs-dir)
+ HADOOP_HDFS_DIR=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --namenode-dir)
+ HADOOP_NN_DIR=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --datanode-dir)
+ HADOOP_DN_DIR=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapred-dir)
+ HADOOP_MAPRED_DIR=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --namenode-url)
+ HADOOP_NN_HOST=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --jobtracker-url)
+ HADOOP_JT_HOST=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --log-dir)
+ HADOOP_LOG_DIR=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --pid-dir)
+ HADOOP_PID_DIR=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --replication)
+ HADOOP_REPLICATION=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --taskscheduler)
+ HADOOP_TASK_SCHEDULER=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+AUTOSETUP=${AUTOSETUP:-1}
+JAVA_HOME=${JAVA_HOME:-/usr/java/default}
+HADOOP_NN_HOST=${HADOOP_NN_HOST:-hdfs://`hostname`:9000/}
+HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
+HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
+HADOOP_JT_HOST=${HADOOP_JT_HOST:-`hostname`:9001}
+HADOOP_HDFS_DIR=${HADOOP_HDFS_DIR:-/var/lib/hadoop/hdfs}
+HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
+HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-/var/log/hadoop}
+HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/log/hadoop}
+HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
+HADOOP_REPLICATION=${HADOOP_RELICATION:-3}
+HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
+
+if [ "${AUTOMATED}" != "1" ]; then
+ echo "Setup Hadoop Configuration"
+ echo
+ echo -n "Where would you like to put config directory? (${HADOOP_CONF_DIR}) "
+ read USER_HADOOP_CONF_DIR
+ echo -n "Where would you like to put log directory? (${HADOOP_LOG_DIR}) "
+ read USER_HADOOP_LOG_DIR
+ echo -n "Where would you like to put pid directory? (${HADOOP_PID_DIR}) "
+ read USER_HADOOP_PID_DIR
+ echo -n "What is the url of the namenode? (${HADOOP_NN_HOST}) "
+ read USER_HADOOP_NN_HOST
+ echo -n "Where would you like to put namenode data directory? (${HADOOP_NN_DIR}) "
+ read USER_HADOOP_NN_DIR
+ echo -n "Where would you like to put datanode data directory? (${HADOOP_DN_DIR}) "
+ read USER_HADOOP_DN_DIR
+ echo -n "What is the url of the jobtracker? (${HADOOP_JT_HOST}) "
+ read USER_HADOOP_JT_HOST
+ echo -n "Where would you like to put jobtracker/tasktracker data directory? (${HADOOP_MAPRED_DIR}) "
+ read USER_HADOOP_MAPRED_DIR
+ echo -n "Where is JAVA_HOME directory? (${JAVA_HOME}) "
+ read USER_JAVA_HOME
+ echo -n "Would you like to create directories/copy conf files to localhost? (Y/n) "
+ read USER_AUTOSETUP
+ echo
+ JAVA_HOME=${USER_USER_JAVA_HOME:-$JAVA_HOME}
+ HADOOP_NN_HOST=${USER_HADOOP_NN_HOST:-$HADOOP_NN_HOST}
+ HADOOP_NN_DIR=${USER_HADOOP_NN_DIR:-$HADOOP_NN_DIR}
+ HADOOP_DN_DIR=${USER_HADOOP_DN_DIR:-$HADOOP_DN_DIR}
+ HADOOP_JT_HOST=${USER_HADOOP_JT_HOST:-$HADOOP_JT_HOST}
+ HADOOP_HDFS_DIR=${USER_HADOOP_HDFS_DIR:-$HADOOP_HDFS_DIR}
+ HADOOP_MAPRED_DIR=${USER_HADOOP_MAPRED_DIR:-$HADOOP_MAPRED_DIR}
+ HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
+ HADOOP_LOG_DIR=${USER_HADOOP_LOG_DIR:-$HADOOP_LOG_DIR}
+ HADOOP_PID_DIR=${USER_HADOOP_PID_DIR:-$HADOOP_PID_DIR}
+ HADOOP_CONF_DIR=${USER_HADOOP_CONF_DIR:-$HADOOP_CONF_DIR}
+ AUTOSETUP=${USER_AUTOSETUP:-y}
+ echo "Review your choices:"
+ echo
+ echo "Config directory : ${HADOOP_CONF_DIR}"
+ echo "Log directory : ${HADOOP_LOG_DIR}"
+ echo "PID directory : ${HADOOP_PID_DIR}"
+ echo "Namenode url : ${HADOOP_NN_HOST}"
+ echo "Namenode directory : ${HADOOP_NN_DIR}"
+ echo "Datanode directory : ${HADOOP_DN_DIR}"
+ echo "Jobtracker url : ${HADOOP_JT_HOST}"
+ echo "Mapreduce directory : ${HADOOP_MAPRED_DIR}"
+ echo "Task scheduler : ${HADOOP_TASK_SCHEDULER}"
+ echo "JAVA_HOME directory : ${JAVA_HOME}"
+ echo "Create dirs/copy conf files : ${AUTOSETUP}"
+ echo
+ echo -n "Proceed with generate configuration? (y/N) "
+ read CONFIRM
+ if [ "${CONFIRM}" != "y" ]; then
+ echo "User aborted setup, exiting..."
+ exit 1
+ fi
+fi
+
+rm -f core-site.xml >/dev/null
+rm -f hdfs-site.xml >/dev/null
+rm -f mapred-site.xml >/dev/null
+rm -f hadoop-env.sh >/dev/null
+
+template_generator ${HADOOP_HOME}/templates/conf/core-site.xml core-site.xml
+template_generator ${HADOOP_HOME}/templates/conf/hdfs-site.xml hdfs-site.xml
+template_generator ${HADOOP_HOME}/templates/conf/mapred-site.xml mapred-site.xml
+template_generator ${HADOOP_HOME}/templates/conf/hadoop-env.sh hadoop-env.sh
+
+chown root:hadoop hadoop-env.sh
+chmod 755 hadoop-env.sh
+
+if [ "${AUTOSETUP}" == "1" -o "${AUTOSETUP}" == "y" ]; then
+ mkdir -p ${HADOOP_HDFS_DIR}
+ mkdir -p ${HADOOP_NN_DIR}
+ mkdir -p ${HADOOP_DN_DIR}
+ mkdir -p ${HADOOP_MAPRED_DIR}
+ mkdir -p ${HADOOP_CONF_DIR}
+ mkdir -p ${HADOOP_LOG_DIR}
+ mkdir -p ${HADOOP_LOG_DIR}/hdfs
+ mkdir -p ${HADOOP_LOG_DIR}/mapred
+ mkdir -p ${HADOOP_PID_DIR}
+ chown hdfs:hadoop ${HADOOP_HDFS_DIR}
+ chown hdfs:hadoop ${HADOOP_NN_DIR}
+ chown hdfs:hadoop ${HADOOP_DN_DIR}
+ chown mapred:hadoop ${HADOOP_MAPRED_DIR}
+ chown root:hadoop ${HADOOP_LOG_DIR}
+ chmod 775 ${HADOOP_LOG_DIR}
+ chmod 775 ${HADOOP_PID_DIR}
+ chown hdfs:hadoop ${HADOOP_LOG_DIR}/hdfs
+ chown mapred:hadoop ${HADOOP_LOG_DIR}/mapred
+ cp -f *.xml ${HADOOP_CONF_DIR}
+ cp -f hadoop-env.sh ${HADOOP_CONF_DIR}
+ echo "Configuration setup is completed."
+ if [[ "$HADOOP_NN_HOST" =~ "`hostname`" ]]; then
+ echo "Proceed to run hadoop-setup-hdfs.sh on namenode."
+ fi
+else
+ echo
+ echo "Configuration file has been generated, please copy:"
+ echo
+ echo "core-site.xml"
+ echo "hdfs-site.xml"
+ echo "mapred-site.xml"
+ echo "hadoop-env.sh"
+ echo
+ echo " to ${HADOOP_CONF_DIR} on all nodes, and proceed to run hadoop-setup-hdfs.sh on namenode."
+fi