You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2011/11/01 23:21:53 UTC
svn commit: r1196329 - in
/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src: common/hadoop/
common/hadoop/conf.pseudo/ deb/hadoop/ rpm/hadoop/SOURCES/ rpm/hadoop/SPECS/
Author: rvs
Date: Tue Nov 1 22:21:52 2011
New Revision: 1196329
URL: http://svn.apache.org/viewvc?rev=1196329&view=rev
Log:
updating hadoop to the state of trunk
Added:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.nofiles.conf
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.postinst
- copied, changed from r1196317, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.daemon.postinst.tpl
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.install
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.postinst
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml Tue Nov 1 22:21:52 2011
@@ -31,22 +31,4 @@
<name>dfs.name.dir</name>
<value>/var/lib/hadoop/cache/hadoop/dfs/name</value>
</property>
-
- <!-- Enable Hue Plugins -->
- <property>
- <name>dfs.namenode.plugins</name>
- <value>org.apache.hadoop.thriftfs.NamenodePlugin</value>
- <description>Comma-separated list of namenode plug-ins to be activated.
- </description>
- </property>
- <property>
- <name>dfs.datanode.plugins</name>
- <value>org.apache.hadoop.thriftfs.DatanodePlugin</value>
- <description>Comma-separated list of datanode plug-ins to be activated.
- </description>
- </property>
- <property>
- <name>dfs.thrift.address</name>
- <value>0.0.0.0:10090</value>
- </property>
</configuration>
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml Tue Nov 1 22:21:52 2011
@@ -22,16 +22,4 @@
<name>mapred.job.tracker</name>
<value>localhost:8021</value>
</property>
-
- <!-- Enable Hue plugins -->
- <property>
- <name>mapred.jobtracker.plugins</name>
- <value>org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin</value>
- <description>Comma-separated list of jobtracker plug-ins to be activated.
- </description>
- </property>
- <property>
- <name>jobtracker.thrift.address</name>
- <value>0.0.0.0:9290</value>
- </property>
</configuration>
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default Tue Nov 1 22:21:52 2011
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+export HADOOP_HOME_WARN_SUPPRESS=true
export HADOOP_HOME=/usr/lib/hadoop
export HADOOP_NAMENODE_USER=hdfs
export HADOOP_SECONDARYNAMENODE_USER=hdfs
@@ -21,3 +21,5 @@ export HADOOP_JOBTRACKER_USER=mapred
export HADOOP_TASKTRACKER_USER=mapred
export HADOOP_IDENT_STRING=hadoop
export HADOOP_PID_DIR=/var/run/hadoop
+export HADOOP_LOG_DIR=/var/log/hadoop
+export HADOOP_CONF_DIR=/etc/hadoop/conf
Added: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.nofiles.conf
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.nofiles.conf?rev=1196329&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.nofiles.conf (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.nofiles.conf Tue Nov 1 22:21:52 2011
@@ -0,0 +1,2 @@
+hdfs - nofile 32768
+mapred - nofile 32768
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control Tue Nov 1 22:21:52 2011
@@ -16,7 +16,7 @@
Source: hadoop
Section: misc
Priority: extra
-Maintainer: Todd Lipcon <to...@cloudera.com>
+Maintainer: Bigtop <bi...@incubator.apache.org>
Build-Depends: debhelper (>= 6), ant, ant-optional, liblzo2-dev, python, libzip-dev,automake, autoconf (>= 2.61), sharutils, g++ (>= 4), git-core, libfuse-dev, libssl-dev
Standards-Version: 3.8.0
Homepage: http://hadoop.apache.org/core/
@@ -24,7 +24,7 @@ Homepage: http://hadoop.apache.org/core/
Package: hadoop
Provides: hadoop
Architecture: all
-Depends: ${shlibs:Depends}, ${misc:Depends}, adduser
+Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, bigtop-utils
Recommends: hadoop-native
Description: A software platform for processing vast amounts of data
Hadoop is a software platform that lets one easily write and
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.install
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.install?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.install (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.install Tue Nov 1 22:21:52 2011
@@ -1 +1,2 @@
/usr/lib/hadoop/sbin
+/usr/lib/hadoop/bin/jsvc*
Copied: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.postinst (from r1196317, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default)
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.postinst?p2=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.postinst&p1=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default&r1=1196317&r2=1196329&rev=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-sbin.postinst Tue Nov 1 22:21:52 2011
@@ -1,3 +1,5 @@
+#!/bin/sh
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,11 +15,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_NAMENODE_USER=hdfs
-export HADOOP_SECONDARYNAMENODE_USER=hdfs
-export HADOOP_DATANODE_USER=hdfs
-export HADOOP_JOBTRACKER_USER=mapred
-export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hadoop
-export HADOOP_PID_DIR=/var/run/hadoop
+# postinst script for hadoop
+
+set -e
+
+case "$1" in
+ configure)
+ find /usr/lib/hadoop -name task-controller -exec chown root:mapred {} \;
+ find /usr/lib/hadoop -name task-controller -exec chmod 4750 {} \;
+ ;;
+
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
Added: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.daemon.postinst.tpl
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.daemon.postinst.tpl?rev=1196329&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.daemon.postinst.tpl (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.daemon.postinst.tpl Tue Nov 1 22:21:52 2011
@@ -0,0 +1,10 @@
+#!/bin/sh
+set -e
+if [ -x "/etc/init.d/hadoop-@HADOOP_DAEMON@" ]; then
+ update-rc.d hadoop-@HADOOP_DAEMON@ defaults >/dev/null
+ if [ -x "`which invoke-rc.d 2>/dev/null`" ]; then
+ invoke-rc.d hadoop-@HADOOP_DAEMON@ start || :
+ else
+ /etc/init.d/hadoop-@HADOOP_DAEMON@ start || :
+ fi
+fi
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install Tue Nov 1 22:21:52 2011
@@ -2,4 +2,5 @@
/usr/lib/hadoop
/etc/hadoop/conf.empty
/etc/default/hadoop
+/etc/security/limits.d/hadoop.nofiles.conf
/usr/share/doc/hadoop
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.postinst
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.postinst?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.postinst (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.postinst Tue Nov 1 22:21:52 2011
@@ -34,7 +34,12 @@ case "$1" in
# We don't want to do this recursively since we may be reinstalling, in which case
# users have their own cache/<username> directories which shouldn't be stolen
chown root:hadoop /var/lib/hadoop/ /var/lib/hadoop/cache/ /var/lib/hadoop/cache/hadoop/
- chmod 1777 /var/lib/hadoop/cache/
+ chmod 1777 /var/lib/hadoop/cache/
+ chmod g+w /var/lib/hadoop/cache/hadoop/
+ mkdir -p /var/log/hadoop || :
+ touch /var/log/hadoop/SecurityAuth.audit
+ chgrp hadoop /var/log/hadoop/SecurityAuth.audit
+ chmod g+w /var/log/hadoop/SecurityAuth.audit
update-alternatives --install /etc/hadoop/conf hadoop-conf /etc/hadoop/conf.empty 10
;;
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules Tue Nov 1 22:21:52 2011
@@ -39,8 +39,8 @@ endif
build: build-stamp
build-stamp:
dh_testdir
- env HADOOP_VERSION=${hadoop_version} \
- sh debian/do-component-build -Divy.home=`pwd`/debian/.ivy
+ env HADOOP_VERSION=${hadoop_version} HADOOP_ARCH=${native_dir} \
+ bash debian/do-component-build -Divy.home=`pwd`/debian/.ivy
touch $@
clean:
@@ -82,6 +82,8 @@ namenode secondarynamenode jobtracker ta
sed -e "s|@HADOOP_DAEMON@|$@|" \
-e "s|@HADOOP_MAJOR_VERSION@|$hadoop_version|" \
-e "s|@DAEMON_USER@|$($@_user)|" $< > debian/hadoop-$@.init
+ # FIXME: workaround for BIGTOP-105
+ sed -e "s|@HADOOP_DAEMON@|$@|" < debian/hadoop.daemon.postinst.tpl > debian/hadoop-$@.postinst
install-indep: namenode secondarynamenode jobtracker tasktracker datanode
mkdir -p debian/tmp/etc/default
@@ -95,10 +97,18 @@ install-indep: namenode secondarynamenod
rm -Rf debian/hadoop/usr/lib/hadoop/bin/fuse_dfs
rm -Rf debian/hadoop/usr/bin/hadoop-fuse-dfs
rm -Rf debian/hadoop/usr/lib/hadoop/contrib/fuse-dfs
+ rm -Rf debian/hadoop/usr/lib/hadoop/hdfs/contrib/fuse-dfs
+
+ # FIXME: The following is a workaround for BIGTOP-139
+ rm -Rf debian/hadoop/usr/lib/hadoop/bin/jsvc
+ rm -Rf debian/hadoop/usr/lib/hadoop/hdfs/bin/jsvc
+ rm -Rf debian/hadoop/usr/lib/hadoop/bin/task-controller
install-arch:
mkdir -p debian/tmp/etc/default
cp debian/hadoop-fuse.default debian/tmp/etc/default/hadoop-fuse
+ mkdir -p debian/tmp/etc/security/limits.d
+ cp debian/hadoop.nofiles.conf debian/tmp/etc/security/limits.d
dh_install --sourcedir=debian/tmp -s
# Must not depend on anything. This is to be called by
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl Tue Nov 1 22:21:52 2011
@@ -48,6 +48,19 @@ install -d -m 0775 -o root -g hadoop /va
. /etc/default/hadoop
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+ . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+. $HADOOP_HOME/bin/hadoop-config.sh
+
+# FIXME: this needs to be removed once hadoop-config.sh stop clobbering HADOOP_HOME
+. /etc/default/hadoop
+
+
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
DAEMON_SCRIPT=$HADOOP_HOME/bin/hadoop-daemon.sh
NAME=hadoop-@HADOOP_DAEMON@
@@ -137,13 +150,27 @@ hadoop_stop_pidfile() {
}
start() {
- $HADOOP_HOME/bin/hadoop-daemon.sh start @HADOOP_DAEMON@ $DAEMON_FLAGS
-
+ TARGET_USER_NAME="HADOOP_`echo @HADOOP_DAEMON@ | tr a-z A-Z`_USER"
+ TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
+
if [ "@HADOOP_DAEMON@" = "datanode" ]; then
- # Some processes are slow to start
- sleep $SLEEP_TIME
+ # The following needs to be removed once HDFS-1943 gets finally put to rest.
+ # The logic of this ugly hack is this: IFF we do NOT have jsvc installed it is
+ # guaranteed that we can NOT be running in a secure mode and thus we need to
+ # workaround HDFS-1943 (start as non-root). As soon as jsvc gets installed
+ # we are assuming a secure installation and starting a data node as root.
+ # This leaves 2 corner cases:
+ # 1. HADOOP_DATANODE_USER being set to root
+ # 2. jsvc is installed but Hadoop is configures to run in an unsecure mode
+ # Both will currently fail
+ if [ -f $HADOOP_HOME/libexec/jsvc.amd64 -o -f $HADOOP_HOME/libexec/jsvc.i386 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
+ TARGET_USER=root
+ fi
fi
+ su -s /bin/bash $TARGET_USER -c "$HADOOP_HOME/bin/hadoop-daemon.sh start @HADOOP_DAEMON@ $DAEMON_FLAGS"
+ # Some processes are slow to start
+ sleep $SLEEP_TIME
}
stop() {
$HADOOP_HOME/bin/hadoop-daemon.sh stop @HADOOP_DAEMON@
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl Tue Nov 1 22:21:52 2011
@@ -20,12 +20,19 @@
# chkconfig: 2345 85 15
# description: Hadoop @HADOOP_DAEMON@
-source /etc/rc.d/init.d/functions
-source /etc/default/hadoop
+. /etc/rc.d/init.d/functions
+. /etc/default/hadoop
-source @HADOOP_COMMON_ROOT@/bin/hadoop-config.sh
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+ . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+. @HADOOP_COMMON_ROOT@/bin/hadoop-config.sh
# FIXME: this needs to be removed once hadoop-config.sh stop clobbering HADOOP_HOME
-source /etc/default/hadoop
+. /etc/default/hadoop
RETVAL_SUCCESS=0
@@ -43,15 +50,30 @@ SLEEP_TIME=5
start() {
echo -n $"Starting $desc (hadoop-@HADOOP_DAEMON@): "
- daemon @HADOOP_COMMON_ROOT@/bin/hadoop-daemon.sh --config "@HADOOP_CONF_DIR@" start @HADOOP_DAEMON@ $DAEMON_FLAGS
- RETVAL=$?
+
+ TARGET_USER_NAME="HADOOP_`echo @HADOOP_DAEMON@ | tr a-z A-Z`_USER"
+ TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
if [ "@HADOOP_DAEMON@" = "datanode" ]; then
- # Some processes are slow to start
- sleep $SLEEP_TIME
- checkstatus
- RETVAL=$?
+ # The following needs to be removed once HDFS-1943 gets finally put to rest.
+ # The logic of this ugly hack is this: IFF we do NOT have jsvc installed it is
+ # guaranteed that we can NOT be running in a secure mode and thus we need to
+ # workaround HDFS-1943 (start as non-root). As soon as jsvc gets installed
+ # we are assuming a secure installation and starting a data node as root.
+ # This leaves 2 corner cases:
+ # 1. HADOOP_DATANODE_USER being set to root
+ # 2. jsvc is installed but Hadoop is configures to run in an unsecure mode
+ # Both will currently fail
+ if [ -f $HADOOP_HOME/libexec/jsvc.amd64 -o -f $HADOOP_HOME/libexec/jsvc.i386 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
+ TARGET_USER=root
+ fi
fi
+ su -s /bin/bash $TARGET_USER -c "@HADOOP_COMMON_ROOT@/bin/hadoop-daemon.sh --config '@HADOOP_CONF_DIR@' start @HADOOP_DAEMON@ $DAEMON_FLAGS"
+
+ # Some processes are slow to start
+ sleep $SLEEP_TIME
+ checkstatus
+ RETVAL=$?
[ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE
return $RETVAL
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse Tue Nov 1 22:21:52 2011
@@ -31,13 +31,20 @@
# Short-Description: Hadoop @HADOOP_DAEMON@
### END INIT INFO
-source /lib/lsb/init-functions
-source /etc/default/hadoop
+. /lib/lsb/init-functions
+. /etc/default/hadoop
-source @HADOOP_COMMON_ROOT@/bin/hadoop-config.sh
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+ . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+. @HADOOP_COMMON_ROOT@/bin/hadoop-config.sh
# FIXME: this needs to be removed once hadoop-config.sh stop clobbering HADOOP_HOME
-source /etc/default/hadoop
+. /etc/default/hadoop
RETVAL_SUCCESS=0
@@ -45,6 +52,7 @@ STATUS_RUNNING=0
STATUS_DEAD=1
STATUS_DEAD_AND_LOCK=2
STATUS_NOT_RUNNING=3
+STATUS_OTHER_ERROR=102
ERROR_PROGRAM_NOT_INSTALLED=5
@@ -58,22 +66,36 @@ HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/ru
PIDFILE="$HADOOP_PID_DIR/hadoop-$HADOOP_IDENT_STRING-@HADOOP_DAEMON@.pid"
LOCKFILE="/var/lock/subsys/hadoop-@HADOOP_DAEMON@"
desc="Hadoop @HADOOP_DAEMON@ daemon"
-USER="@HADOOP_DAEMON_USER@"
SLEEP_TIME=5
start() {
[ -x $exec ] || exit $ERROR_PROGRAM_NOT_INSTALLED
[ -f $config ] || exit $ERROR_PROGRAM_NOT_CONFIGURED
log_success_msg "Starting $desc (hadoop-@HADOOP_DAEMON@): "
- start_daemon -u $USER $EXEC_PATH --config "$CONFIG_PATH" start @HADOOP_DAEMON@ $DAEMON_FLAGS
- RETVAL=$?
+
+ TARGET_USER_NAME="HADOOP_`echo @HADOOP_DAEMON@ | tr a-z A-Z`_USER"
+ TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
if [ "@HADOOP_DAEMON@" = "datanode" ]; then
- # Some processes are slow to start
- sleep $SLEEP_TIME
- checkstatusofproc
- RETVAL=$?
+ # The following needs to be removed once HDFS-1943 gets finally put to rest.
+ # The logic of this ugly hack is this: IFF we do NOT have jsvc installed it is
+ # guaranteed that we can NOT be running in a secure mode and thus we need to
+ # workaround HDFS-1943 (start as non-root). As soon as jsvc gets installed
+ # we are assuming a secure installation and starting a data node as root.
+ # This leaves 2 corner cases:
+ # 1. HADOOP_DATANODE_USER being set to root
+ # 2. jsvc is installed but Hadoop is configures to run in an unsecure mode
+ # Both will currently fail
+ if [ -f $HADOOP_HOME/libexec/jsvc.amd64 -o -f $HADOOP_HOME/libexec/jsvc.i386 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
+ TARGET_USER=root
+ fi
fi
+ su -s /bin/bash $TARGET_USER -c "$EXEC_PATH --config '$CONFIG_PATH' start @HADOOP_DAEMON@ $DAEMON_FLAGS"
+
+ # Some processes are slow to start
+ sleep $SLEEP_TIME
+ checkstatusofproc
+ RETVAL=$?
[ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE
return $RETVAL
@@ -81,7 +103,7 @@ start() {
stop() {
log_success_msg "Stopping $desc (hadoop-@HADOOP_DAEMON@): "
- start_daemon -u $USER $EXEC_PATH --config "$CONFIG_PATH" stop @HADOOP_DAEMON@
+ start_daemon $EXEC_PATH --config "$CONFIG_PATH" stop @HADOOP_DAEMON@
RETVAL=$?
[ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
@@ -93,8 +115,10 @@ stop() {
RETVAL=$?
# Stopping a non running process should be a success
- if [ $RETVAL -eq $STATUS_NOT_RUNNING ]; then
+ if [ $RETVAL -ne $STATUS_RUNNING ]; then
RETVAL=0
+ else
+ RETVAL=$STATUS_OTHER_ERROR
fi
fi
}
@@ -106,7 +130,13 @@ restart() {
checkstatusofproc(){
- pidofproc -p $PIDFILE java > /dev/null
+ # Under certain conditions datanode manifests as jsvc.exec
+ if [ "@HADOOP_DAEMON@" = "datanode" -a -x @HADOOP_COMMON_ROOT@/sbin/Linux-amd64-64/jsvc ] ; then
+ PROC_NAME="jsvc"
+ else
+ PROC_NAME="su"
+ fi
+ pidofproc -p $PIDFILE $PROC_NAME > /dev/null
}
checkstatus(){
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec?rev=1196329&r1=1196328&r2=1196329&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec Tue Nov 1 22:21:52 2011
@@ -15,6 +15,12 @@
#
# Hadoop RPM spec file
#
+
+# FIXME: we need to disable a more strict checks on native files for now,
+# since Hadoop build system makes it difficult to pass the kind of flags
+# that would make newer RPM debuginfo generation scripts happy.
+%undefine _missing_build_ids_terminate_build
+
%define hadoop_name hadoop
%define etc_hadoop /etc/%{name}
%define etc_yarn /etc/yarn
@@ -25,7 +31,6 @@
%define log_hadoop %{log_hadoop_dirname}/%{name}
%define bin_hadoop %{_bindir}
%define man_hadoop %{_mandir}
-%define doc_hadoop %{_docdir}/%{name}-%{hadoop_version}
%define src_hadoop /usr/src/%{name}
%define hadoop_username mapred
%define hadoop_services namenode secondarynamenode datanode jobtracker tasktracker
@@ -55,6 +60,7 @@
/usr/lib/rpm/brp-python-bytecompile ; \
%{nil}
+%define doc_hadoop %{_docdir}/%{name}-%{hadoop_version}
%define alternatives_cmd alternatives
%global initd_dir %{_sysconfdir}/rc.d/init.d
%endif
@@ -73,11 +79,13 @@
/usr/lib/rpm/brp-compress ; \
%{nil}
+%define doc_hadoop %{_docdir}/%{name}
%define alternatives_cmd update-alternatives
%global initd_dir %{_sysconfdir}/rc.d
%endif
%if 0%{?mgaversion}
+%define doc_hadoop %{_docdir}/%{name}-%{hadoop_version}
%define alternatives_cmd update-alternatives
%global initd_dir %{_sysconfdir}/rc.d/init.d
%endif
@@ -108,13 +116,14 @@ Source5: hadoop-init.tmpl.suse
Source6: hadoop.1
Source7: hadoop-fuse-dfs.1
Source8: hadoop-fuse.default
+Source9: hadoop.nofiles.conf
Buildroot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
BuildRequires: python >= 2.4, git, fuse-devel,fuse, automake, autoconf
-Requires: textutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service
+Requires: coreutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service, bigtop-utils
Provides: hadoop
%if %{?suse_version:1}0
-BuildRequires: libfuse2, libopenssl-devel, gcc-c++, ant, ant-nodeps, ant-trax, liblzo-devel
+BuildRequires: libfuse2, libopenssl-devel, gcc-c++, ant, ant-nodeps, ant-trax
# Required for init scripts
Requires: sh-utils, insserv
%endif
@@ -122,14 +131,14 @@ Requires: sh-utils, insserv
# CentOS 5 does not have any dist macro
# So I will suppose anything that is not Mageia or a SUSE will be a RHEL/CentOS/Fedora
%if %{!?suse_version:1}0 && %{!?mgaversion:1}0
-BuildRequires: fuse-libs, libtool, redhat-rpm-config, lzo-devel
+BuildRequires: fuse-libs, libtool, redhat-rpm-config, lzo-devel, openssl-devel
# Required for init scripts
Requires: sh-utils, redhat-lsb
%endif
%if 0%{?mgaversion}
-BuildRequires: libfuse-devel, libfuse2 , libopenssl-devel, gcc-c++, ant, libtool, automake, autoconf, liblzo-devel, libzlib-devel
-Requires: chkconfig, xinetd-simple-services, libzlib
+BuildRequires: libfuse-devel, libfuse2 , libopenssl-devel, gcc-c++, ant, libtool, automake, autoconf, liblzo-devel, zlib-devel
+Requires: chkconfig, xinetd-simple-services, zlib, initscripts
%endif
@@ -300,7 +309,7 @@ before continuing operation.
# This assumes that you installed Java JDK 5 and set JAVA5_HOME
# This assumes that you installed Forrest and set FORREST_HOME
-env HADOOP_VERSION=%{hadoop_version} bash %{SOURCE1}
+env HADOOP_VERSION=%{hadoop_version} HADOOP_ARCH=%{hadoop_arch} bash %{SOURCE1}
%clean
%__rm -rf $RPM_BUILD_ROOT
@@ -365,6 +374,8 @@ done
%__cp $RPM_SOURCE_DIR/hadoop.default $RPM_BUILD_ROOT/etc/default/hadoop
%__cp $RPM_SOURCE_DIR/hadoop-fuse.default $RPM_BUILD_ROOT/etc/default/hadoop-fuse
+%__install -d -m 0755 $RPM_BUILD_ROOT/etc/security/limits.d
+%__install -m 0644 %{SOURCE9} $RPM_BUILD_ROOT/etc/security/limits.d/hadoop.nofiles.conf
# /var/lib/hadoop/cache
%__install -d -m 1777 $RPM_BUILD_ROOT/var/lib/%{name}/cache
@@ -393,6 +404,11 @@ getent passwd hdfs >/dev/null || /usr/sb
--slave /etc/%{hadoop_name} %{hadoop_name}-etc %{etc_hadoop} \
--slave %{man_hadoop}/man1/%{hadoop_name}.1.*z %{hadoop_name}-man %{man_hadoop}/man1/%{name}.1.*z
+chmod g+w /var/lib/hadoop/cache/hadoop/
+mkdir -p /var/log/hadoop || :
+touch /var/log/hadoop/SecurityAuth.audit
+chgrp hadoop /var/log/hadoop/SecurityAuth.audit
+chmod g+w /var/log/hadoop/SecurityAuth.audit
%preun
if [ "$1" = 0 ]; then
@@ -409,6 +425,7 @@ fi
%defattr(-,root,root)
%config(noreplace) %{etc_hadoop}/conf.empty
%config(noreplace) /etc/default/hadoop
+%config(noreplace) /etc/security/limits.d/hadoop.nofiles.conf
%{lib_hadoop}
%{bin_hadoop}/%{name}
%{man_hadoop}/man1/hadoop.1.*z
@@ -416,6 +433,14 @@ fi
%attr(0775,root,hadoop) %{log_hadoop}
%exclude %{lib_hadoop}/lib/native
+%exclude %{lib_hadoop}/sbin/%{hadoop_arch}
+%exclude %{lib_hadoop}/bin/fuse_dfs
+%exclude %{lib_hadoop}/contrib/fuse-dfs
+%exclude %{lib_hadoop}/hdfs/contrib/fuse-dfs
+# FIXME: The following is a workaround for BIGTOP-139
+# %exclude %{lib_hadoop}/bin/task-controller
+%exclude %{lib_hadoop}/bin/jsvc*
+%exclude %{lib_hadoop}/hdfs/bin/jsvc*
%files doc
%defattr(-,root,root)
@@ -432,7 +457,6 @@ fi
%files %1 \
%defattr(-,root,root) \
%{initd_dir}/%{name}-%1 \
-%{lib_hadoop}/bin/hadoop-daemon.sh \
%post %1 \
chkconfig --add %{name}-%1 \
%2 \
@@ -480,9 +504,12 @@ fi
%files fuse
%defattr(-,root,root)
-%attr(0755,root,root) %{bin_hadoop}/hadoop-fuse-dfs
-%attr(0755,root,root) %{man_hadoop}/man1/hadoop-fuse-dfs.1.gz
%config(noreplace) /etc/default/hadoop-fuse
+%{lib_hadoop}/contrib/fuse-dfs
+%attr(0755,root,root) %{lib_hadoop}/bin/fuse_dfs
+%attr(0755,root,root) %{lib_hadoop}/bin/fuse_dfs_wrapper.sh
+%attr(0755,root,root) %{bin_hadoop}/hadoop-fuse-dfs
+%attr(0644,root,root) %{man_hadoop}/man1/hadoop-fuse-dfs.1.*
%files pipes
%defattr(-,root,root)
@@ -501,5 +528,9 @@ fi
%defattr(-,root,root)
%dir %{lib_hadoop}/sbin
%dir %{lib_hadoop}/sbin/%{hadoop_arch}
-# %attr(4754,root,mapred) %{lib_hadoop}/sbin/%{hadoop_arch}/task-controller
+# %attr(4750,root,mapred) %{lib_hadoop}/sbin/%{hadoop_arch}/task-controller
%attr(0755,root,root) %{lib_hadoop}/sbin/%{hadoop_arch}/jsvc
+
+# FIXME: The following is a workaround for BIGTOP-139
+# %attr(4750,root,mapred) %{lib_hadoop}/bin/task-controller
+%attr(0755,root,root) %{lib_hadoop}/bin/jsvc*