You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2012/03/14 22:47:08 UTC
svn commit: r1300739 - in /incubator/bigtop/branches/hadoop-0.23: ./
bigtop-packages/src/common/hadoop/ bigtop-packages/src/deb/hadoop/
bigtop-packages/src/rpm/hadoop/SPECS/
Author: rvs
Date: Wed Mar 14 21:47:07 2012
New Revision: 1300739
URL: http://svn.apache.org/viewvc?rev=1300739&view=rev
Log:
BIGTOP-449. hadoop package cleanup for hadoop-0.23
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop-layout.sh
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hdfs.default
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/httpfs.default
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/init.d.tmpl
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-httpfs.install
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
incubator/bigtop/branches/hadoop-0.23/bigtop.mk
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop-layout.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop-layout.sh?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop-layout.sh (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop-layout.sh Wed Mar 14 21:47:07 2012
@@ -21,3 +21,10 @@ YARN_DIR="./"
YARN_LIB_JARS_DIR="lib"
MAPRED_DIR="./"
MAPRED_LIB_JARS_DIR="lib"
+
+HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-"/usr/lib/hadoop/libexec"}
+HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop/conf"}
+HADOOP_COMMON_HOME=${HADOOP_COMMON_HOME:-"/usr/lib/hadoop"}
+HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME:-"/usr/lib/hadoop-hdfs"}
+HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-"/usr/lib/hadoop-mapreduce"}
+YARN_HOME=${YARN_HOME:-"/usr/lib/hadoop-yarn"}
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default Wed Mar 14 21:47:07 2012
@@ -23,3 +23,4 @@ export HADOOP_COMMON_HOME=/usr/lib/hadoo
export HADOOP_HDFS_HOME=/usr/lib/hadoop-hdfs
export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce
export YARN_HOME=/usr/lib/hadoop-yarn
+export JSVC_HOME=/usr/lib/bigtop-jsvc
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hdfs.default
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hdfs.default?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hdfs.default (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hdfs.default Wed Mar 14 21:47:07 2012
@@ -20,3 +20,5 @@ export HADOOP_DATANODE_USER=hdfs
export HADOOP_IDENT_STRING=hdfs
# export HADOOP_SECURE_DN_USER=hdfs
+# export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop-hdfs
+# export HADOOP_SECURE_DN_LOG_DIR=/var/log/hadoop-hdfs
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/httpfs.default
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/httpfs.default?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/httpfs.default (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/httpfs.default Wed Mar 14 21:47:07 2012
@@ -16,9 +16,9 @@ export HTTPFS_USER=httpfs
export HTTPFS_CONFIG=/etc/hadoop-httpfs/conf
export HTTPFS_LOG=/var/log/hadoop-httpfs/
export HTTPFS_TEMP=/var/run/hadoop-httpfs/
+export HTTPFS_CATALINA_HOME=/usr/lib/bigtop-tomcat
export CATALINA_PID=/var/run/hadoop-httpfs/hadoop-httpfs-httpfs.pid
export CATALINA_BASE=/usr/lib/hadoop-httpfs
export CATALINA_TMPDIR=/var/run/hadoop-httpfs/
-export CATALINA_HOME=/usr/lib/bigtop-tomcat
# HTTPFS_HTTP_PORT
# HTTPFS_ADMIN_PORT
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/init.d.tmpl
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/init.d.tmpl?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/init.d.tmpl (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/init.d.tmpl Wed Mar 14 21:47:07 2012
@@ -56,44 +56,6 @@ fi
generate_start() {
cat <<'__EOT__'
-start() {
- [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
- [ -d $CONF_DIR ] || exit $ERROR_PROGRAM_NOT_CONFIGURED
- log_success_msg "Starting ${DESC}: "
-
- TARGET_USER_NAME="HADOOP_`echo datanode | tr a-z A-Z`_USER"
- TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
-
- # The following needs to be removed once HDFS-1943 gets finally put to rest.
- # The logic of this ugly hack is this: IFF we do NOT have jsvc installed it is
- # guaranteed that we can NOT be running in a secure mode and thus we need to
- # workaround HDFS-1943 (start as non-root). As soon as jsvc gets installed
- # we are assuming a secure installation and starting a data node as root.
- # This leaves 2 corner cases:
- # 1. HADOOP_DATANODE_USER being set to root
- # 2. jsvc is installed but Hadoop is configures to run in an unsecure mode
- # Both will currently fail
- if [ -f $HADOOP_PREFIX/libexec/jsvc ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
- TARGET_USER=root
- fi
-
- export HADOOP_IDENT_STRING=$TARGET_USER
- su -s /bin/bash $TARGET_USER -c "$EXEC_PATH --config '$CONF_DIR' start $DAEMON_FLAGS"
-
- # Some processes are slow to start
- sleep $SLEEP_TIME
- checkstatusofproc
- RETVAL=$?
-
- [ $RETVAL -eq $RETVAL_SUCCESS ] && touch $LOCKFILE
- return $RETVAL
-}
-__EOT__
-
-}
-generate_start() {
-
-cat <<'__EOT__'
start() {
[ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh Wed Mar 14 21:47:07 2012
@@ -172,8 +172,6 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-d
. /usr/lib/bigtop-utils/bigtop-detect-javahome
fi
-. /etc/default/hadoop
-
exec ${component#${PREFIX}} "\$@"
EOF
chmod 755 $wrapper
@@ -188,6 +186,7 @@ cp ${DISTRO_DIR}/hadoop-layout.sh ${SYST
install -d -m 0755 ${HADOOP_DIR}
cp ${BUILD_DIR}/share/hadoop/common/*.jar ${HADOOP_DIR}/
cp ${BUILD_DIR}/share/hadoop/common/lib/hadoop-auth*.jar ${HADOOP_DIR}/
+cp ${BUILD_DIR}/share/hadoop/mapreduce/lib/hadoop-annotations*.jar ${HADOOP_DIR}/
install -d -m 0755 ${MAPREDUCE_DIR}
cp ${BUILD_DIR}/share/hadoop/mapreduce/hadoop-mapreduce*.jar ${MAPREDUCE_DIR}
cp ${BUILD_DIR}/share/hadoop/tools/lib/*.jar ${MAPREDUCE_DIR}
@@ -200,7 +199,6 @@ chmod 644 ${HADOOP_DIR}/*.jar ${MAPREDUC
# lib jars
install -d -m 0755 ${HADOOP_DIR}/lib
cp ${BUILD_DIR}/share/hadoop/common/lib/*.jar ${HADOOP_DIR}/lib
-rm -f ${HADOOP_DIR}/lib/hadoop*.jar
install -d -m 0755 ${MAPREDUCE_DIR}/lib
cp ${BUILD_DIR}/share/hadoop/mapreduce/lib/*.jar ${MAPREDUCE_DIR}/lib
install -d -m 0755 ${HDFS_DIR}/lib
@@ -270,8 +268,6 @@ chmod 644 $MAN_DIR/man1/hadoop.1.gz
# HTTPFS
install -d -m 0755 ${HTTPFS_DIR}/sbin
cp ${BUILD_DIR}/sbin/httpfs.sh ${HTTPFS_DIR}/sbin/
-install -d -m 0755 ${HTTPFS_DIR}/libexec
-cp ${BUILD_DIR}/libexec/httpfs-config.sh ${HTTPFS_DIR}/libexec/
cp -r ${BUILD_DIR}/share/hadoop/httpfs/tomcat/webapps ${HTTPFS_DIR}/
cp -r ${BUILD_DIR}/share/hadoop/httpfs/tomcat/conf ${HTTPFS_DIR}/
chmod 644 ${HTTPFS_DIR}/conf/*
@@ -290,19 +286,16 @@ cp ${BUILD_DIR}/etc/hadoop/log4j.propert
install -d -m 0755 ${HADOOP_DIR}/etc
ln -s ${HADOOP_ETC_DIR##${PREFIX}}/conf ${HADOOP_DIR}/etc/hadoop
-# FIXME: Provide convenience links for log/run in hdfs and mapreduce (HADOOP-7939)
+# Create log, var and lib
install -d -m 0755 $PREFIX/var/{log,run,lib}/hadoop-hdfs
install -d -m 0755 $PREFIX/var/{log,run,lib}/hadoop-yarn
install -d -m 0755 $PREFIX/var/{log,run,lib}/hadoop-mapreduce
-# FIXME: The following needs to be untangled upstream (HADOOP-7939)
-cp ${BUILD_DIR}/share/hadoop/mapreduce/hadoop-mapreduce-client-shuffle*.jar ${HADOOP_DIR}/lib/
-cp ${BUILD_DIR}/share/hadoop/mapreduce/hadoop-mapreduce-client-core*.jar ${HADOOP_DIR}/lib/
-
# Remove all source and test jars and create version-less symlinks to offer integration point with other projects
for DIR in ${HADOOP_DIR} ${HDFS_DIR} ${YARN_DIR} ${MAPREDUCE_DIR} ${HTTPFS_DIR} ; do
(cd $DIR &&
rm -fv *-test.jar *-sources.jar
+ rm -fv lib/hadoop-*.jar
for j in hadoop-*.jar; do
if [[ $j =~ hadoop-(.*)-${HADOOP_VERSION}.jar ]]; then
name=${BASH_REMATCH[1]}
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install Wed Mar 14 21:47:07 2012
@@ -1,7 +1,6 @@
/etc/security/limits.d/hdfs.conf
/etc/hadoop/conf.empty/hdfs-site.xml
/usr/lib/hadoop-hdfs
-/usr/lib/hadoop/libexec/jsvc
/usr/lib/hadoop/libexec/hdfs-config.sh
/usr/bin/hdfs
/var/lib/hadoop-hdfs
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-httpfs.install
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-httpfs.install?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-httpfs.install (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-httpfs.install Wed Mar 14 21:47:07 2012
@@ -1,3 +1,4 @@
/etc/default/hadoop-httpfs
/etc/hadoop-httpfs
+/usr/lib/hadoop/libexec/httpfs-config.sh
/usr/lib/hadoop-httpfs
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules Wed Mar 14 21:47:07 2012
@@ -73,9 +73,6 @@ install: build
--native-build-string=${native_dir} \
--installed-lib-dir=/usr/lib/hadoop
- ln -fs /usr/lib/bigtop-utils/jsvc debian/tmp/usr/lib/hadoop/libexec/jsvc
- ln -fs /usr/lib/bigtop-tomcat/bin debian/tmp/usr/lib/hadoop-httpfs/bin
-
# Run dh_lintian if we've got it - this doesn't exist on debhelper 6 (eg Ubuntu Hardy)
(dh_lintian) || /bin/true
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec Wed Mar 14 21:47:07 2012
@@ -361,7 +361,7 @@ Hadoop Filesystem Library
%prep
# %setup -n %{name}-%{hadoop_base_version}-src
-%setup -n apache-hadoop-common-2cbb181
+%setup -n apache-hadoop-common-f616c85
%build
# This assumes that you installed Java JDK 6 and set JAVA_HOME
@@ -397,12 +397,6 @@ bash %{SOURCE2} \
--installed-lib-dir=%{lib_hadoop} \
--man-dir=$RPM_BUILD_ROOT%{man_hadoop} \
-# Replace the bundled jsvc with a link to a bigtop-jsvc one
-%__ln_s -f %{libexecdir}/bigtop-utils/jsvc $RPM_BUILD_ROOT/%{lib_hadoop}/libexec/jsvc
-
-# Provide a symlink to the bigtop-tomcat
-%__ln_s -f /usr/lib/bigtop-tomcat/bin $RPM_BUILD_ROOT/%{lib_httpfs}/bin
-
# Init.d scripts
%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/
@@ -512,7 +506,6 @@ fi
%config(noreplace) /etc/security/limits.d/hdfs.conf
%{lib_hdfs}
%{lib_hadoop}/libexec/hdfs-config.sh
-%{lib_hadoop}/libexec/jsvc
%{bin_hadoop}/hdfs
%attr(0775,hdfs,hadoop) %{run_hdfs}
%attr(0775,hdfs,hadoop) %{log_hdfs}
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop.mk
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop.mk?rev=1300739&r1=1300738&r2=1300739&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop.mk (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop.mk Wed Mar 14 21:47:07 2012
@@ -24,7 +24,7 @@ HADOOP_RELEASE_VERSION=1
HADOOP_TARBALL_DST=$(HADOOP_NAME)-$(HADOOP_BASE_VERSION).tar.gz
#HADOOP_TARBALL_SRC=$(HADOOP_NAME)-$(HADOOP_BASE_VERSION)-src.tar.gz
#HADOOP_SITE=$(APACHE_MIRROR)/hadoop/common/$(HADOOP_NAME)-$(HADOOP_BASE_VERSION)/
-HADOOP_TARBALL_SRC=2cbb181
+HADOOP_TARBALL_SRC=f616c85
HADOOP_SITE=https://github.com/apache/hadoop-common/tarball
$(eval $(call PACKAGE,hadoop,HADOOP))