You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by bm...@apache.org on 2012/04/11 22:48:05 UTC
svn commit: r1324993 - in /incubator/bigtop/branches/hadoop-0.23:
bigtop-packages/src/common/hadoop/do-component-build
bigtop-packages/src/common/hadoop/install_hadoop.sh
bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec bigtop.mk
Author: bmahe
Date: Wed Apr 11 20:48:04 2012
New Revision: 1324993
URL: http://svn.apache.org/viewvc?rev=1324993&view=rev
Log:
BIGTOP-422. Need to start packaging fuse for Apache Hadoop.
Bump also upstream Apache Hadoop version
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
incubator/bigtop/branches/hadoop-0.23/bigtop.mk
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build?rev=1324993&r1=1324992&r2=1324993&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build Wed Apr 11 20:48:04 2012
@@ -26,7 +26,7 @@ mkdir build/src
# Build artifacts
MAVEN_SKIP_TESTS="-DskipTests -DskipTest -DskipITs"
-mvn $BUNDLE_SNAPPY -Dcontainer-executor.conf.dir=/etc/hadoop/conf -Pdist -Pnative -Psrc -Dtar ${MAVEN_SKIP_TESTS} install "$@"
+mvn $BUNDLE_SNAPPY -Dcontainer-executor.conf.dir=/etc/hadoop/conf -Pdist -Pnative -Pfuse -Psrc -Dtar ${MAVEN_SKIP_TESTS} install "$@"
mvn site site:stage ${MAVEN_SKIP_TESTS} $@
(cd build ; tar --strip-components=1 -xzvf ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz)
@@ -38,3 +38,6 @@ cp -r target/site/* build/share/doc/
# Create a manifest for hadoop client package
(cd hadoop-client/target/hadoop-client-*/share/hadoop/client/lib ; ls) > build/hadoop-client.list
+
+# Copy fuse output to the build directory
+cp hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/target/fuse-dfs/src/{fuse_dfs,fuse_dfs_wrapper.sh} build/bin/
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh?rev=1324993&r1=1324992&r2=1324993&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh Wed Apr 11 20:48:04 2012
@@ -219,7 +219,7 @@ cp -ra ${BUILD_DIR}/share/hadoop/hdfs/we
# bin
install -d -m 0755 ${HADOOP_DIR}/bin
-cp -a ${BUILD_DIR}/bin/{hadoop,rcc} ${HADOOP_DIR}/bin
+cp -a ${BUILD_DIR}/bin/{hadoop,rcc,fuse_dfs,fuse_dfs_wrapper.sh} ${HADOOP_DIR}/bin
install -d -m 0755 ${HDFS_DIR}/bin
cp -a ${BUILD_DIR}/bin/hdfs ${HDFS_DIR}/bin
install -d -m 0755 ${YARN_DIR}/bin
@@ -231,9 +231,9 @@ cp -a ${BUILD_DIR}/bin/mapred ${YARN_DIR
# sbin
install -d -m 0755 ${HADOOP_DIR}/sbin
-cp -a ${BUILD_DIR}/sbin/{hadoop-daemon,hadoop-daemons,slaves,start-all,stop-all}.sh ${HADOOP_DIR}/sbin
+cp -a ${BUILD_DIR}/sbin/{hadoop-daemon,hadoop-daemons,slaves}.sh ${HADOOP_DIR}/sbin
install -d -m 0755 ${HDFS_DIR}/sbin
-cp -a ${BUILD_DIR}/sbin/{distribute-exclude,refresh-namenodes,start-balancer,start-dfs,start-secure-dns,stop-balancer,stop-dfs,stop-secure-dns}.sh ${HDFS_DIR}/sbin
+cp -a ${BUILD_DIR}/sbin/{distribute-exclude,refresh-namenodes}.sh ${HDFS_DIR}/sbin
install -d -m 0755 ${YARN_DIR}/sbin
cp -a ${BUILD_DIR}/sbin/{yarn-daemon,yarn-daemons}.sh ${YARN_DIR}/sbin
install -d -m 0755 ${MAPREDUCE_DIR}/sbin
@@ -258,6 +258,34 @@ for library in `cd ${BUILD_DIR}/lib ; ls
ln -s ${library} ${HADOOP_NATIVE_LIB_DIR}/${library/.so.*/}.so
done
+# Install fuse wrapper
+fuse_wrapper=${BIN_DIR}/hadoop-fuse-dfs
+cat > $fuse_wrapper << EOF
+#!/bin/bash
+
+/sbin/modprobe fuse
+
+export HADOOP_HOME=\${HADOOP_HOME:-${HADOOP_DIR#${PREFIX}}}
+
+if [ -f /etc/default/hadoop-fuse ]
+then . /etc/default/hadoop-fuse
+fi
+
+export HADOOP_LIBEXEC_DIR=/${SYSTEM_LIBEXEC_DIR#${PREFIX}}
+
+if [ "\${LD_LIBRARY_PATH}" = "" ]; then
+ export LD_LIBRARY_PATH=/usr/lib
+ for f in \`find \${JAVA_HOME}/jre/lib -name client -prune -o -name libjvm.so -exec dirname {} \;\`; do
+ export LD_LIBRARY_PATH=\$f:\${LD_LIBRARY_PATH}
+ done
+fi
+
+env \${HADOOP_HOME}/bin/fuse_dfs \$@
+EOF
+
+chmod 755 $fuse_wrapper
+
+
# conf
install -d -m 0755 $HADOOP_ETC_DIR/conf.empty
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec?rev=1324993&r1=1324992&r2=1324993&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec Wed Apr 11 20:48:04 2012
@@ -337,8 +337,10 @@ The History server keeps records of the
%package client
Summary: Hadoop client side dependencies
Group: System/Daemons
-Requires: %{name} = %{version}-%{release}, %{name}-hdfs = %{version}-%{release}
-Requires: %{name}-yarn = %{version}-%{release}, %{name}-mapreduce = %{version}-%{release}
+Requires: %{name} = %{version}-%{release}
+Requires: %{name}-hdfs = %{version}-%{release}
+Requires: %{name}-yarn = %{version}-%{release}
+Requires: %{name}-mapreduce = %{version}-%{release}
%description client
Installation of this package will provide you with all the dependencies for Hadoop clients.
@@ -346,9 +348,12 @@ Installation of this package will provid
%package conf-pseudo
Summary: Hadoop installation in pseudo-distributed mode
Group: System/Daemons
-Requires: %{name} = %{version}-%{release}, %{name}-hdfs-namenode = %{version}-%{release}
-Requires: %{name}-hdfs-datanode = %{version}-%{release}, %{name}-hdfs-secondarynamenode = %{version}-%{release}
-Requires: %{name}-yarn-resourcemanager = %{version}-%{release}, %{name}-yarn-nodemanager = %{version}-%{release}
+Requires: %{name} = %{version}-%{release}
+Requires: %{name}-hdfs-namenode = %{version}-%{release}
+Requires: %{name}-hdfs-datanode = %{version}-%{release}
+Requires: %{name}-hdfs-secondarynamenode = %{version}-%{release}
+Requires: %{name}-yarn-resourcemanager = %{version}-%{release}
+Requires: %{name}-yarn-nodemanager = %{version}-%{release}
Requires: %{name}-mapreduce-historyserver = %{version}-%{release}
%description conf-pseudo
@@ -371,9 +376,30 @@ AutoReq: no
%description libhdfs
Hadoop Filesystem Library
+
+%package fuse
+Summary: Mountable HDFS
+Group: Development/Libraries
+Requires: %{name} = %{version}-%{release}
+Requires: %{name}-libhdfs = %{version}-%{release}
+Requires: fuse
+AutoReq: no
+
+%if %{?suse_version:1}0
+Requires: libfuse2
+%else
+Requires: fuse-libs
+%endif
+
+
+%description fuse
+These projects (enumerated below) allow HDFS to be mounted (on most flavors of Unix) as a standard file system using
+
+
%prep
# %setup -n %{name}-%{hadoop_base_version}-src
-%setup -n apache-hadoop-common-f616c85
+%setup -n apache-hadoop-common-8c0466d
+
%build
# This assumes that you installed Java JDK 6 and set JAVA_HOME
@@ -626,3 +652,11 @@ fi
%{_includedir}/hdfs.h
# -devel should be its own package
#%doc %{_docdir}/libhdfs-%{hadoop_version}
+
+%files fuse
+%defattr(-,root,root)
+%attr(0755,root,root) %{lib_hadoop}/bin/fuse_dfs
+%attr(0755,root,root) %{lib_hadoop}/bin/fuse_dfs_wrapper.sh
+%attr(0755,root,root) %{bin_hadoop}/hadoop-fuse-dfs
+
+
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop.mk
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop.mk?rev=1324993&r1=1324992&r2=1324993&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop.mk (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop.mk Wed Apr 11 20:48:04 2012
@@ -18,13 +18,13 @@ BIGTOP_VERSION=0.3.0-hadoop23-incubating
# Hadoop 0.20.0-based hadoop package
HADOOP_NAME=hadoop
HADOOP_RELNOTES_NAME=Apache Hadoop
-HADOOP_BASE_VERSION=0.23.3-SNAPSHOT
-HADOOP_PKG_VERSION=0.23.3
+HADOOP_BASE_VERSION=2.0.0-SNAPSHOT
+HADOOP_PKG_VERSION=2.0.0
HADOOP_RELEASE_VERSION=1
HADOOP_TARBALL_DST=$(HADOOP_NAME)-$(HADOOP_BASE_VERSION).tar.gz
#HADOOP_TARBALL_SRC=$(HADOOP_NAME)-$(HADOOP_BASE_VERSION)-src.tar.gz
#HADOOP_SITE=$(APACHE_MIRROR)/hadoop/common/$(HADOOP_NAME)-$(HADOOP_BASE_VERSION)/
-HADOOP_TARBALL_SRC=f616c85
+HADOOP_TARBALL_SRC=8c0466d
HADOOP_SITE=https://github.com/apache/hadoop-common/tarball
$(eval $(call PACKAGE,hadoop,HADOOP))