You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2015/02/09 21:51:16 UTC
bigtop git commit: BIGTOP-1179. Add Apache Tez to bigtop
Repository: bigtop
Updated Branches:
refs/heads/master b00e9da8f -> f506701c8
BIGTOP-1179. Add Apache Tez to bigtop
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/f506701c
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/f506701c
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/f506701c
Branch: refs/heads/master
Commit: f506701c8e8cd1b9e596c3f728996c988705e7d0
Parents: b00e9da
Author: Olaf Flebbe <o....@science-computing.de>
Authored: Thu Nov 27 11:54:41 2014 +0100
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Mon Feb 9 21:49:50 2015 +0100
----------------------------------------------------------------------
.../src/common/tez/do-component-build | 31 +++++
bigtop-packages/src/common/tez/install_tez.sh | 122 +++++++++++++++++++
bigtop-packages/src/common/tez/tez-site.xml | 27 ++++
bigtop-packages/src/common/tez/tez.1 | 27 ++++
bigtop-packages/src/deb/tez/compat | 1 +
bigtop-packages/src/deb/tez/control | 29 +++++
bigtop-packages/src/deb/tez/copyright | 16 +++
bigtop-packages/src/deb/tez/rules | 42 +++++++
bigtop-packages/src/deb/tez/source/format | 1 +
bigtop-packages/src/deb/tez/tez.install | 3 +
bigtop-packages/src/rpm/tez/RPMS/.gitignore | 1 +
bigtop-packages/src/rpm/tez/SPECS/tez.spec | 110 +++++++++++++++++
bigtop.mk | 14 +++
13 files changed, 424 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/common/tez/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/tez/do-component-build b/bigtop-packages/src/common/tez/do-component-build
new file mode 100644
index 0000000..88f802a
--- /dev/null
+++ b/bigtop-packages/src/common/tez/do-component-build
@@ -0,0 +1,31 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+. `dirname $0`/bigtop.bom
+
+BUILD_TEZ_OPTS="clean package \
+-Dtar -Dhadoop.version=${HADOOP_VERSION} \
+-Phadoop24 -P!hadoop26
+-DskipTests"
+
+#mvn versions:set -DnewVersion=${TEZ_VERSION}
+
+mvn ${BUILD_TEZ_OPTS} "$@"
+
+
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/common/tez/install_tez.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/tez/install_tez.sh b/bigtop-packages/src/common/tez/install_tez.sh
new file mode 100644
index 0000000..c533dcd
--- /dev/null
+++ b/bigtop-packages/src/common/tez/install_tez.sh
@@ -0,0 +1,122 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --build-dir=DIR path to tez dist.dir
+ --prefix=PREFIX path to install into
+
+ Optional options:
+ --doc-dir=DIR path to install docs into [/usr/share/doc/tez]
+ --lib-dir=DIR path to install tez home [/usr/lib/tez]
+ --installed-lib-dir=DIR path where lib-dir will end up on target system
+ --bin-dir=DIR path to install bins [/usr/bin]
+ --examples-dir=DIR path to install examples [doc-dir/examples]
+ ... [ see source for more similar options ]
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'prefix:' \
+ -l 'doc-dir:' \
+ -l 'lib-dir:' \
+ -l 'installed-lib-dir:' \
+ -l 'bin-dir:' \
+ -l 'examples-dir:' \
+ -l 'conf-dir:' \
+ -l 'sbin-dir:' \
+ -l 'libexec-dir:' \
+ -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+ case "$1" in
+ --prefix)
+ PREFIX=$2 ; shift 2
+ ;;
+ --build-dir)
+ BUILD_DIR=$2 ; shift 2
+ ;;
+ --doc-dir)
+ DOC_DIR=$2 ; shift 2
+ ;;
+ --lib-dir)
+ LIB_DIR=$2 ; shift 2
+ ;;
+ --installed-lib-dir)
+ INSTALLED_LIB_DIR=$2 ; shift 2
+ ;;
+ --examples-dir)
+ EXAMPLES_DIR=$2 ; shift 2
+ ;;
+ --libexec-dir)
+ LIBEXEC_DIR=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+for var in PREFIX BUILD_DIR ; do
+ if [ -z "$(eval "echo \$$var")" ]; then
+ echo Missing param: $var
+ usage
+ fi
+done
+
+MAN_DIR=${MAN_DIR:-/usr/share/man/man1}
+DOC_DIR=${DOC_DIR:-/usr/share/doc/tez}
+LIB_DIR=${LIB_DIR:-/usr/lib/tez}
+CONF_DIR=${CONF_DIR:-/etc/tez/conf}
+LIBEXEC_DIR=${LIBEXEC_DIR:-${LIB_DIR}/tez}
+
+install -d -m 0755 $PREFIX/$MAN_DIR
+gzip -c tez.1 > $PREFIX/$MAN_DIR/tez.1.gz
+
+install -d -m 0755 $PREFIX/$LIB_DIR
+install -d -m 0755 $PREFIX/$LIB_DIR/lib
+install -d -m 0755 $PREFIX/$DOC_DIR
+install -d -m 0755 $PREFIX/$CONF_DIR
+install -d -m 0755 $PREFIX/$MAN_DIR
+install -d -m 0755 $PREFIX/$LIBEXEC_DIR
+
+cp tez-site.xml $PREFIX/$CONF_DIR
+
+tar -C $PREFIX/$LIB_DIR -xzf $BUILD_DIR/tez-dist/target/tez*-minimal.tar.gz
+
+#test ! -d $BUILD_DIR/lib || cp -ra $BUILD_DIR/lib/* ${PREFIX}/${LIB_DIR}/lib/
+#test ! -d $BUILD_DIR || cp $BUILD_DIR/tez*.jar $PREFIX/$LIB_DIR
+#test ! -d $BUILD_DIR/docs || cp -a $BUILD_DIR/docs/* $PREFIX/$DOC_DIR
+#test ! -d $BUILD_DIR/libexec || cp -a $BUILD_DIR/libexec/* $PREFIX/$LIBEXEC_DIR
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/common/tez/tez-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/tez/tez-site.xml b/bigtop-packages/src/common/tez/tez-site.xml
new file mode 100644
index 0000000..cd29d26
--- /dev/null
+++ b/bigtop-packages/src/common/tez/tez-site.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+<property>
+<name>tez.lib.uris</name>
+<value>${fs.default.name}/apps/tez/,${fs.default.name}/apps/tez/lib/</value>
+</property>
+
+<property>
+<name>tez.use.cluster.hadoop-libs</name>
+<value>true</value>
+</property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/common/tez/tez.1
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/tez/tez.1 b/bigtop-packages/src/common/tez/tez.1
new file mode 100644
index 0000000..177d1d7
--- /dev/null
+++ b/bigtop-packages/src/common/tez/tez.1
@@ -0,0 +1,27 @@
+.\" Licensed to the Apache Software Foundation (ASF) under one or more
+.\" contributor license agreements. See the NOTICE file distributed with
+.\" this work for additional information regarding copyright ownership.
+.\" The ASF licenses this file to You under the Apache License, Version 2.0
+.\" (the "License"); you may not use this file except in compliance with
+.\" the License. You may obtain a copy of the License at
+.\"
+.\" http://www.apache.org/licenses/LICENSE-2.0
+.\"
+.\" Unless required by applicable law or agreed to in writing, software
+.\" distributed under the License is distributed on an "AS IS" BASIS,
+.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.\" See the License for the specific language governing permissions and
+.\" limitations under the License.
+.\"
+.\" Process this file with
+.\" groff -man -Tascii tez.1
+.\"
+.TH tez 1 "January 2014 " Linux "User Manuals"
+
+.SH NAME
+The Apache Tez project is aimed at building an application framework
+which allows for a complex directed-acyclic-graph of tasks for
+processing data. It is currently built atop Apache Hadoop YARN
+
+.SH COPYRIGHT
+Copyright (C) 2010 The Apache Software Foundation. All rights reserved.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/compat
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/tez/compat b/bigtop-packages/src/deb/tez/compat
new file mode 100644
index 0000000..ec63514
--- /dev/null
+++ b/bigtop-packages/src/deb/tez/compat
@@ -0,0 +1 @@
+9
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/control
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/tez/control b/bigtop-packages/src/deb/tez/control
new file mode 100644
index 0000000..93e6604
--- /dev/null
+++ b/bigtop-packages/src/deb/tez/control
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+Source: tez
+Section: misc
+Priority: extra
+Maintainer: Bigtop <de...@bigtop.apache.org>
+Build-Depends: debhelper (>= 9)
+Standards-Version: 3.9.4
+Homepage: http://tez.apache.org/
+
+Package: tez
+Architecture: all
+Depends: hadoop-hdfs (>= 2.4.1), bigtop-utils
+Description: Framework for building complex DAG of tasks.
+ The Apache Tez project is aimed at building an application framework
+ which allows for a complex directed-acyclic-graph of tasks for
+ processing data. It is currently built atop Apache Hadoop YARN
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/copyright
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/tez/copyright b/bigtop-packages/src/deb/tez/copyright
new file mode 100644
index 0000000..17bcffe
--- /dev/null
+++ b/bigtop-packages/src/deb/tez/copyright
@@ -0,0 +1,16 @@
+Format: http://dep.debian.net/deps/dep5
+Source: http://tez.apache.org
+Upstream-Name: Apache Tez Incubator
+
+Files: *
+Copyright: 2008-2014, The Apache Software Foundation
+License: Apache-2.0
+
+Files debian/*
+Copyright: 2014, The Apache Software Foundation
+License: Apache-2.0
+
+License: Apache-2.0
+ On Debian systems, the complete text of the Apache 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/tez/rules b/bigtop-packages/src/deb/tez/rules
new file mode 100644
index 0000000..467de0a
--- /dev/null
+++ b/bigtop-packages/src/deb/tez/rules
@@ -0,0 +1,42 @@
+#!/usr/bin/make -f
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -*- makefile -*-
+SHELL=/bin/bash
+
+# Uncomment this to turn on verbose mode.
+export DH_VERBOSE=1
+
+# This has to be exported to make some magic below work.
+export DH_OPTIONS
+
+tez_pkg_name=tez
+
+%:
+ dh $@
+
+override_dh_auto_build:
+ bash debian/do-component-build -Dmaven.repo.local=${HOME}/.m2/repository
+
+
+override_dh_auto_install:
+ cp debian/tez.1 .
+ cp debian/tez-site.xml .
+ bash -x debian/install_tez.sh \
+ --build-dir=. \
+ --doc-dir=usr/share/doc/${tez_pkg_name}-doc \
+ --prefix=debian/tmp
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/source/format
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/tez/source/format b/bigtop-packages/src/deb/tez/source/format
new file mode 100644
index 0000000..163aaf8
--- /dev/null
+++ b/bigtop-packages/src/deb/tez/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/deb/tez/tez.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/tez/tez.install b/bigtop-packages/src/deb/tez/tez.install
new file mode 100644
index 0000000..dbcba69
--- /dev/null
+++ b/bigtop-packages/src/deb/tez/tez.install
@@ -0,0 +1,3 @@
+usr/lib/tez
+usr/share/man/man1
+etc/tez/conf
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/rpm/tez/RPMS/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/tez/RPMS/.gitignore b/bigtop-packages/src/rpm/tez/RPMS/.gitignore
new file mode 100644
index 0000000..1ab8c79
--- /dev/null
+++ b/bigtop-packages/src/rpm/tez/RPMS/.gitignore
@@ -0,0 +1 @@
+repodata
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop-packages/src/rpm/tez/SPECS/tez.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/tez/SPECS/tez.spec b/bigtop-packages/src/rpm/tez/SPECS/tez.spec
new file mode 100644
index 0000000..0082056
--- /dev/null
+++ b/bigtop-packages/src/rpm/tez/SPECS/tez.spec
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+%define tez_home /usr/lib/%{name}
+%define lib_tez %{tez_home}/lib
+%define man_dir %{_mandir}
+
+
+%if %{!?suse_version:1}0 && %{!?mgaversion:1}0
+
+%define __os_install_post \
+ /usr/lib/rpm/redhat/brp-compress ; \
+ /usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} ; \
+ /usr/lib/rpm/redhat/brp-strip-comment-note %{__strip} %{__objdump} ; \
+ /usr/lib/rpm/brp-python-bytecompile ; \
+ %{nil}
+
+%define doc_tez %{_docdir}/tez-%{tez_version}
+
+%endif
+
+
+%if %{?suse_version:1}0
+
+# Only tested on openSUSE 11.4. le'ts update it for previous
+#release when confirmed
+%if 0%{suse_version} > 1130
+# Define an empty suse_check for compatibility with older sles
+%define suse_check \
+%endif
+
+%define doc_tez %{_docdir}/tez
+%define alternatives_cmd update-alternatives
+%define __os_install_post \
+ %{suse_check} ; \
+ /usr/lib/rpm/brp-compress ; \
+ %{nil}
+
+%endif
+
+Name: tez
+Version: %{tez_version}
+Release: %{tez_release}
+Summary:Apache Tez is the Hadoop enhanced Map/Reduce module.
+URL: http://tez.apache.org
+Group: Development/Libraries
+Buildroot: %{_topdir}/INSTALL/%{name}-%{version}
+License: Apache License v2.0
+Source0: apache-%{name}-%{tez_base_version}-src.tar.gz
+Source1: do-component-build
+Source2: install_tez.sh
+Source3: tez.1
+Source4: tez-site.xml
+BuildArch: noarch
+Requires: hadoop hadoop-hdfs hadoop-yarn hadoop-mapreduce
+
+%if 0%{?mgaversion}
+Requires: bsh-utils
+%else
+Requires: sh-utils
+%endif
+
+
+%description
+The Apache Tez project is aimed at building an application framework
+which allows for a complex directed-acyclic-graph of tasks for
+processing data. It is currently built atop Apache Hadoop YARN
+
+%prep
+%setup -q -n %{name}-%{tez_base_version}
+
+%build
+env TEZ_VERSION=%{version} bash %{SOURCE1}
+
+%install
+%__rm -rf $RPM_BUILD_ROOT
+
+cp %{SOURCE3} %{SOURCE4} .
+sh %{SOURCE2} \
+ --build-dir=. \
+ --doc-dir=%{doc_tez} \
+ --libexec-dir=%{libexec_tez} \
+ --prefix=$RPM_BUILD_ROOT
+
+%pre
+
+%post
+
+%preun
+
+#######################
+#### FILES SECTION ####
+#######################
+%files
+%defattr(-,root,root)
+%{tez_home}
+%doc %{doc_tez}
+%{man_dir}/man1/tez.1.*
+%{tez_home}/tez-*.jar
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f506701c/bigtop.mk
----------------------------------------------------------------------
diff --git a/bigtop.mk b/bigtop.mk
index af9045b..3925992 100644
--- a/bigtop.mk
+++ b/bigtop.mk
@@ -103,6 +103,20 @@ HIVE_SITE=$(APACHE_MIRROR)$(HIVE_DOWNLOAD_PATH)
HIVE_ARCHIVE=$(APACHE_ARCHIVE)$(HIVE_DOWNLOAD_PATH)
$(eval $(call PACKAGE,hive,HIVE))
+# TEZ
+TEZ_NAME=tez
+TEZ_RELNOTES_NAME=Apache TEZ
+TEZ_PKG_NAME=tez
+TEZ_BASE_VERSION=0.6.0
+TEZ_PKG_VERSION=$(TEZ_BASE_VERSION)
+TEZ_RELEASE_VERSION=1
+TEZ_TARBALL_DST=apache-tez-$(TEZ_BASE_VERSION)-src.tar.gz
+TEZ_TARBALL_SRC=$(TEZ_TARBALL_DST)
+TEZ_DOWNLOAD_PATH=/tez/$(TEZ_BASE_VERSION)
+TEZ_SITE=$(APACHE_MIRROR)$(TEZ_DOWNLOAD_PATH)
+TEZ_ARCHIVE=$(APACHE_ARCHIVE)$(TEZ_DOWNLOAD_PATH)
+$(eval $(call PACKAGE,tez,TEZ))
+
# Sqoop
SQOOP_NAME=sqoop
SQOOP_RELNOTES_NAME=Sqoop