You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2016/03/18 03:22:33 UTC
[01/13] bigtop git commit: BIGTOP-2220 : flume-agent.init incorrectly
handles flume.conf [Forced Update!]
Repository: bigtop
Updated Branches:
refs/heads/BIGTOP-2320 77a1f3cb8 -> 18370b22d (forced update)
BIGTOP-2220 : flume-agent.init incorrectly handles flume.conf
Signed-off-by: Youngwoo Kim <yw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/03a3f349
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/03a3f349
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/03a3f349
Branch: refs/heads/BIGTOP-2320
Commit: 03a3f34942cd419f0332e93b376feaf576db42f7
Parents: 1e97841
Author: zenmyo <tz...@gmail.com>
Authored: Thu Jan 7 14:21:11 2016 +0900
Committer: Youngwoo Kim <yw...@apache.org>
Committed: Fri Feb 26 10:21:37 2016 +0900
----------------------------------------------------------------------
bigtop-packages/src/common/flume/flume-agent.init | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/03a3f349/bigtop-packages/src/common/flume/flume-agent.init
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/flume/flume-agent.init b/bigtop-packages/src/common/flume/flume-agent.init
index 1d041a2..ec179c0 100644
--- a/bigtop-packages/src/common/flume/flume-agent.init
+++ b/bigtop-packages/src/common/flume/flume-agent.init
@@ -197,10 +197,14 @@ checkallstatus() {
# Common function to perform user action on all flume conf files
#
run_functions_on_conf() {
+ if [ -f ${FLUME_CONF_DIR}/flume.conf ]; then
+ echo $FLUME_AGENT_NAME
+ $1
+ fi
+ agent_conf_pattern="${FLUME_CONF_DIR}/flume-.*\.conf"
for f in ${FLUME_CONF_DIR}/*
do
- file_ext=${f##*.}
- if [ "$file_ext" = "conf" ]; then
+ if [ $(echo $f | grep -e ${agent_conf_pattern}) ]; then
conf_file=${f%.*}
file_name=${conf_file##*/}
agent_name=${file_name#*-}
[12/13] bigtop git commit: BIGTOP-2323. Build slaves toolchain needs
to be updated for HAWQ
Posted by co...@apache.org.
BIGTOP-2323. Build slaves toolchain needs to be updated for HAWQ
thrift-devel is still an issue for Ubuntu
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b8ab3931
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b8ab3931
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b8ab3931
Branch: refs/heads/BIGTOP-2320
Commit: b8ab3931e7167af5a34d876717195e89e2812fa1
Parents: 8a946b7
Author: Konstantin Boudnik <co...@apache.org>
Authored: Thu Feb 11 19:33:30 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Mar 17 19:20:01 2016 -0700
----------------------------------------------------------------------
bigtop_toolchain/manifests/installer.pp | 2 ++
bigtop_toolchain/manifests/libhdfs.pp | 45 ++++++++++++++++++++++++++++
bigtop_toolchain/manifests/packages.pp | 11 ++++---
bigtop_toolchain/manifests/protobuf.pp | 11 ++++---
4 files changed, 61 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/installer.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/installer.pp b/bigtop_toolchain/manifests/installer.pp
index 246fcaf..060d4bd 100644
--- a/bigtop_toolchain/manifests/installer.pp
+++ b/bigtop_toolchain/manifests/installer.pp
@@ -20,6 +20,8 @@ class bigtop_toolchain::installer {
include bigtop_toolchain::gradle
include bigtop_toolchain::protobuf
include bigtop_toolchain::packages
+## The hack workaround until such times when the lib is embedded into Hawq
+ include bigtop_toolchain::libhdfs
include bigtop_toolchain::env
include bigtop_toolchain::user
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/libhdfs.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/libhdfs.pp b/bigtop_toolchain/manifests/libhdfs.pp
new file mode 100644
index 0000000..4698108
--- /dev/null
+++ b/bigtop_toolchain/manifests/libhdfs.pp
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class bigtop_toolchain::libhdfs {
+
+ include bigtop_toolchain::packages
+
+
+ case $operatingsystem {
+ /(?i:(centos|fedora|amazon))/: {
+
+ $apache_prefix = nearest_apache_mirror()
+ $hackrepourl = "https://bintray.com/artifact/download/wangzw/rpm/centos7/x86_64"
+ $libhdfs = "libhdfs3-2.2.31-1.el7.centos.x86_64.rpm"
+ $libhdfs_dev = "libhdfs3-devel-2.2.31-1.el7.centos.x86_64.rpm"
+
+ exec {"download":
+ path => "/usr/bin",
+ cwd => "/usr/src",
+ command => "curl -L $hackrepourl/$libhdfs -o $libhdfs ; curl -L $hackrepourl/$libhdfs_dev -o $libhdfs_dev",
+ creates => "/usr/src/$libhdfs, /usr/src/$libhdfs_dev",
+ }
+
+ exec {"install":
+ path => "/usr/bin",
+ cwd => "/usr/src",
+ command => "yum install -y $libhdfs $libhdfs_dev",
+ require => [ Exec[ 'download' ], Package[ $packages::pkgs ] ],
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/packages.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp
index 26959e2..edd01da 100644
--- a/bigtop_toolchain/manifests/packages.pp
+++ b/bigtop_toolchain/manifests/packages.pp
@@ -22,16 +22,19 @@ class bigtop_toolchain::packages {
} else {
$mysql_devel="mariadb-devel"
}
- $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
}
- /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
+ $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "readline-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel", "thrift-devel" ]
+ exec { '/usr/bin/yum install -y epel-release':
+ } -> Package <| |>
+ }
+ /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "readline-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
# fix package dependencies: BIGTOP-2120 and BIGTOP-2152
exec { '/usr/bin/zypper remove -y krb5-mini':
} -> exec {'/usr/bin/zypper install -y libopenssl-devel':
} -> Package <| |>
}
- Amazon: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel" ] }
- /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev" ]
+ Amazon: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "gperf", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "readline-devel", "snappy-devel" ] }
+ /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "libcurl4-openssl-dev", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "libapr1-dev", "bison", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libbz2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "gperf", "uuid", "libgsasl7-dev", "libevent-dev", "libjson-c-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libreadline-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev" ]
exec { "apt-update":
command => "/usr/bin/apt-get update"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/protobuf.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/protobuf.pp b/bigtop_toolchain/manifests/protobuf.pp
index c71dec2..53d539b 100644
--- a/bigtop_toolchain/manifests/protobuf.pp
+++ b/bigtop_toolchain/manifests/protobuf.pp
@@ -21,13 +21,15 @@ class bigtop_toolchain::protobuf {
/Ubuntu|Debian/: {
case $architecture {
'amd64' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5585371/+files/"
+ $url1= "http://launchpadlibrarian.net/166035876/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb"
+ $url2= "http://launchpadlibrarian.net/166035877/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb"
$arch= "amd64" }
'ppc64le' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5604345/+files"
$arch= "ppc64el" }
}
}
}
-
+
case $operatingsystem{
/Ubuntu|Debian/: {
$libprotobuf8 = "libprotobuf8_2.5.0-9ubuntu1_$arch.deb"
@@ -35,13 +37,14 @@ class bigtop_toolchain::protobuf {
$protobuf_compiler = "protobuf-compiler_2.5.0-9ubuntu1_$arch.deb"
exec { "download protobuf":
+ path => "/usr/bin",
cwd => "/usr/src",
- command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl -L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler",
- creates => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler" ]
+ command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl -L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler; curl -L $url1 -o libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb; curl -L $url2 -o libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb",
+ creates => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler", "/usr/src/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb", "/usr/src/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb" ]
}
exec { "install protobuf":
cwd => "/usr/src",
- command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler",
+ command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb",
require => EXEC["download protobuf"],
}
}
[07/13] bigtop git commit: Updating latest download link to point to
1.1.0
Posted by co...@apache.org.
Updating latest download link to point to 1.1.0
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/78ca8080
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/78ca8080
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/78ca8080
Branch: refs/heads/BIGTOP-2320
Commit: 78ca80807abd6da4ad81d1aaaa957728f77ca115
Parents: 72ee14c
Author: Roman Shaposhnik <rv...@apache.org>
Authored: Sat Mar 12 19:33:04 2016 -0800
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Sat Mar 12 19:33:04 2016 -0800
----------------------------------------------------------------------
src/site/site.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/78ca8080/src/site/site.xml
----------------------------------------------------------------------
diff --git a/src/site/site.xml b/src/site/site.xml
index ef5b00f..27bb052 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -67,7 +67,7 @@
<menu name="Releases">
<item name="Release Notes" href="release-notes.html"/>
<item name="0.3.0-incubating" href="http://archive.apache.org/dist/incubator/bigtop/bigtop-0.3.0-incubating/"/>
- <item name="1.0.0" href="http://www.apache.org/dyn/closer.lua/bigtop/bigtop-1.0.0/"/>
+ <item name="1.1.0" href="http://www.apache.org/dyn/closer.lua/bigtop/bigtop-1.1.0/"/>
</menu>
<menu name="Project Information">
[10/13] bigtop git commit: BIGTOP-2327. Bump up Phoenix to 4.7.0
Posted by co...@apache.org.
BIGTOP-2327. Bump up Phoenix to 4.7.0
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/374cf9b7
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/374cf9b7
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/374cf9b7
Branch: refs/heads/BIGTOP-2320
Commit: 374cf9b7431b2268e429d406e422586bea93ae7c
Parents: 2805e17
Author: Youngwoo Kim <yw...@apache.org>
Authored: Sun Mar 13 23:18:24 2016 +0900
Committer: Youngwoo Kim <yw...@apache.org>
Committed: Mon Mar 14 20:44:12 2016 +0900
----------------------------------------------------------------------
.../src/common/phoenix/do-component-build | 4 +-
.../src/common/phoenix/install_phoenix.sh | 56 +++-----------------
bigtop-packages/src/deb/phoenix/control | 5 --
.../src/deb/phoenix/phoenix-pherf.install | 1 -
.../src/rpm/phoenix/SPECS/phoenix.spec | 13 -----
bigtop.bom | 2 +-
pom.xml | 2 +-
7 files changed, 12 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/374cf9b7/bigtop-packages/src/common/phoenix/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/phoenix/do-component-build b/bigtop-packages/src/common/phoenix/do-component-build
index 9eaf3bd..1f71864 100644
--- a/bigtop-packages/src/common/phoenix/do-component-build
+++ b/bigtop-packages/src/common/phoenix/do-component-build
@@ -24,7 +24,9 @@ mvn -DskipTests \
-Dpig.version=$PIG_VERSION \
-Dflume.version=$FLUME_VERSION \
-Dspark.version=${SPARK_VERSION} \
- clean install package "$@"
+ clean install "$@"
rm -rf build
mkdir build
tar -C build --strip-components=1 -xzf phoenix-assembly/target/phoenix-$PHOENIX_VERSION.tar.gz
+cp -ra dev/release_files/* build/
+cp -ra phoenix-pherf/config build/bin/
http://git-wip-us.apache.org/repos/asf/bigtop/blob/374cf9b7/bigtop-packages/src/common/phoenix/install_phoenix.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/phoenix/install_phoenix.sh b/bigtop-packages/src/common/phoenix/install_phoenix.sh
index a6ab9f7..b1b25b7 100755
--- a/bigtop-packages/src/common/phoenix/install_phoenix.sh
+++ b/bigtop-packages/src/common/phoenix/install_phoenix.sh
@@ -100,7 +100,6 @@ MAN_DIR=${MAN_DIR:-/usr/share/man/man1}
DOC_DIR=${DOC_DIR:-/usr/share/doc/phoenix}
LIB_DIR=${LIB_DIR:-/usr/lib/phoenix}
BIN_DIR=${BIN_DIR:-/usr/lib/phoenix/bin}
-PHERF_DIR=${PHERF_DIR:-/usr/lib/phoenix/pherf}
ETC_DIR=${ETC_DIR:-/etc/phoenix}
CONF_DIR=${CONF_DIR:-${ETC_DIR}/conf.dist}
@@ -111,18 +110,22 @@ install -d -m 0755 $PREFIX/$DOC_DIR
install -d -m 0755 $PREFIX/$MAN_DIR
install -d -m 0755 $PREFIX/$ETC_DIR
install -d -m 0755 $PREFIX/$CONF_DIR
-install -d -m 0755 $PREFIX/$PHERF_DIR
install -d -m 0755 $PREFIX/var/lib/phoenix
install -d -m 0755 $PREFIX/var/log/phoenix
cp -ra $BUILD_DIR/{bin,lib} $PREFIX/$LIB_DIR/
rm $PREFIX/$LIB_DIR/lib/phoenix-*.jar
-chmod 755 $PREFIX/$BIN_DIR/*.py
cp -a $BUILD_DIR/*.jar $PREFIX/$LIB_DIR/
cp -a $BUILD_DIR/lib/phoenix-*.jar $PREFIX/$LIB_DIR/
+
+chmod 755 $PREFIX/$BIN_DIR/*.py
+chmod 755 $PREFIX/$BIN_DIR/*.sh
+
# Remove sources jar
rm $PREFIX/$LIB_DIR/phoenix-*-sources.jar
-cp -a $BUILD_DIR/examples $PREFIX/$DOC_DIR
+
+cp -a $BUILD_DIR/{LICENSE,NOTICE} $PREFIX/$DOC_DIR/
+cp -ra $BUILD_DIR/examples $PREFIX/$DOC_DIR
# Remove the executable bit from jars to avoid lintian warnings
find $PREFIX/$LIB_DIR -name '*.jar' -exec chmod a-x {} \;
@@ -137,48 +140,3 @@ ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-thin-client.jar` $PREFIX/$LIB_DIR/phoen
# phoenix-server for placing on the HBase regionserver classpath
ln -s `cd $PREFIX/$LIB_DIR ; ls phoenix*-server.jar` $PREFIX/$LIB_DIR/phoenix-server.jar
-# Pherf
-unzip $BUILD_DIR/phoenix-pherf/phoenix-pherf-*-cluster.zip -d $BUILD_DIR/phoenix-pherf/
-mv $BUILD_DIR/phoenix-pherf/phoenix-pherf-*-cluster/* $PREFIX/$PHERF_DIR/
-rm -rf $PREFIX/$PHERF_DIR/config/env.sh
-
-cat >>$PREFIX/$PHERF_DIR/config/env.sh <<EOF
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Required variable to point to Java installation
-#JAVA_HOME=
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
- . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-# Absolute path the the unzipped directory of the HBase installation
-# This is required if you build using the default or cluster profile
-# Cluster profile assumes you want to pick up dependencies from HBase classpath
-# Not required in standalone.
-HBASE_PATH=/usr/lib/hbase
-
-# Add a space seperated list of -D environment args. "-Dkey1-val1 -Dkey2=val2"
-ENV_PROPS=""
-
-# Uncomment if you would like to remotely debug
-#REMOTE_DEBUG="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=6666"
-EOF
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/374cf9b7/bigtop-packages/src/deb/phoenix/control
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/phoenix/control b/bigtop-packages/src/deb/phoenix/control
index 5b0ed7f..4e1208a 100644
--- a/bigtop-packages/src/deb/phoenix/control
+++ b/bigtop-packages/src/deb/phoenix/control
@@ -40,8 +40,3 @@ Description: The Phoenix Query Server provides an alternative means for
interaction with Phoenix and HBase. Soon this will enable access from
environments other than the JVM.
-Package: phoenix-pherf
-Architecture: all
-Depends: phoenix (= ${source:Version})
-Description: Pherf is a standalone tool that can perform performance and functional testing
- through Phoenix
http://git-wip-us.apache.org/repos/asf/bigtop/blob/374cf9b7/bigtop-packages/src/deb/phoenix/phoenix-pherf.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/phoenix/phoenix-pherf.install b/bigtop-packages/src/deb/phoenix/phoenix-pherf.install
deleted file mode 100644
index 1d98a76..0000000
--- a/bigtop-packages/src/deb/phoenix/phoenix-pherf.install
+++ /dev/null
@@ -1 +0,0 @@
-/usr/lib/phoenix/pherf
http://git-wip-us.apache.org/repos/asf/bigtop/blob/374cf9b7/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec b/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
index c6c1273..c52d0c0 100644
--- a/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
+++ b/bigtop-packages/src/rpm/phoenix/SPECS/phoenix.spec
@@ -73,7 +73,6 @@
%endif
-
Name: phoenix
Version: %{phoenix_version}
Release: %{phoenix_release}
@@ -117,15 +116,6 @@ The Phoenix Query Server provides an alternative means for interaction
with Phoenix and HBase. Soon this will enable access from environments
other than the JVM.
-%package pherf
-Summary: Phoenix and HBase SQL Performance Test Framework
-Group: Development/Libraries
-Requires: phoenix = %{version}-%{release}
-
-%description pherf
-Pherf is a standalone tool that can perform performance and functional testing
-through Phoenix
-
%prep
%setup -n %{name}-%{phoenix_base_version}-src
@@ -191,9 +181,6 @@ fi
%config(noreplace) %{etc_phoenix_conf_dist}
%config(noreplace) %{_sysconfdir}/default/phoenix
-%files pherf
-%{phoenix_home}/pherf
-
%define service_macro() \
%files %1 \
%attr(0755,root,root)/%{initd_dir}/%{name}-%1 \
http://git-wip-us.apache.org/repos/asf/bigtop/blob/374cf9b7/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index e3808a5..2a709bf 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -301,7 +301,7 @@ bigtop {
* HBase version in the BOM.
*/
phoenix.hbase ='HBase-0.98'
- version { base = "4.6.0-${phoenix.hbase}"; pkg = '4.6.0'; release = 1 }
+ version { base = "4.7.0-${phoenix.hbase}"; pkg = '4.7.0'; release = 1 }
tarball { destination = "$name-${version.base}-src.tar.gz"
source = destination }
url { download_path = "/$name/$name-${version.base}/src"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/374cf9b7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4c08530..1e3ef63 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,7 +51,7 @@
<solr.version>4.6.0</solr.version>
<spark.version>1.3.1</spark.version>
<kafka.version>0.8.1.1</kafka.version>
- <phoenix.version>4.3.1</phoenix.version>
+ <phoenix.version>4.7.0</phoenix.version>
<kite.version>1.1.0</kite.version>
<spark-smoke.version>${project.version}</spark-smoke.version>
[04/13] bigtop git commit: BIGTOP-2352. Packaged Vagrant box has
incorrect host info on restart
Posted by co...@apache.org.
BIGTOP-2352. Packaged Vagrant box has incorrect host info on restart
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/50783d7c
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/50783d7c
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/50783d7c
Branch: refs/heads/BIGTOP-2320
Commit: 50783d7ceb54ff9afab4bff3df7ccde74b109e44
Parents: 8f0645d
Author: Konstantin Boudnik <co...@boudnik.org>
Authored: Wed Mar 2 09:35:28 2016 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Mar 3 10:14:03 2016 -0800
----------------------------------------------------------------------
bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile | 4 ++
bigtop-deploy/vm/vagrant-puppet-vm/config_hosts | 46 ++++++++++++++++++++
bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh | 44 +++++++++++++++++++
3 files changed, 94 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/50783d7c/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
index ad13f5d..63aff20 100755
--- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
@@ -72,6 +72,10 @@ chkconfig iptables off
# which miss leads some daemons to bind on 127.0.0.1 instead of public or private IP
sed -i "/127.0.0.1/d" /etc/hosts
echo "Bigtop yum repo = #{repo}"
+# Prepare host manipulation files needed in standalone box
+cp /bigtop-home/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts /etc/init.d
+cp /bigtop-home/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh /usr/bin
+chkconfig --add config_hosts
# Prepare puppet configuration file
mkdir -p /etc/puppet/hieradata
cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet
http://git-wip-us.apache.org/repos/asf/bigtop/blob/50783d7c/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts b/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts
new file mode 100755
index 0000000..c584a78
--- /dev/null
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Hosts reconfiguration script
+# chkconfig: 345 70 1
+# description: config_hosts
+
+# Licensed to Hortonworks, Inc. under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. Hortonworks, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+NAME="config_hosts"
+LOG="/var/log/config_hosts.log"
+
+source /etc/rc.d/init.d/functions
+
+case "$1" in
+start)
+ printf "%-50s\n" "Starting $NAME..." | tee -a $LOG
+
+ printf "%-50s\n" "Updating IP..." | tee -a $LOG
+ bash /usr/bin/gen_hosts.sh
+
+;;
+stop)
+ date >> $LOG
+;;
+
+restart)
+;;
+
+*)
+ echo "Usage: $0 {start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/bigtop/blob/50783d7c/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh b/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh
new file mode 100755
index 0000000..c629c85
--- /dev/null
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh
@@ -0,0 +1,44 @@
+# Licensed to Hortonworks, Inc. under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. Hortonworks, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "# Do not remove the following line, or various programs" > /etc/hosts
+echo "# that require network functionality will fail." >> /etc/hosts
+echo "127.0.0.1 localhost.localdomain localhost" >> /etc/hosts
+
+function get_inet_iface(){
+ route | grep default | awk '{print $8}'
+}
+
+function get_ip() {
+ ip addr | grep 'inet ' | grep -E "($(get_inet_iface))" | awk '{ print $2 }' | awk -F'/' '{print $1}'
+}
+
+HOST=$(get_ip)
+NUM=5
+while [ -z "$HOST" ]; do
+ HOST=$(get_ip)
+ sleep 5
+ NUM=$(($NUM-1))
+ if [ $NUM -le 0 ]; then
+ HOST="127.0.0.1"
+ echo "Failed to update IP"
+ break
+ fi
+done
+echo "$HOST `hostname`" >> /etc/hosts
+
+echo 0 > /proc/sys/kernel/hung_task_timeout_secs
+ethtool -K $(get_inet_iface) tso off
[05/13] bigtop git commit: BIGTOP-2285. Add qfs rpm and debian
packaging code
Posted by co...@apache.org.
BIGTOP-2285. Add qfs rpm and debian packaging code
Signed-off-by: Olaf Flebbe <of...@fleb.be>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/3ed2eeea
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/3ed2eeea
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/3ed2eeea
Branch: refs/heads/BIGTOP-2320
Commit: 3ed2eeea8df34915187adb2cbe1caabcd18656df
Parents: 50783d7
Author: Faraaz Sareshwala <fs...@quantcast.com>
Authored: Wed Jan 13 15:29:46 2016 -0800
Committer: Olaf Flebbe <of...@fleb.be>
Committed: Sun Mar 6 11:46:19 2016 +0100
----------------------------------------------------------------------
MAINTAINERS.txt | 1 +
.../src/common/qfs/do-component-build | 133 +++++++
bigtop-packages/src/common/qfs/install_qfs.sh | 237 ++++++++++++
.../src/common/qfs/qfs-chunkserver.svc | 89 +++++
.../src/common/qfs/qfs-metaserver.svc | 89 +++++
bigtop-packages/src/common/qfs/qfs-webui.svc | 89 +++++
bigtop-packages/src/deb/qfs/changelog | 1 +
bigtop-packages/src/deb/qfs/compat | 1 +
bigtop-packages/src/deb/qfs/control | 99 +++++
bigtop-packages/src/deb/qfs/copyright | 15 +
.../src/deb/qfs/qfs-chunkserver.dirs | 2 +
.../src/deb/qfs/qfs-chunkserver.install | 3 +
.../deb/qfs/qfs-chunkserver.lintian-overrides | 2 +
.../src/deb/qfs/qfs-chunkserver.postinst | 35 ++
.../src/deb/qfs/qfs-chunkserver.postrm | 32 ++
.../src/deb/qfs/qfs-chunkserver.preinst | 34 ++
bigtop-packages/src/deb/qfs/qfs-client.install | 33 ++
.../src/deb/qfs/qfs-client.lintian-overrides | 3 +
bigtop-packages/src/deb/qfs/qfs-dev.install | 27 ++
.../src/deb/qfs/qfs-dev.lintian-overrides | 2 +
bigtop-packages/src/deb/qfs/qfs-fuse.install | 1 +
.../src/deb/qfs/qfs-fuse.lintian-overrides | 2 +
bigtop-packages/src/deb/qfs/qfs-hadoop.install | 1 +
.../src/deb/qfs/qfs-hadoop.lintian-overrides | 2 +
bigtop-packages/src/deb/qfs/qfs-java.install | 1 +
.../src/deb/qfs/qfs-java.lintian-overrides | 1 +
bigtop-packages/src/deb/qfs/qfs-metaserver.dirs | 2 +
.../src/deb/qfs/qfs-metaserver.install | 4 +
.../deb/qfs/qfs-metaserver.lintian-overrides | 2 +
.../src/deb/qfs/qfs-metaserver.postinst | 35 ++
.../src/deb/qfs/qfs-metaserver.postrm | 32 ++
.../src/deb/qfs/qfs-metaserver.preinst | 34 ++
bigtop-packages/src/deb/qfs/qfs-python.install | 1 +
.../src/deb/qfs/qfs-python.lintian-overrides | 1 +
bigtop-packages/src/deb/qfs/qfs-webui.dirs | 2 +
bigtop-packages/src/deb/qfs/qfs-webui.install | 2 +
.../src/deb/qfs/qfs-webui.lintian-overrides | 1 +
bigtop-packages/src/deb/qfs/qfs-webui.postinst | 35 ++
bigtop-packages/src/deb/qfs/qfs-webui.postrm | 32 ++
bigtop-packages/src/deb/qfs/qfs-webui.preinst | 34 ++
.../src/deb/qfs/qfs.lintian-overrides | 2 +
bigtop-packages/src/deb/qfs/rules | 81 +++++
bigtop-packages/src/deb/qfs/source/format | 1 +
bigtop-packages/src/rpm/qfs/SPECS/qfs.spec | 360 +++++++++++++++++++
bigtop-packages/src/templates/init.d.tmpl | 8 +-
bigtop.bom | 9 +
bigtop_toolchain/manifests/packages.pp | 6 +-
packages.gradle | 4 +
48 files changed, 1617 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/MAINTAINERS.txt
----------------------------------------------------------------------
diff --git a/MAINTAINERS.txt b/MAINTAINERS.txt
index 4954801..0b1f4a6 100644
--- a/MAINTAINERS.txt
+++ b/MAINTAINERS.txt
@@ -17,6 +17,7 @@ oozie evans ye, rvs
phoenix: andrew purtell, youngwoo kim
pig: evans ye, daniel dai
puppet recipes: jay vyas, cos, evans ye, rvs
+qfs: fsareshwala <fs...@quantcast.com>
smoke-tests: jay vyas, david capwell
spark: jay vyas, youngwoo kim
sqoop: sean mackrory, youngwoo kim
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/common/qfs/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/qfs/do-component-build b/bigtop-packages/src/common/qfs/do-component-build
new file mode 100644
index 0000000..dd269d0
--- /dev/null
+++ b/bigtop-packages/src/common/qfs/do-component-build
@@ -0,0 +1,133 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# For debugging:
+# set -ex
+
+# Autodetect JAVA_HOME if not defined
+source /usr/lib/bigtop-utils/bigtop-detect-javahome
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --qfs-version=VERSION version of qfs we are installing
+ --hadoop-version=VERSION version of hadoop qfs built against
+ --python=PATH path to python
+
+ Optional options:
+ --python3=PATH path to python 3
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'hadoop-version:' \
+ -l 'qfs-version:' \
+ -l 'python:' \
+ -l 'python3:' \
+ -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+ case "$1" in
+ --hadoop-version)
+ HADOOP_VERSION=$2 ; shift 2
+ ;;
+ --qfs-version)
+ QFS_VERSION=$2 ; shift 2
+ ;;
+ --python)
+ PYTHON_PATH=$2 ; shift 2
+ ;;
+ --python3)
+ PYTHON3_PATH=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+for var in QFS_VERSION HADOOP_VERSION PYTHON_PATH ; do
+ if [ -z "$(eval "echo \$$var")" ]; then
+ echo Missing required param: $var
+ usage
+ fi
+done
+
+# bigtop installs maven on its own and not through the package manager
+export PATH=/usr/local/maven/bin:$PATH
+command -v mvn > /dev/null 2>&1 || {
+ echo >&2 "Maven is required but not found in PATH. Check your installation."
+ exit 1
+}
+
+PYTHON3_PATH=${PYTHON3_PATH:-}
+
+# Build and install QFS into the build/release directory
+make
+
+# Build QFS Java library JAR and Hadoop plugin JAR.
+# NOTE(fsareshwala): we have src/java/javabuild.sh to do this for us, but that
+# script does some git magic to determine the release numbers and hashes from
+# git tags. However, in bigtop, we have an entirely different set of git objects
+# and will get the wrong tag name and release hash if we use javabuild.sh as it
+# currently is. Until qfs can update its build process upstream, we replicate
+# what javabuild.sh would do.
+get_hadoop_qfs_profile() {
+ myversion="`echo "$HADOOP_VERSION" | cut -d. -f 1-2`"
+ myversionmaj="`echo "$HADOOP_VERSION" | cut -d. -f 1`"
+
+ if [ x"$myversion" = x"1.0" -o x"$myversion" = x"1.1" ]; then
+ echo "hadoop_branch1_profile"
+ elif [ x"$myversion" = x"0.23" ]; then
+ echo "hadoop_trunk_profile"
+ elif [ x"$myversionmaj" = x"2" ]; then
+ echo "hadoop_trunk_profile,hadoop_trunk_profile_2"
+ else
+ echo "Unsupported Hadoop release version."
+ exit 1
+ fi
+}
+
+HADOOP_QFS_PROFILE=$(get_hadoop_qfs_profile)
+cd src/java
+mvn -Dqfs.release.version=${QFS_VERSION} --projects qfs-access package
+mvn -P $HADOOP_QFS_PROFILE -Dqfs.release.version=${QFS_VERSION} \
+ -Dhadoop.release.version=${HADOOP_VERSION} -Dtest.build.data=/tmp \
+ package
+cd ../..
+
+# Build QFS Python libraries.
+cd build/release
+$PYTHON_PATH ../../src/cc/access/kfs_setup.py build
+if [ ! -z "$PYTHON3_PATH" ]; then
+ $PYTHON3_PATH ../../src/cc/access/kfs_setup.py build
+fi
+cd ..
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/common/qfs/install_qfs.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/qfs/install_qfs.sh b/bigtop-packages/src/common/qfs/install_qfs.sh
new file mode 100644
index 0000000..10ef537
--- /dev/null
+++ b/bigtop-packages/src/common/qfs/install_qfs.sh
@@ -0,0 +1,237 @@
+#!/bin/bash -x
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --prefix=PREFIX path to install into
+ --qfs-version=VERSION version of qfs we are installing
+ --hadoop-version=VERSION version of hadoop qfs built against
+ --python=PYTHON the path to python
+
+ Optional options:
+ --bin-dir=DIR path to install binaries (default: /usr/bin)
+ --lib-dir=DIR path to install libraries (default: /usr/lib)
+ --etc-dir=DIR path to install configuration (default: /etc/qfs)
+ --include-dir=DIR path to install devel headers (default: /usr/include)
+ --data-dir=DIR path to install various data (default: /usr/share/qfs)
+ --var-dir=DIR path to keep variable data (default: /var/qfs)
+ --hadoop-home=DIR path where hadoop lives (default: /usr/lib/hadoop)
+ --python3=PYTHON3 path to python 3
+ --python-extra=PYTHON_EXTRA extra arguments to pass to python when installing
+
+ Note: you will have to change the init scripts with the new paths you use
+ above if you deviate from the defaults.
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'prefix:' \
+ -l 'bin-dir:' \
+ -l 'lib-dir:' \
+ -l 'etc-dir:' \
+ -l 'include-dir:' \
+ -l 'data-dir:' \
+ -l 'var-dir:' \
+ -l 'hadoop-home:' \
+ -l 'qfs-version:' \
+ -l 'hadoop-version:' \
+ -l 'python:' \
+ -l 'python3:' \
+ -l 'python-extra:' \
+ -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+ case "$1" in
+ --prefix)
+ PREFIX=$2 ; shift 2
+ ;;
+ --bin-dir)
+ BIN_DIR=$2 ; shift 2
+ ;;
+ --lib-dir)
+ LIB_DIR=$2 ; shift 2
+ ;;
+ --etc-dir)
+ ETC_DIR=$2 ; shift 2
+ ;;
+ --include-dir)
+ INCLUDE_DIR=$2 ; shift 2
+ ;;
+ --data-dir)
+ DATA_DIR=$2 ; shift 2
+ ;;
+ --var-dir)
+ VAR_DIR=$2 ; shift 2
+ ;;
+ --hadoop-home)
+ HADOOP_HOME=$2 ; shift 2
+ ;;
+ --qfs-version)
+ QFS_VERSION=$2 ; shift 2
+ ;;
+ --hadoop-version)
+ HADOOP_VERSION=$2 ; shift 2
+ ;;
+ --python)
+ PYTHON_PATH=$2 ; shift 2
+ ;;
+ --python3)
+ PYTHON3_PATH=$2 ; shift 2
+ ;;
+ --python-extra)
+ PYTHON_EXTRA=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+for var in PREFIX QFS_VERSION HADOOP_VERSION PYTHON_PATH ; do
+ if [ -z "$(eval "echo \$$var")" ]; then
+ echo Missing required param: $var
+ usage
+ fi
+done
+
+PREFIX=$(readlink -f $PREFIX)
+BIN_DIR=$PREFIX/${BIN_DIR:-/usr/bin/qfs}
+LIB_DIR=$PREFIX/${LIB_DIR:-/usr/lib/qfs}
+ETC_DIR=$PREFIX/${ETC_DIR:-/etc/qfs}
+INCLUDE_DIR=$PREFIX/${INCLUDE_DIR:-/usr/include}
+DATA_DIR=$PREFIX/${DATA_DIR:-/usr/share/qfs}
+VAR_DIR=$PREFIX/${VAR_DIR:-/var/qfs}
+HADOOP_HOME=$PREFIX/${HADOOP_HOME:-/usr/lib/hadoop}
+PYTHON_EXTRA=${PYTHON_EXTRA:-}
+
+mkdir -p $BIN_DIR
+install -m 755 build/release/bin/filelister $BIN_DIR
+install -m 755 build/release/bin/logcompactor $BIN_DIR
+install -m 755 build/release/bin/metaserver $BIN_DIR
+install -m 755 build/release/bin/qfsfsck $BIN_DIR
+install -m 755 build/release/bin/qfs_fuse $BIN_DIR
+install -m 755 build/release/bin/chunkserver $BIN_DIR
+install -m 755 build/release/bin/chunkscrubber $BIN_DIR
+
+install -m 755 build/release/bin/devtools/checksum $BIN_DIR
+install -m 755 build/release/bin/devtools/dirtree_creator $BIN_DIR
+install -m 755 build/release/bin/devtools/dtokentest $BIN_DIR
+install -m 755 build/release/bin/devtools/logger $BIN_DIR/qfslogger
+install -m 755 build/release/bin/devtools/rand-sfmt $BIN_DIR
+install -m 755 build/release/bin/devtools/requestparser $BIN_DIR
+install -m 755 build/release/bin/devtools/sortedhash $BIN_DIR
+install -m 755 build/release/bin/devtools/sslfiltertest $BIN_DIR
+install -m 755 build/release/bin/devtools/stlset $BIN_DIR
+
+install -m 755 build/release/bin/emulator/rebalanceexecutor $BIN_DIR
+install -m 755 build/release/bin/emulator/rebalanceplanner $BIN_DIR
+install -m 755 build/release/bin/emulator/replicachecker $BIN_DIR
+
+install -m 755 build/release/bin/examples/qfssample $BIN_DIR
+
+install -m 755 build/release/bin/tools/cpfromqfs $BIN_DIR
+install -m 755 build/release/bin/tools/cptoqfs $BIN_DIR
+install -m 755 build/release/bin/tools/qfs $BIN_DIR
+install -m 755 build/release/bin/tools/qfsadmin $BIN_DIR
+install -m 755 build/release/bin/tools/qfscat $BIN_DIR
+install -m 755 build/release/bin/tools/qfsdataverify $BIN_DIR
+install -m 755 build/release/bin/tools/qfsfileenum $BIN_DIR
+install -m 755 build/release/bin/tools/qfshibernate $BIN_DIR
+install -m 755 build/release/bin/tools/qfsping $BIN_DIR
+install -m 755 build/release/bin/tools/qfsput $BIN_DIR
+install -m 755 build/release/bin/tools/qfsshell $BIN_DIR
+install -m 755 build/release/bin/tools/qfsstats $BIN_DIR
+install -m 755 build/release/bin/tools/qfstoggleworm $BIN_DIR
+
+install -m 755 scripts/qfs_backup $BIN_DIR
+install -m 755 scripts/qfs_checkpoint_prune.py $BIN_DIR/qfs_checkpoint_prune
+install -m 755 scripts/qfs_log_prune.py $BIN_DIR/qfs_log_prune
+
+mkdir -p $LIB_DIR
+install -m 644 build/release/lib/libgf_complete.so $LIB_DIR
+install -m 644 build/release/lib/libgf_complete.so.1 $LIB_DIR
+install -m 644 build/release/lib/libgf_complete.so.1.0.0 $LIB_DIR
+install -m 644 build/release/lib/libJerasure.so $LIB_DIR
+install -m 644 build/release/lib/libJerasure.so.2 $LIB_DIR
+install -m 644 build/release/lib/libJerasure.so.2.0.0 $LIB_DIR
+install -m 644 build/release/lib/libqfs_access.so $LIB_DIR
+install -m 644 build/release/lib/libqfs_client.so $LIB_DIR
+install -m 644 build/release/lib/libqfs_common.so $LIB_DIR
+install -m 644 build/release/lib/libqfsc.so $LIB_DIR
+install -m 644 build/release/lib/libqfs_io.so $LIB_DIR
+install -m 644 build/release/lib/libqfskrb.so $LIB_DIR
+install -m 644 build/release/lib/libqfs_qcdio.so $LIB_DIR
+install -m 644 build/release/lib/libqfs_qcrs.so $LIB_DIR
+
+install -m 644 build/release/lib/static/libgf_complete.a $LIB_DIR
+install -m 644 build/release/lib/static/libJerasure.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfsc.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfs_client.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfs_common.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfs_emulator.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfs_io.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfskrb.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfs_meta.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfs_qcdio.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfs_qcrs.a $LIB_DIR
+install -m 644 build/release/lib/static/libqfs_tools.a $LIB_DIR
+
+mkdir -p $ETC_DIR/logrotate.d
+install -m 644 contrib/logrotate/qfs-chunkserver $ETC_DIR/logrotate.d
+install -m 644 contrib/logrotate/qfs-metaserver $ETC_DIR/logrotate.d
+install -m 644 contrib/logrotate/qfs-webui $ETC_DIR/logrotate.d
+
+mkdir -p $INCLUDE_DIR
+cp -a build/release/include/* $INCLUDE_DIR
+
+mkdir -p $DATA_DIR/webui
+cp -rp webui/* $DATA_DIR/webui
+
+mkdir -p $DATA_DIR/java
+install -m 644 build/java/qfs-access/qfs-access-${QFS_VERSION}.jar $DATA_DIR/java
+
+mkdir -p $HADOOP_HOME/lib
+install -m 644 build/java/hadoop-qfs/hadoop-${HADOOP_VERSION}-qfs-${QFS_VERSION}.jar ${HADOOP_HOME}/lib
+
+cd build/release
+$PYTHON_PATH ../../src/cc/access/kfs_setup.py install --skip-build --root=$PREFIX --prefix=/usr $PYTHON_EXTRA
+if [ ! -z "$PYTHON3_PATH" ]; then
+ $PYTHON3_PATH ../../src/cc/access/kfs_setup.py install --skip-build --root=$PREFIX --prefix=/usr $PYTHON_EXTRA
+fi
+cd ..
+
+mkdir -p $VAR_DIR/metaserver/checkpoint
+mkdir -p $VAR_DIR/log/qfs
+mkdir -p $VAR_DIR/run/qfs
+mkdir -p $VAR_DIR/lib/qfs
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/common/qfs/qfs-chunkserver.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/qfs/qfs-chunkserver.svc b/bigtop-packages/src/common/qfs/qfs-chunkserver.svc
new file mode 100644
index 0000000..304d9cf
--- /dev/null
+++ b/bigtop-packages/src/common/qfs/qfs-chunkserver.svc
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DAEMON="chunkserver"
+DESC="QFS Chunkserver"
+PROC_NAME="chunkserver"
+EXEC_PATH="/usr/bin/${PROC_NAME}"
+SVC_USER="qfs"
+WORKING_DIR="/var/lib/qfs"
+CONF_DIR="/etc/qfs"
+CONF_FILE="ChunkServer.prp"
+LOGFILE="/var/log/qfs/${DAEMON}.log"
+PIDFILE="/var/run/qfs/${DAEMON}.pid"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+ [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+ log_success_msg "Starting $DESC ($DAEMON): "
+
+ checkstatusofproc
+ status=$?
+ if [ "$status" -eq "$STATUS_RUNNING" ]; then
+ log_success_msg "$DESC is running"
+ exit 0
+ fi
+
+ echo "Starting $DESC: "
+
+ mkdir -p $(dirname $PIDFILE)
+ su -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && nohup $EXEC_PATH $CONF_DIR/$CONF_FILE > $LOGFILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE"
+ sleep 3
+
+ checkstatusofproc
+ RETVAL=$?
+ if [ $RETVAL -eq $STATUS_RUNNING ]; then
+ touch $LOCKFILE
+ log_success_msg "Starting $DESC (${DAEMON}): "
+ else
+ log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL"
+ fi
+ return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+ checkstatusofproc
+ if [ "$?" = "$STATUS_RUNNING" ] ; then
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -n $PID ]; then
+ kill -TERM $PID &>/dev/null
+ fi
+ fi
+ RETVAL=$?
+ else
+ RETVAL=$RETVAL_SUCCESS
+ fi
+
+ if [ $RETVAL -eq $RETVAL_SUCCESS ]; then
+ rm -f $LOCKFILE $PIDFILE
+ log_success_msg "Stopped ${DESC}: "
+ else
+ log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL"
+ fi
+ return $RETVAL
+}
+
+__EOT__
+
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/common/qfs/qfs-metaserver.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/qfs/qfs-metaserver.svc b/bigtop-packages/src/common/qfs/qfs-metaserver.svc
new file mode 100644
index 0000000..69b8170
--- /dev/null
+++ b/bigtop-packages/src/common/qfs/qfs-metaserver.svc
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DAEMON="metaserver"
+DESC="QFS Metaserver"
+PROC_NAME="metaserver"
+EXEC_PATH="/usr/bin/${PROC_NAME}"
+SVC_USER="qfs"
+WORKING_DIR="/var/lib/qfs"
+CONF_DIR="/etc/qfs"
+CONF_FILE="MetaServer.prp"
+LOGFILE="/var/log/qfs/${DAEMON}.log"
+PIDFILE="/var/run/qfs/${DAEMON}.pid"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+ [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+ log_success_msg "Starting $DESC ($DAEMON): "
+
+ checkstatusofproc
+ status=$?
+ if [ "$status" -eq "$STATUS_RUNNING" ]; then
+ log_success_msg "$DESC is running"
+ exit 0
+ fi
+
+ echo "Starting $DESC: "
+
+ mkdir -p $(dirname $PIDFILE)
+ su -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && nohup $EXEC_PATH $CONF_DIR/$CONF_FILE > $LOGFILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE"
+ sleep 3
+
+ checkstatusofproc
+ RETVAL=$?
+ if [ $RETVAL -eq $STATUS_RUNNING ]; then
+ touch $LOCKFILE
+ log_success_msg "Starting $DESC (${DAEMON}): "
+ else
+ log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL"
+ fi
+ return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+ checkstatusofproc
+ if [ "$?" = "$STATUS_RUNNING" ] ; then
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -n $PID ]; then
+ kill -TERM $PID &>/dev/null
+ fi
+ fi
+ RETVAL=$?
+ else
+ RETVAL=$RETVAL_SUCCESS
+ fi
+
+ if [ $RETVAL -eq $RETVAL_SUCCESS ]; then
+ rm -f $LOCKFILE $PIDFILE
+ log_success_msg "Stopped ${DESC}: "
+ else
+ log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL"
+ fi
+ return $RETVAL
+}
+
+__EOT__
+
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/common/qfs/qfs-webui.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/qfs/qfs-webui.svc b/bigtop-packages/src/common/qfs/qfs-webui.svc
new file mode 100644
index 0000000..04add6d
--- /dev/null
+++ b/bigtop-packages/src/common/qfs/qfs-webui.svc
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DAEMON="webui"
+DESC="QFS WebUI"
+PROC_NAME="python"
+EXEC_PATH="/usr/bin/${PROC_NAME} /usr/share/qfs/webui/qfsstatus.py"
+SVC_USER="qfs"
+WORKING_DIR="/var/lib/qfs"
+CONF_DIR="/usr/share/qfs/webui"
+CONF_FILE="server.conf"
+LOGFILE="/var/log/qfs/${DAEMON}.log"
+PIDFILE="/var/run/qfs/${DAEMON}.pid"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+ [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+ log_success_msg "Starting $DESC ($DAEMON): "
+
+ checkstatusofproc
+ status=$?
+ if [ "$status" -eq "$STATUS_RUNNING" ]; then
+ log_success_msg "$DESC is running"
+ exit 0
+ fi
+
+ echo "Starting $DESC: "
+
+ mkdir -p $(dirname $PIDFILE)
+ su -s /bin/bash $SVC_USER -c "cd $WORKING_DIR && nohup $EXEC_PATH $CONF_DIR/$CONF_FILE > $LOGFILE 2>&1 < /dev/null & "'echo $! '"> $PIDFILE"
+ sleep 3
+
+ checkstatusofproc
+ RETVAL=$?
+ if [ $RETVAL -eq $STATUS_RUNNING ]; then
+ touch $LOCKFILE
+ log_success_msg "Starting $DESC (${DAEMON}): "
+ else
+ log_failure_msg "Failure to start $DESC (${DAEMON}). Return value: $RETVAL"
+ fi
+ return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+ checkstatusofproc
+ if [ "$?" = "$STATUS_RUNNING" ] ; then
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -n $PID ]; then
+ kill -TERM $PID &>/dev/null
+ fi
+ fi
+ RETVAL=$?
+ else
+ RETVAL=$RETVAL_SUCCESS
+ fi
+
+ if [ $RETVAL -eq $RETVAL_SUCCESS ]; then
+ rm -f $LOCKFILE $PIDFILE
+ log_success_msg "Stopped ${DESC}: "
+ else
+ log_failure_msg "Failed to stop ${DESC}. Return value: $RETVAL"
+ fi
+ return $RETVAL
+}
+
+__EOT__
+
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/changelog
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/changelog b/bigtop-packages/src/deb/qfs/changelog
new file mode 100644
index 0000000..d4858cd
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/changelog
@@ -0,0 +1 @@
+--- This is auto-generated
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/compat
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/compat b/bigtop-packages/src/deb/qfs/compat
new file mode 100644
index 0000000..ec63514
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/compat
@@ -0,0 +1 @@
+9
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/control
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/control b/bigtop-packages/src/deb/qfs/control
new file mode 100644
index 0000000..2bce55d
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/control
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Source: qfs
+Section: misc
+Priority: extra
+Maintainer: Apache Bigtop <de...@bigtop.apache.org>
+Build-Depends: cmake, libboost-regex-dev, xfslibs-dev, libssl-dev, python2.7-dev | python3-dev, libfuse-dev, dh-python, debhelper (>= 7.0.50~)
+Standards-Version: 3.9.4
+Homepage: https://quantcast.github.io/qfs
+
+Package: qfs
+Depends: qfs-client (= ${binary:Version}), qfs-chunkserver (= ${binary:Version}), qfs-metaserver (= ${binary:Version}), qfs-dev (= ${binary:Version}), qfs-fuse (= ${binary:Version}), qfs-hadoop (= ${binary:Version}), qfs-java (= ${binary:Version}), qfs-python (= ${binary:Version}), qfs-webui (= ${binary:Version})
+Architecture: any
+Description: Metapackage for all QFS components
+ Quantcast File System (QFS) is a high-performance, fault-tolerant, distributed
+ file system developed to support MapReduce processing, or other applications
+ reading and writing large files sequentially.
+
+Package: qfs-client
+Depends: ${shlibs:Depends}, ${misc:Depends}, python
+Architecture: any
+Description: Client binaries and libraries used to link against qfs
+ This package provides base client binaries and libraries used to interact with
+ a running qfs cluster and link against qfs APIs to build your own software that
+ interacts with a running qfs cluster.
+
+Package: qfs-chunkserver
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Recommends: logrotate
+Architecture: any
+Description: Executables required to run the Quantcast File System chunkserver
+ The QFS chunkserver service hosts the binary contents of the QFS distributed
+ filesystem. A metaserver coordinates many data nodes running this service,
+ replicating data amongst chunkservers in a redundant fashion.
+
+Package: qfs-metaserver
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Recommends: logrotate
+Architecture: any
+Description: Executables required to run the Quantcast File System metaserver
+ This package contains the executables required to run the Quantcast File System
+ metaserver service, which tracks the location of data chunks distributed across
+ QFS chunkservers.
+
+Package: qfs-dev
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Architecture: any
+Description: Files needed for building Quantcast File System-based applications
+ The QFS devel package contains the headers, static libraries, and developer
+ tool binaries required to develop applications which build against QFS.
+
+Package: qfs-fuse
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Architecture: any
+Description: Support for mounting the Quantcast File System under FUSE
+ This package contains the qfs_fuse executable which is required when mounting
+ QFS distributed filesystems under FUSE.
+
+Package: qfs-hadoop
+Depends: ${misc:Depends}
+Architecture: any
+Description: Quantcast File System plugin JAR for Hadoop
+ This package contains a plugin JAR to enable QFS to serve as a drop-in
+ replacement for HDFS under Hadoop.
+
+Package: qfs-java
+Depends: ${misc:Depends}
+Architecture: any
+Description: Java libraries for accessing the Quantcast File System
+ This package contains a JAR which enables Java applications to access QFS via
+ its JNI interface.
+
+Package: qfs-python
+Depends: ${shlibs:Depends}, ${misc:Depends}, ${python:Depends}
+Architecture: any
+Description: Python libraries for accessing the Quantcast File System
+ This package contains the libraries required to access the Quantcast File
+ System libraries via Python.
+
+Package: qfs-webui
+Depends: ${misc:Depends}, ${misc:Depends}, python
+Recommends: logrotate
+Architecture: all
+Description: Quantcast File System metaserver/chunkserver web frontend
+ This package contains several Python scripts which provide a simple Hadoop-like
+ Web UI for viewing Quantcast File Server chunkserver and metaserver status.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/copyright
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/copyright b/bigtop-packages/src/deb/qfs/copyright
new file mode 100644
index 0000000..5d02a4e
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/copyright
@@ -0,0 +1,15 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: qfs
+Source: https://quantcast.github.io/qfs
+
+Files: *
+Copyright: 2010-2011, The Apache Software Foundation
+License: Apache-2.0
+
+Files debian/*
+Copyright: 2011, The Apache Software Foundation
+License: Apache-2.0
+
+License: Apache-2.0
+ On Debian systems, the complete text of the Apache 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs b/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs
new file mode 100644
index 0000000..725c19a
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.dirs
@@ -0,0 +1,2 @@
+/var/log/qfs
+/var/lib/qfs
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-chunkserver.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.install b/bigtop-packages/src/deb/qfs/qfs-chunkserver.install
new file mode 100644
index 0000000..9f9390c
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.install
@@ -0,0 +1,3 @@
+/usr/bin/chunkserver
+/usr/bin/chunkscrubber
+/etc/qfs/logrotate.d/qfs-chunkserver
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides
new file mode 100644
index 0000000..ca75f34
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.lintian-overrides
@@ -0,0 +1,2 @@
+qfs-chunkserver binary: new-package-should-close-itp-bug
+qfs-chunkserver binary: binary-without-manpage
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst
new file mode 100644
index 0000000..40b5ea5
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postinst
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ configure)
+ update-rc.d qfs-chunkserver defaults >/dev/null || exit 1
+ chown qfs:qfs -R /var/log/qfs
+ chown qfs:qfs -R /var/run/qfs
+ chown qfs:qfs -R /var/lib/qfs
+ ;;
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm
new file mode 100644
index 0000000..d6098a7
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.postrm
@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ purge)
+ update-rc.d -f qfs-chunkserver remove > /dev/null || exit 1
+ ;;
+ remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
+ ;;
+ *)
+ echo "postrm called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst b/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst
new file mode 100644
index 0000000..7416ecd
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-chunkserver.preinst
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ install|upgrade)
+ getent group qfs >/dev/null || groupadd -r qfs
+ getent passwd qfs > /dev/null || useradd -c "qfs" -s /bin/bash -g qfs \
+ -d /var/lib/qfs qfs 2> /dev/null || :
+ ;;
+ abort-upgrade)
+ ;;
+ *)
+ echo "preinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-client.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-client.install b/bigtop-packages/src/deb/qfs/qfs-client.install
new file mode 100644
index 0000000..225dc0f
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-client.install
@@ -0,0 +1,33 @@
+/usr/bin/qfs
+/usr/bin/qfsadmin
+/usr/bin/qfscat
+/usr/bin/qfsdataverify
+/usr/bin/qfsfileenum
+/usr/bin/qfsfsck
+/usr/bin/qfshibernate
+/usr/bin/qfsping
+/usr/bin/qfsput
+/usr/bin/qfsshell
+/usr/bin/qfsstats
+/usr/bin/qfstoggleworm
+/usr/bin/qfssample
+/usr/bin/qfs_backup
+/usr/bin/qfs_checkpoint_prune
+/usr/bin/qfs_log_prune
+/usr/bin/cpfromqfs
+/usr/bin/cptoqfs
+
+/usr/lib/qfs/libgf_complete.so
+/usr/lib/qfs/libgf_complete.so.1
+/usr/lib/qfs/libgf_complete.so.1.0.0
+/usr/lib/qfs/libJerasure.so
+/usr/lib/qfs/libJerasure.so.2
+/usr/lib/qfs/libJerasure.so.2.0.0
+/usr/lib/qfs/libqfs_access.so
+/usr/lib/qfs/libqfs_client.so
+/usr/lib/qfs/libqfs_common.so
+/usr/lib/qfs/libqfsc.so
+/usr/lib/qfs/libqfs_io.so
+/usr/lib/qfs/libqfskrb.so
+/usr/lib/qfs/libqfs_qcdio.so
+/usr/lib/qfs/libqfs_qcrs.so
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides
new file mode 100644
index 0000000..a694309
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-client.lintian-overrides
@@ -0,0 +1,3 @@
+qfs-client binary: new-package-should-close-itp-bug
+qfs-client binary: binary-or-shlib-defines-rpath
+qfs-client binary: binary-without-manpage
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-dev.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-dev.install b/bigtop-packages/src/deb/qfs/qfs-dev.install
new file mode 100644
index 0000000..8371fa0
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-dev.install
@@ -0,0 +1,27 @@
+/usr/bin/checksum
+/usr/bin/dirtree_creator
+/usr/bin/dtokentest
+/usr/bin/qfslogger
+/usr/bin/rand-sfmt
+/usr/bin/rebalanceexecutor
+/usr/bin/rebalanceplanner
+/usr/bin/replicachecker
+/usr/bin/requestparser
+/usr/bin/sortedhash
+/usr/bin/sslfiltertest
+/usr/bin/stlset
+
+/usr/include
+
+/usr/lib/qfs/libJerasure.a
+/usr/lib/qfs/libgf_complete.a
+/usr/lib/qfs/libqfs_client.a
+/usr/lib/qfs/libqfs_common.a
+/usr/lib/qfs/libqfs_emulator.a
+/usr/lib/qfs/libqfs_io.a
+/usr/lib/qfs/libqfs_meta.a
+/usr/lib/qfs/libqfs_qcdio.a
+/usr/lib/qfs/libqfs_qcrs.a
+/usr/lib/qfs/libqfs_tools.a
+/usr/lib/qfs/libqfsc.a
+/usr/lib/qfs/libqfskrb.a
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides
new file mode 100644
index 0000000..b1ab344
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-dev.lintian-overrides
@@ -0,0 +1,2 @@
+qfs-dev binary: new-package-should-close-itp-bug
+qfs-dev binary: binary-without-manpage
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-fuse.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-fuse.install b/bigtop-packages/src/deb/qfs/qfs-fuse.install
new file mode 100644
index 0000000..35b2e03
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-fuse.install
@@ -0,0 +1 @@
+/usr/bin/qfs_fuse
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides
new file mode 100644
index 0000000..3daf409
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-fuse.lintian-overrides
@@ -0,0 +1,2 @@
+qfs-fuse binary: new-package-should-close-itp-bug
+qfs-fuse binary: binary-without-manpage
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-hadoop.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-hadoop.install b/bigtop-packages/src/deb/qfs/qfs-hadoop.install
new file mode 100644
index 0000000..c8f20de
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-hadoop.install
@@ -0,0 +1 @@
+/usr/lib/hadoop
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides
new file mode 100644
index 0000000..3e40193
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-hadoop.lintian-overrides
@@ -0,0 +1,2 @@
+qfs-hadoop binary: new-package-should-close-itp-bug
+qfs-hadoop binary: jar-not-in-usr-share
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-java.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-java.install b/bigtop-packages/src/deb/qfs/qfs-java.install
new file mode 100644
index 0000000..0fb7754
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-java.install
@@ -0,0 +1 @@
+/usr/share/qfs/java
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides
new file mode 100644
index 0000000..64e9d8a
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-java.lintian-overrides
@@ -0,0 +1 @@
+qfs-java binary: new-package-should-close-itp-bug
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs b/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs
new file mode 100644
index 0000000..725c19a
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.dirs
@@ -0,0 +1,2 @@
+/var/log/qfs
+/var/lib/qfs
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-metaserver.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.install b/bigtop-packages/src/deb/qfs/qfs-metaserver.install
new file mode 100644
index 0000000..c1a57ee
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.install
@@ -0,0 +1,4 @@
+/usr/bin/metaserver
+/usr/bin/filelister
+/usr/bin/logcompactor
+/etc/qfs/logrotate.d/qfs-metaserver
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides
new file mode 100644
index 0000000..a8afec7
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.lintian-overrides
@@ -0,0 +1,2 @@
+qfs-metaserver binary: new-package-should-close-itp-bug
+qfs-metaserver binary: binary-without-manpage
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst b/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst
new file mode 100644
index 0000000..f10e01d
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.postinst
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ configure)
+ update-rc.d qfs-metaserver defaults >/dev/null || exit 1
+ chown qfs:qfs -R /var/log/qfs
+ chown qfs:qfs -R /var/run/qfs
+ chown qfs:qfs -R /var/lib/qfs
+ ;;
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm b/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm
new file mode 100644
index 0000000..63ef9c5
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.postrm
@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ purge)
+ update-rc.d -f qfs-metaserver remove > /dev/null || exit 1
+ ;;
+ remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
+ ;;
+ *)
+ echo "postrm called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst b/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst
new file mode 100644
index 0000000..7416ecd
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-metaserver.preinst
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ install|upgrade)
+ getent group qfs >/dev/null || groupadd -r qfs
+ getent passwd qfs > /dev/null || useradd -c "qfs" -s /bin/bash -g qfs \
+ -d /var/lib/qfs qfs 2> /dev/null || :
+ ;;
+ abort-upgrade)
+ ;;
+ *)
+ echo "preinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-python.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-python.install b/bigtop-packages/src/deb/qfs/qfs-python.install
new file mode 100644
index 0000000..5ad7ef7
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-python.install
@@ -0,0 +1 @@
+/usr/lib/python2.7
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides
new file mode 100644
index 0000000..ea76f7c
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-python.lintian-overrides
@@ -0,0 +1 @@
+qfs-python binary: new-package-should-close-itp-bug
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-webui.dirs
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.dirs b/bigtop-packages/src/deb/qfs/qfs-webui.dirs
new file mode 100644
index 0000000..725c19a
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-webui.dirs
@@ -0,0 +1,2 @@
+/var/log/qfs
+/var/lib/qfs
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-webui.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.install b/bigtop-packages/src/deb/qfs/qfs-webui.install
new file mode 100644
index 0000000..767f4fe
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-webui.install
@@ -0,0 +1,2 @@
+/usr/share/qfs/webui
+/etc/qfs/logrotate.d/qfs-webui
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides
new file mode 100644
index 0000000..d735945
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-webui.lintian-overrides
@@ -0,0 +1 @@
+qfs-webui binary: new-package-should-close-itp-bug
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-webui.postinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.postinst b/bigtop-packages/src/deb/qfs/qfs-webui.postinst
new file mode 100644
index 0000000..0b0cb5c
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-webui.postinst
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ configure)
+ update-rc.d qfs-webui defaults >/dev/null || exit 1
+ chown qfs:qfs -R /var/log/qfs
+ chown qfs:qfs -R /var/run/qfs
+ chown qfs:qfs -R /var/lib/qfs
+ ;;
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-webui.postrm
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.postrm b/bigtop-packages/src/deb/qfs/qfs-webui.postrm
new file mode 100644
index 0000000..10975ff
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-webui.postrm
@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ purge)
+ update-rc.d -f qfs-webui remove > /dev/null || exit 1
+ ;;
+ remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
+ ;;
+ *)
+ echo "postrm called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs-webui.preinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs-webui.preinst b/bigtop-packages/src/deb/qfs/qfs-webui.preinst
new file mode 100644
index 0000000..7416ecd
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs-webui.preinst
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+case "$1" in
+ install|upgrade)
+ getent group qfs >/dev/null || groupadd -r qfs
+ getent passwd qfs > /dev/null || useradd -c "qfs" -s /bin/bash -g qfs \
+ -d /var/lib/qfs qfs 2> /dev/null || :
+ ;;
+ abort-upgrade)
+ ;;
+ *)
+ echo "preinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/qfs.lintian-overrides
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/qfs.lintian-overrides b/bigtop-packages/src/deb/qfs/qfs.lintian-overrides
new file mode 100644
index 0000000..5090488
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/qfs.lintian-overrides
@@ -0,0 +1,2 @@
+qfs binary: new-package-should-close-itp-bug
+qfs binary: empty-binary-package
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/rules b/bigtop-packages/src/deb/qfs/rules
new file mode 100755
index 0000000..56743ea
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/rules
@@ -0,0 +1,81 @@
+#!/usr/bin/make -f
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+export DH_VERBOSE=1
+
+# This has to be exported to make some magic below work.
+export DH_OPTIONS
+
+# Build configuration -- override with environment variables
+SHELL := /bin/bash
+HADOOP_VERSION ?= 1.0.4
+HADOOP_HOME ?= /usr/lib/hadoop
+PYTHON_PATH ?= $(shell command -v python)
+PYTHON3_PATH ?= $(shell command -v python3)
+
+BIN_DIR ?= /usr/bin
+LIB_DIR ?= /usr/lib/qfs
+ETC_DIR ?= /etc/qfs
+INCLUDE_DIR ?= /usr/include
+DATA_DIR ?= /usr/share/qfs
+VAR_DIR ?= /var
+
+SERVICES ?= metaserver chunkserver webui
+
+%:
+ dh $@
+
+override_dh_auto_build:
+ bash debian/do-component-build \
+ --qfs-version=$(QFS_BASE_VERSION) \
+ --hadoop-version=$(HADOOP_VERSION) \
+ --python=$(PYTHON_PATH) \
+ --python3=$(PYTHON3_PATH)
+
+override_dh_auto_install:
+ bash -x debian/install_qfs.sh \
+ --prefix=debian/tmp \
+ --qfs-version=$(QFS_BASE_VERSION) \
+ --hadoop-version=$(HADOOP_VERSION) \
+ --python=$(PYTHON_PATH) \
+ --python3=$(PYTHON3_PATH) \
+ --python-extra='--install-layout=deb' \
+ --bin-dir=$(BIN_DIR) \
+ --lib-dir=$(LIB_DIR) \
+ --etc-dir=$(ETC_DIR) \
+ --include-dir=$(INCLUDE_DIR) \
+ --data-dir=$(DATA_DIR) \
+ --var-dir=$(VAR_DIR)
+
+ for service in $(SERVICES); do \
+ initd_dir=debian/qfs-$$service/etc/init.d ; \
+ mkdir -p $$initd_dir ; \
+ bash debian/init.d.tmpl debian/qfs-$$service.svc deb $$initd_dir/qfs-$$service ; \
+ done
+
+override_dh_shlibdeps:
+ dh_shlibdeps -l$(shell pwd)/build/release/lib
+
+override_dh_fixperms:
+ dh_fixperms
+ # executable-not-elf-or-script
+ # upstream qfs has executable gifs -- later version fixes but hasn't been
+ # released yet
+ chmod -c a-x debian/qfs-webui/usr/share/qfs/webui/files/sorttable/*.gif
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/deb/qfs/source/format
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/qfs/source/format b/bigtop-packages/src/deb/qfs/source/format
new file mode 100644
index 0000000..163aaf8
--- /dev/null
+++ b/bigtop-packages/src/deb/qfs/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec b/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec
new file mode 100644
index 0000000..599c4ed
--- /dev/null
+++ b/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec
@@ -0,0 +1,360 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+%define qfs_name qfs
+
+%define bin_dir /usr/bin
+%define lib_dir /usr/lib/qfs
+%define etc_dir %{_sysconfdir}/qfs
+%define include_dir /usr/include
+%define data_dir /usr/share/qfs
+%define var_dir /var
+%define qfs_services chunkserver metaserver webui
+
+%global HADOOP_VERSION %{hadoop_version}
+
+%if %{?!HADOOP_VERSION:1}0
+%global HADOOP_VERSION 1.0.4
+%endif
+
+%if %{?!HADOOP_HOME:1}0
+%global HADOOP_HOME /usr/lib/hadoop
+%endif
+
+# Required for init.d scripts
+%global initd_dir %{_sysconfdir}/init.d
+%if %{?suse_version:1}0
+Requires: insserv
+%global initd_dir %{_sysconfdir}/rc.d
+%else
+Requires: /lib/lsb/init-functions
+%global initd_dir %{_sysconfdir}/rc.d/init.d
+%endif
+
+Name: qfs
+Summary: Quantcast Filesystem (QFS) Meta Package
+Version: %{qfs_version}
+Release: %{qfs_release}
+URL: https://quantcast.github.io/qfs
+Group: Development/Libraries
+License: ASL 2.0
+
+BuildRequires: boost-devel >= 1.3.4
+BuildRequires: cmake >= 2.4.7
+BuildRequires: fuse-devel
+BuildRequires: gcc-c++
+BuildRequires: krb5-devel
+BuildRequires: libuuid-devel
+BuildRequires: net-tools
+BuildRequires: openssl-devel
+BuildRequires: xfsprogs-devel
+
+Requires: boost >= 1.3.4
+Requires: bigtop-utils
+Requires: openssl
+Requires: qfs-chunkserver
+Requires: qfs-devel
+Requires: qfs-fuse
+Requires: qfs-hadoop
+Requires: qfs-java
+Requires: qfs-metaserver
+Requires: qfs-python
+Requires: qfs-webui
+
+Source0: %{qfs_name}-%{qfs_base_version}.tar.gz
+
+%description
+Quantcast File System (QFS) is a high-performance, fault-tolerant, distributed
+file system developed to support MapReduce processing, or other applications
+reading and writing large files sequentially.
+
+%package client
+Group: Applications/System
+Summary: Client binaries and libraries used to link against qfs
+
+%description client
+This package provides base client binaries and libraries used to interact with a
+running qfs cluster and link against qfs APIs to build your own software that
+interacts with a running qfs cluster.
+
+%package chunkserver
+Group: System Environment/Daemons
+Summary: Executables required to run the Quantcast File System chunkserver
+
+%description chunkserver
+The QFS chunkserver service hosts the binary contents of the QFS distributed
+filesystem. A metaserver coordinates many data nodes running this service,
+replicating data amongst chunkservers in a redundant fashion.
+
+
+%package devel
+Group: Development/Libraries
+Summary: Files needed for building Quantcast File System-based applications
+
+%description devel
+The QFS devel package contains the headers, static libraries, and developer
+tool binaries required to develop applications which build against QFS.
+
+
+%package fuse
+Group: Applications/System
+Summary: Support for mounting the Quantcast File System under FUSE
+
+%description fuse
+This package contains the qfs_fuse executable which is required when mounting
+QFS distributed filesystems under FUSE.
+
+
+%package hadoop
+Group: Development/Libraries
+Summary: Quantcast File System plugin JAR for Hadoop
+
+%description hadoop
+This package contains a plugin JAR to enable QFS to serve as a drop-in
+replacement for HDFS under Hadoop.
+
+
+%package java
+Group: Development/Libraries
+Summary: Java libraries for accessing the Quantcast File System
+
+%description java
+This package contains a JAR which enables Java applications to access QFS via
+its JNI interface.
+
+
+%package metaserver
+Group: System Environment/Daemons
+Summary: Executables required to run the Quantcast File System metaserver
+
+%description metaserver
+This package contains the executables required to run the Quantcast File System
+metaserver service, which tracks the location of data chunks distributed across
+QFS chunkservers.
+
+
+%package python
+Group: Development/Libraries
+Summary: Python libraries for accessing the Quantcast File System
+
+%description python
+This package contains the libraries required to access the Quantcast File
+System libraries via Python.
+
+
+%package webui
+Group: System Environment/Daemons
+Requires: python
+Summary: Quantcast File System metaserver/chunkserver web frontend
+
+%description webui
+This package contains several Python scripts which provide a simple Hadoop-like
+Web UI for viewing Quantcast File Server chunkserver and metaserver status.
+
+%prep
+%setup -n %{qfs_name}-%{qfs_base_version}
+
+echo $RPM_SOURCE_DIR
+echo $RPM_BUILD_ROOT
+exit
+
+%build
+%if 0%{?with_python3}
+PTYHON3_PATH=%{__python3}
+%endif # with_python3
+
+bash $RPM_SOURCE_DIR/do-component-build \
+ --hadoop-version=%{HADOOP_VERSION} \
+ --qfs-version=%{qfs_version} \
+ --python=%{__python} \
+ --python3=$PYTHON3_PATH
+
+%install
+rm -rf $RPM_BUILD_ROOT
+mkdir -p $RPM_BUILD_ROOT
+
+sh $RPM_SOURCE_DIR/install_qfs.sh \
+ --prefix=$RPM_BUILD_ROOT \
+ --hadoop-version=%{HADOOP_VERSION} \
+ --qfs-version=%{qfs_version} \
+ --python=%{__python} \
+ --python3=$PYTHON3_PATH \
+ --bin-dir=%{bin_dir} \
+ --lib-dir=%{lib_dir} \
+ --etc-dir=%{etc_dir} \
+ --include-dir=%{include_dir} \
+ --data-dir=%{data_dir} \
+ --var-dir=%{var_dir}
+
+for service in %{qfs_services}
+do
+ init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{qfs_name}-${service}
+ bash $RPM_SOURCE_DIR/init.d.tmpl $RPM_SOURCE_DIR/%{qfs_name}-${service}.svc rpm $init_file
+done
+
+# NOTE(fsareshwala): %pre sections copy pasted :(
+%pre chunkserver
+getent group qfs >/dev/null || groupadd -r qfs
+getent passwd qfs >/dev/null || useradd -c "qfs" -s /sbin/nologin -g qfs -r \
+ -d %{var_dir}/lib/qfs qfs 2> /dev/null || :
+
+%pre metaserver
+getent group qfs >/dev/null || groupadd -r qfs
+getent passwd qfs >/dev/null || useradd -c "qfs" -s /sbin/nologin -g qfs -r \
+ -d %{var_dir}/lib/qfs qfs 2> /dev/null || :
+
+%pre webui
+getent group qfs >/dev/null || groupadd -r qfs
+getent passwd qfs >/dev/null || useradd -c "qfs" -s /sbin/nologin -g qfs -r \
+ -d %{var_dir}/lib/qfs qfs 2> /dev/null || :
+
+%post chunkserver
+/sbin/chkconfig --add qfs-chunkserver
+
+%preun chunkserver
+if [ $1 -eq 0 ]; then
+ /sbin/service qfs-chunkserver stop &>/dev/null || :
+ /sbin/chkconfig --del qfs-chunkserver
+fi
+
+%post metaserver
+/sbin/chkconfig --add qfs-metaserver
+
+%preun metaserver
+if [ $1 -eq 0 ]; then
+ /sbin/service qfs-metaserver stop &>/dev/null || :
+ /sbin/chkconfig --del qfs-metaserver
+fi
+
+%post webui
+/sbin/chkconfig --add qfs-webui
+
+%preun webui
+if [ $1 -eq 0 ]; then
+ /sbin/service qfs-webui stop &>/dev/null || :
+ /sbin/chkconfig --del qfs-webui
+fi
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files client
+%defattr(-,root,root,755)
+%{bin_dir}/qfs
+%{bin_dir}/qfsadmin
+%{bin_dir}/qfscat
+%{bin_dir}/qfsdataverify
+%{bin_dir}/qfsfileenum
+%{bin_dir}/qfsfsck
+%{bin_dir}/qfshibernate
+%{bin_dir}/qfsping
+%{bin_dir}/qfsput
+%{bin_dir}/qfsshell
+%{bin_dir}/qfsstats
+%{bin_dir}/qfstoggleworm
+%{bin_dir}/qfssample
+%{bin_dir}/qfs_backup
+%{bin_dir}/qfs_checkpoint_prune
+%{bin_dir}/qfs_log_prune
+%{bin_dir}/cpfromqfs
+%{bin_dir}/cptoqfs
+
+%{lib_dir}/libgf_complete.so
+%{lib_dir}/libgf_complete.so.1
+%{lib_dir}/libgf_complete.so.1.0.0
+%{lib_dir}/libJerasure.so
+%{lib_dir}/libJerasure.so.2
+%{lib_dir}/libJerasure.so.2.0.0
+%{lib_dir}/libqfs_access.so
+%{lib_dir}/libqfs_client.so
+%{lib_dir}/libqfs_common.so
+%{lib_dir}/libqfsc.so
+%{lib_dir}/libqfs_io.so
+%{lib_dir}/libqfskrb.so
+%{lib_dir}/libqfs_qcdio.so
+%{lib_dir}/libqfs_qcrs.so
+
+%files metaserver
+%defattr(-,root,root,-)
+%{bin_dir}/metaserver
+%{bin_dir}/filelister
+%{bin_dir}/logcompactor
+%attr(0755,qfs,qfs) %{var_dir}/log/qfs
+%attr(0755,qfs,qfs) %{var_dir}/lib/qfs
+%{initd_dir}/qfs-metaserver
+%config(noreplace) %{etc_dir}/logrotate.d/qfs-metaserver
+
+%files chunkserver
+%defattr(-,root,root,-)
+%{bin_dir}/chunkserver
+%{bin_dir}/chunkscrubber
+%attr(0755,qfs,qfs) %{var_dir}/log/qfs
+%attr(0755,qfs,qfs) %{var_dir}/lib/qfs
+%{initd_dir}/qfs-chunkserver
+%config(noreplace) %{etc_dir}/logrotate.d/qfs-chunkserver
+
+%files devel
+%defattr(-,root,root,-)
+%{bin_dir}/checksum
+%{bin_dir}/dirtree_creator
+%{bin_dir}/dtokentest
+%{bin_dir}/qfslogger
+%{bin_dir}/rand-sfmt
+%{bin_dir}/rebalanceexecutor
+%{bin_dir}/rebalanceplanner
+%{bin_dir}/replicachecker
+%{bin_dir}/requestparser
+%{bin_dir}/sortedhash
+%{bin_dir}/sslfiltertest
+%{bin_dir}/stlset
+
+%{include_dir}/**
+
+%{lib_dir}/libJerasure.a
+%{lib_dir}/libgf_complete.a
+%{lib_dir}/libqfs_client.a
+%{lib_dir}/libqfs_common.a
+%{lib_dir}/libqfs_emulator.a
+%{lib_dir}/libqfs_io.a
+%{lib_dir}/libqfs_meta.a
+%{lib_dir}/libqfs_qcdio.a
+%{lib_dir}/libqfs_qcrs.a
+%{lib_dir}/libqfs_tools.a
+%{lib_dir}/libqfsc.a
+%{lib_dir}/libqfskrb.a
+
+%files fuse
+%defattr(-,root,root,-)
+%{bin_dir}/qfs_fuse
+
+%files hadoop
+%defattr(-,root,root,-)
+%{HADOOP_HOME}/lib/hadoop-%{HADOOP_VERSION}-qfs-%{qfs_version}.jar
+
+%files java
+%defattr(-,root,root,-)
+%{data_dir}/java/qfs-access-%{qfs_version}.jar
+
+%files python
+%defattr(-,root,root,-)
+%{python_sitearch}/**
+
+%files webui
+%defattr(-,root,root,-)
+%{data_dir}/webui
+%attr(0755,qfs,qfs) %{var_dir}/log/qfs
+%attr(0755,qfs,qfs) %{var_dir}/lib/qfs
+%{initd_dir}/qfs-webui
+%config(noreplace) %{etc_dir}/logrotate.d/qfs-webui
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop-packages/src/templates/init.d.tmpl
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/templates/init.d.tmpl b/bigtop-packages/src/templates/init.d.tmpl
index 8391ad5..7971758 100755
--- a/bigtop-packages/src/templates/init.d.tmpl
+++ b/bigtop-packages/src/templates/init.d.tmpl
@@ -131,7 +131,7 @@ generate_extra_commands() {
cat <<'__EOT__'
*)
- echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart}"
+ echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart|force-reload}"
exit 1
__EOT__
@@ -210,7 +210,7 @@ ERROR_PROGRAM_NOT_CONFIGURED=6
RETVAL=0
SLEEP_TIME=5
-PROC_NAME="java"
+PROC_NAME="${PROC_NAME:-java}"
DAEMON="$DAEMON"
DESC="$DESC"
@@ -218,6 +218,8 @@ EXEC_PATH="$EXEC_PATH"
SVC_USER="$SVC_USER"
DAEMON_FLAGS="$DAEMON_FLAGS"
CONF_DIR="$CONF_DIR"
+CONF_FILE="$CONF_FILE"
+LOGFILE="$LOGFILE"
PIDFILE="$PIDFILE"
LOCKDIR="/var/lock/subsys"
LOCKFILE="\$LOCKDIR/$DAEMON"
@@ -291,7 +293,7 @@ service() {
checkstatus
RETVAL=$?
;;
- restart)
+ restart|force-reload)
check_for_root
restart
;;
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index 64e6687..e3808a5 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -422,5 +422,14 @@ bigtop {
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
}
+ 'qfs' {
+ name = 'qfs'
+ relNotes = 'Quantcast Filesystem'
+ version { base = '1.1.4'; pkg = base; release = 1 }
+ tarball { destination = "$name-${version.base}.tar.gz"
+ source = "${version.base}.tar.gz" }
+ url { site = "https://github.com/quantcast/qfs/archive"
+ archive = site }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/bigtop_toolchain/manifests/packages.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp
index f378530..26959e2 100644
--- a/bigtop_toolchain/manifests/packages.pp
+++ b/bigtop_toolchain/manifests/packages.pp
@@ -22,16 +22,16 @@ class bigtop_toolchain::packages {
} else {
$mysql_devel="mariadb-devel"
}
- $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel" ]
+ $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
}
- /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel" ]
+ /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
# fix package dependencies: BIGTOP-2120 and BIGTOP-2152
exec { '/usr/bin/zypper remove -y krb5-mini':
} -> exec {'/usr/bin/zypper install -y libopenssl-devel':
} -> Package <| |>
}
Amazon: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel" ] }
- /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev" ]
+ /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev" ]
exec { "apt-update":
command => "/usr/bin/apt-get update"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3ed2eeea/packages.gradle
----------------------------------------------------------------------
diff --git a/packages.gradle b/packages.gradle
index 44160ef..d36e372 100644
--- a/packages.gradle
+++ b/packages.gradle
@@ -288,6 +288,7 @@ def genTasks = { target ->
def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir
def final BASE_VERSION = config.bigtop.components[target].version.base
def final SRCDEB = "${PKG_NAME}_$PKG_VERSION-${BIGTOP_BUILD_STAMP}.dsc"
+ def final HADOOP_VERSION = config.bigtop.components["hadoop"].version.pkg
exec {
workingDir PKG_OUTPUT_DIR
@@ -300,6 +301,7 @@ def genTasks = { target ->
--preserve-envvar MAVEN3_HOME \
--preserve-envvar MAVEN_OPTS \
--preserve-envvar JAVA_HOME \
+--set-envvar=HADOOP_VERSION=$HADOOP_VERSION \
--set-envvar=${toOldStyleName(target)}_BASE_VERSION=$BASE_VERSION \
--set-envvar=${toOldStyleName(target)}_VERSION=$PKG_VERSION \
--set-envvar=${toOldStyleName(target)}_RELEASE=$BIGTOP_BUILD_STAMP \
@@ -412,12 +414,14 @@ def genTasks = { target ->
def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir
def final PKG_VERSION = config.bigtop.components[target].version.pkg
def final BASE_VERSION = config.bigtop.components[target].version.base
+ def final HADOOP_VERSION = config.bigtop.components["hadoop"].version.pkg
def RELEASE_DIST = "rpmbuild --eval '%{?dist}' 2>/dev/null".execute().text.trim().replaceAll("'",'')
def SRCRPM="$PKG_OUTPUT_DIR/$PKG_NAME-${PKG_VERSION}-$BIGTOP_BUILD_STAMP${RELEASE_DIST}.src.rpm"
def command = [
'--define', "_topdir $PKG_BUILD_DIR/rpm/",
'--define', "${NAME}_base_version $BASE_VERSION",
+ '--define', "hadoop_version ${HADOOP_VERSION}",
'--define', "${NAME}_version ${PKG_VERSION}",
'--define', "${NAME}_release ${BIGTOP_BUILD_STAMP}%{?dist}",
'--rebuild', SRCRPM,
[11/13] bigtop git commit: BIGTOP-2321. Add packaging code for HAWQ
Posted by co...@apache.org.
BIGTOP-2321. Add packaging code for HAWQ
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/8a946b70
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/8a946b70
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/8a946b70
Branch: refs/heads/BIGTOP-2320
Commit: 8a946b7050db980cc36344fe4035bd7b32d11df4
Parents: 374cf9b
Author: Konstantin Boudnik <co...@apache.org>
Authored: Tue Feb 9 08:27:17 2016 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Mar 17 18:52:42 2016 -0700
----------------------------------------------------------------------
.../puppet/modules/hawq/manifests/init.pp | 132 +++++++++++++
.../src/common/hawq/do-component-build | 73 +++++++
bigtop-packages/src/common/hawq/hawq-master.svc | 64 +++++++
.../src/common/hawq/hawq-segment.svc | 65 +++++++
bigtop-packages/src/common/hawq/hawq.1 | 59 ++++++
bigtop-packages/src/common/hawq/hawq.default | 30 +++
bigtop-packages/src/common/hawq/install_hawq.sh | 157 ++++++++++++++++
bigtop-packages/src/deb/hawq/compat | 1 +
bigtop-packages/src/deb/hawq/control | 32 ++++
bigtop-packages/src/deb/hawq/copyright | 15 ++
bigtop-packages/src/deb/hawq/hawq-doc.dirs | 1 +
bigtop-packages/src/deb/hawq/hawq-doc.install | 1 +
bigtop-packages/src/deb/hawq/hawq.dirs | 4 +
bigtop-packages/src/deb/hawq/hawq.install | 3 +
bigtop-packages/src/deb/hawq/hawq.manpages | 1 +
bigtop-packages/src/deb/hawq/hawq.postinst | 47 +++++
bigtop-packages/src/deb/hawq/hawq.preinst | 61 ++++++
bigtop-packages/src/deb/hawq/hawq.prerm | 55 ++++++
bigtop-packages/src/deb/hawq/rules | 55 ++++++
bigtop-packages/src/deb/hawq/source/format | 1 +
bigtop-packages/src/rpm/hawq/BUILD/.gitignore | 0
bigtop-packages/src/rpm/hawq/RPMS/.gitignore | 0
bigtop-packages/src/rpm/hawq/SOURCES/.gitignore | 0
bigtop-packages/src/rpm/hawq/SPECS/hawq.spec | 188 +++++++++++++++++++
bigtop-packages/src/rpm/hawq/SRPMS/.gitignore | 0
bigtop.bom | 10 +
26 files changed, 1055 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/manifests/init.pp b/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
new file mode 100644
index 0000000..3d3d207
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
@@ -0,0 +1,132 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hawq {
+ class deploy ($roles) {
+ if ("hawq" in $roles) {
+ hawq::cluster_node { "hawq-node": }
+ }
+ }
+
+ define cluster_node() {
+ $hadoop_head_node = hiera("bigtop::hadoop_head_node")
+ $hadoop_namenode_port = hiera("hadoop::common_hdfs::hadoop_namenode_port", "8020")
+ $hawq_head = hiera("bigtop::hawq_master_node", "localhost")
+ $hawq_head_port = hiera('bigtop::hawq_master_port', "5432")
+ $hawq_yarn_rm_host = hiera('hadoop::common_yarn::hadoop_rm_host')
+ $hawq_yarn_rm_port = hiera('hadoop::common_yarn::hadoop_rm_port')
+ $hawq_masterdata_dir = hiera('bigtop::hawq_master_datadir', "/var/run/hawq/work/hawq-data-directory/masterdd")
+ $hawq_segmentdata_dir = hiera('bigtop::hawq_segment_datadir', "/hawq/work/hawq-data-directory/segmentdd")
+
+ package { "hawq":
+ ensure => latest,
+ require => Package["libhdfs3-devel"],
+ ## require => for centos this crap needs epel-release
+ }
+
+ file { "/etc/default/hawq":
+ content => template("hawq/hawq.default"),
+ require => Package["hawq"],
+ }
+
+ file { "/etc/hawq/conf":
+ ensure => directory,
+ owner => 'hawq',
+ group => 'hawq',
+ mode => '0755',
+ require => Package["hawq"],
+ }
+ file { "/etc/hawq/conf/hawq-site.xml":
+ content => template('hawq/hawq-site.xml'),
+ require => [File["/etc/hawq/conf"]],
+ owner => 'hawq',
+ group => 'hawq',
+ mode => '0755',
+ }
+ file { "/etc/hawq/conf/gpcheck.cnf":
+ content => template('hawq/gpcheck.cnf'),
+ require => [File["/etc/hawq/conf"]],
+ }
+ file { "/etc/hawq/conf/hdfs-client.xml":
+ content => template('hawq/hdfs-client.xml'),
+ require => [File["/etc/hawq/conf"]],
+ }
+ file { "/etc/hawq/conf/yarn-client.xml":
+ content => template('hawq/yarn-client.xml'),
+ require => [File["/etc/hawq/conf"]],
+ }
+ file { "/etc/hawq/conf/slaves":
+ ensure => file,
+ content => "localhost", ## TODO - this has to be dynamic
+ }
+
+ file { "/etc/sysctl.conf":
+ # TODO overriding sysctl might be a somewhat dangerous, let's figure something better
+ content => template('hawq/sysctl.conf'),
+ }
+ exec { "sysctl reset":
+ path => ['/usr/sbin'],
+ command => 'sysctl -p',
+ require => [ File['/etc/sysctl.conf'] ],
+ }
+
+ exec { "install pygresql modules1":
+ path => ['/usr/bin'],
+ command => 'pip --retries=50 --timeout=300 install pg8000 simplejson unittest2 pygresql pyyaml lockfile paramiko psi',
+ require => [ Package['python-pip', 'postgresql-devel'] ],
+ }
+ exec { "install pygresql modules2":
+ path => ['/usr/bin'],
+ command => 'pip --retries=50 --timeout=300 install http://darcs.idyll.org/~t/projects/figleaf-0.6.1.tar.gz',
+ require => [ Package['python-pip', 'pychecker'], Exec ['install pygresql modules1'] ],
+ ## HAWQ install instructions are suggesting to
+ ## uninstall postgresql postgresql-libs postgresql-devel at this point
+ ## but I don't think it matter, and for sure looks ugly
+ }
+
+ package { "epel-release":
+ ensure => latest,
+ }
+
+ package { "python-pip":
+ ensure => latest,
+ require => [ Package['epel-release'] ],
+ }
+ package { "pychecker":
+ ensure => latest,
+ }
+ package { "postgresql-devel":
+ ensure => latest,
+ }
+ package { "libhdfs3-devel":
+ ensure => latest,
+ }
+
+### TODO init require hdfs to be running. Need to test this
+ exec { "hawk init master":
+ path => ['/usr/bin', '/usr/lib/hawq/bin/lib'],
+ # Silly init will ask if I am really sure I want to init the cluster
+ command => 'bash -x run-init.sh master $GPHOME',
+ require => [ Package['hawq'], Exec ['sysctl reset', 'install pygresql modules2'] ],
+ }
+
+## TODO The expectation is that init will start the service. I don't think so...
+ service { "hawq":
+ ensure => running,
+ require => [ Package["hawq"], File["/etc/default/hawq"], Exec["hawk init master"] ],
+ subscribe => [ Package["hawq"], File["/etc/default/hawq", "/etc/hawq/conf/hawq-site.xml"] ]
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/common/hawq/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/do-component-build b/bigtop-packages/src/common/hawq/do-component-build
new file mode 100755
index 0000000..1c98dd4
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/do-component-build
@@ -0,0 +1,73 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+. `dirname $0`/bigtop.bom
+CWD=`pwd`
+
+# Why? A single script with ksh, which isn't even ksh specific?
+sed -i -e 's#/bin/ksh#/bin/bash#' tools/bin/diskqual.ksh
+
+#docker run --rm=true -v `pwd`:/data -u root rlei/mydocker:latest /bin/sh -c "
+## build libyarn first
+cd depends/libyarn
+mkdir -p build
+cd build
+../bootstrap --prefix=${CWD}/libyarn
+make -j8
+make install
+cd ${CWD}
+ldconfig $CWD/libyarn/lib
+## libyarn should be available in the system at this point
+
+## build libhdfs3
+#wget https://github.com/PivotalRD/libhdfs3/archive/v2.2.31.tar.gz
+#tar zxf v2.2.31.tar.gz
+#cd libhdfs3-2.2.31
+#mkdir -p build
+#cd build
+#../bootstrap
+#make rpm-package
+## need to run diff target for deb and rpm. Figure out how
+
+cd ${CWD}
+# now something like
+# rpm -ivh rpms/RPMS/`uname -p`/*rpm
+## libhdfs3 libhdfs3-devel should be available now
+
+echo "Running the native build"
+date;
+./configure --prefix=${CWD}/hawq \
+ --with-libs=$CWD/libyarn/lib \
+ --with-includes=$CWD/libyarn/include
+if [ $? != 0 ]; then
+ echo HAWQ configure failed.
+ exit 1
+fi
+make -j8
+if [ $? != 0 ]; then
+ echo HAWQ compile failed.
+ exit 1
+fi
+make install
+if [ $? != 0 ]; then
+ echo HAWQ make install failed.
+ exit 1
+fi
+
+cd pxf
+./gradlew distTar
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/common/hawq/hawq-master.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq-master.svc b/bigtop-packages/src/common/hawq/hawq-master.svc
new file mode 100644
index 0000000..4fa5097
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/hawq-master.svc
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DAEMON="hawq-master"
+DESC="master daemon"
+EXEC_PATH="\${HAWQ_HOME}/bin/pg_ctL"
+SVC_USER="hawq"
+PIDFILE="\${HAWQ_PID_DIR}/${DAEMON}.pid"
+SVC_OPTS="-w -t \${HAWQ_TIMEOUT} -m \${HAWQ_SHUTDOWN_MODE} -D \${HAWQ_MASTERDATA_DIR} -l \${HAWQ_LOG_DIR}"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+ [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+ log_success_msg "Starting $DESC (${DAEMON}): "
+
+ checkstatusofproc
+ status=$?
+ if [ "$status" -eq "$STATUS_RUNNING" ]; then
+ log_success_msg "${DESC} is running"
+ exit 0
+ fi
+
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start $SVC_OPTS"
+
+ checkstatusofproc
+ RETVAL=$?
+ [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
+ return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+ log_success_msg "Stopping $DESC (${DAEMON}): "
+
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop $SVC_OPTS"
+
+ sleep 3
+ RETVAL=$?
+
+ [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
+ return $RETVAL
+}
+__EOT__
+
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/common/hawq/hawq-segment.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq-segment.svc b/bigtop-packages/src/common/hawq/hawq-segment.svc
new file mode 100644
index 0000000..b88dc12
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/hawq-segment.svc
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DAEMON="hawq-segment"
+DESC="segment daemon"
+EXEC_PATH="\${HAWQ_HOME}/bin/pg_ctl"
+SVC_USER="hawq"
+PIDFILE="\${HAWQ_PID_DIR}/${DAEMON}.pid"
+SVC_OPTS="-D \${HAWQ_SEGMENTDATA_DIR} -l \${HAWQ_LOG_DIR} -p \${HAWQ_SEGMENT_PORT} \
+ --silent-mode=true -M segment -i"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+ [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+ log_success_msg "Starting $DESC (${DAEMON}): "
+
+ checkstatusofproc
+ status=$?
+ if [ "$status" -eq "$STATUS_RUNNING" ]; then
+ log_success_msg "${DESC} is running"
+ exit 0
+ fi
+
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start $SVC_OPTS"
+
+ checkstatusofproc
+ RETVAL=$?
+ [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
+ return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+ log_success_msg "Stopping $DESC (${DAEMON}): "
+
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop $SVC_OPTS"
+
+ sleep 3
+ RETVAL=$?
+
+ [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
+ return $RETVAL
+}
+__EOT__
+
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/common/hawq/hawq.1
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq.1 b/bigtop-packages/src/common/hawq/hawq.1
new file mode 100644
index 0000000..96c62c0
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/hawq.1
@@ -0,0 +1,59 @@
+.\" Licensed to the Apache Software Foundation (ASF) under one or more
+.\" contributor license agreements. See the NOTICE file distributed with
+.\" this work for additional information regarding copyright ownership.
+.\" The ASF licenses this file to You under the Apache License, Version 2.0
+.\" (the "License"); you may not use this file except in compliance with
+.\" the License. You may obtain a copy of the License at
+.\"
+.\" http://www.apache.org/licenses/LICENSE-2.0
+.\"
+.\" Unless required by applicable law or agreed to in writing, software
+.\" distributed under the License is distributed on an "AS IS" BASIS,
+.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.\" See the License for the specific language governing permissions and
+.\" limitations under the License.
+.\"
+.\" Process this file with
+.\" groff -man -Tascii ignite.1
+.\"
+.TH ignite 1 "Mar 2015 " Linux "User Manuals"
+
+.SH NAME
+Ignite \- Ignite is a filesystem and MR accelerator for Hadoop.
+
+.SH SYNOPSIS
+
+.B ignite
+\fICOMMAND\fR
+
+.SH DESCRIPTION
+
+Ignite is a filesystem and MR accelerator for Hadoop. Use it when you
+you need predictable file system caching, fast MR processing...
+
+Ignite is an open-source, distributed blah-blah
+
+For more information about Ignite, see http://ignite.apache.org/
+
+\fICOMMAND\fR may be one of the following:
+ something do something
+ else do something else
+
+Most commands print help when invoked w/o parameters or with --help.
+
+.SH ENVIRONMENT
+
+.IP JAVA_HOME
+The java implementation to use. Overrides JAVA_HOME.
+
+.IP IGNITE_HOME
+Extra Java CLASSPATH entries.
+
+.IP IGNITE_CONF_DIR
+Alternate conf dir. Default is ${IGNITE_HOME}/conf.
+
+.IP HADOOP_HOME
+Optionally, the Hadoop home to run with.
+
+.SH COPYRIGHT
+2014 - 2015, The Apache Software Foundation
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/common/hawq/hawq.default
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq.default b/bigtop-packages/src/common/hawq/hawq.default
new file mode 100644
index 0000000..f8cf6e1
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/hawq.default
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export HAWQ_HOME="/usr/lib/hawq"
+export HAWQ_CONF_DIR="/etc/hawq/conf"
+
+export HAWQ_PID_DIR="/var/run/hawq"
+export HAWQ_LOG_DIR="/var/log/hawq"
+export HAWQ_WORK_DIR="/var/run/hawq/work"
+export HAWQ_IDENT_STRING=hawq
+
+export PATH=$HAWQ_HOME/bin:$HAWQ_HOME/ext/python/bin:$PATH
+export LD_LIBRARY_PATH=$HAWQ_HOME/lib:$HAWQ_HOME/ext/python/lib:$LD_LIBRARY_PATH
+export PYTHONPATH=$HAWQ_HOME/lib/python:$HAWQ_HOME/lib/python/pygresql:$PYTHONPATH
+export OPENSSL_CONF=$HAWQ_CONF_DIR/openssl.cnf
+export LIBHDFS3_CONF=$HAWQ_CONF_DIR/hdfs-client.xml
+export LIBYARN_CONF=$HAWQ_CONF_DIR/yarn-client.xml
+export HAWQSITE_CONF=$HAWQ_CONF_DIR/hawq-site.xml
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/common/hawq/install_hawq.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/install_hawq.sh b/bigtop-packages/src/common/hawq/install_hawq.sh
new file mode 100755
index 0000000..c12f924
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/install_hawq.sh
@@ -0,0 +1,157 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --build-dir=DIR path to hawq dist.dir
+ --prefix=PREFIX path to install into
+
+ Optional options:
+ --doc-dir=DIR path to install docs into [/usr/share/doc/hawq]
+ --lib-dir=DIR path to install hawq home [/usr/lib/hawq]
+ --installed-lib-dir=DIR path where lib-dir will end up on target system
+ --bin-dir=DIR path to install bins [/usr/bin]
+ --examples-dir=DIR path to install examples [doc-dir/examples]
+ ... [ see source for more similar options ]
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'prefix:' \
+ -l 'doc-dir:' \
+ -l 'lib-dir:' \
+ -l 'installed-lib-dir:' \
+ -l 'bin-dir:' \
+ -l 'examples-dir:' \
+ -l 'conf-dir:' \
+ -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+ case "$1" in
+ --prefix)
+ PREFIX=$2 ; shift 2
+ ;;
+ --build-dir)
+ BUILD_DIR=$2 ; shift 2
+ ;;
+ --doc-dir)
+ DOC_DIR=$2 ; shift 2
+ ;;
+ --lib-dir)
+ LIB_DIR=$2 ; shift 2
+ ;;
+ --installed-lib-dir)
+ INSTALLED_LIB_DIR=$2 ; shift 2
+ ;;
+ --bin-dir)
+ BIN_DIR=$2 ; shift 2
+ ;;
+ --examples-dir)
+ EXAMPLES_DIR=$2 ; shift 2
+ ;;
+ --conf-dir)
+ CONF_DIR=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+for var in PREFIX BUILD_DIR ; do
+ if [ -z "$(eval "echo \$$var")" ]; then
+ echo Missing param: $var
+ usage
+ fi
+done
+
+MAN_DIR=${MAN_DIR:-/usr/share/man/man1}
+DOC_DIR=${DOC_DIR:-/usr/share/doc/hawq}
+LIB_DIR=${LIB_DIR:-/usr/lib/hawq}
+BIN_DIR=${BIN_DIR:-/usr/lib/hawq/bin}
+ETC_DIR=${ETC_DIR:-/etc/hawq}
+CONF_DIR=${CONF_DIR:-${ETC_DIR}/conf.dist}
+
+install -d -m 0755 $PREFIX/$LIB_DIR
+install -d -m 0755 $PREFIX/$LIB_DIR/lib
+install -d -m 0755 $PREFIX/$LIB_DIR/include
+install -d -m 0755 $PREFIX/$LIB_DIR/sbin
+install -d -m 0755 $PREFIX/$LIB_DIR/share
+install -d -m 0755 $PREFIX/$DOC_DIR
+install -d -m 0755 $PREFIX/$BIN_DIR
+install -d -m 0755 $PREFIX/$ETC_DIR
+install -d -m 0755 $PREFIX/$CONF_DIR
+install -d -m 0755 $PREFIX/$MAN_DIR
+install -d -m 0755 $PREFIX/var/run/hawq/work/
+
+UNZIP_DIR=hawq
+cp -ar $UNZIP_DIR/lib $PREFIX/$LIB_DIR
+cp -a $UNZIP_DIR/etc/* $PREFIX/$CONF_DIR
+cp -ra $UNZIP_DIR/bin/* $PREFIX/$BIN_DIR
+cp -ra $UNZIP_DIR/sbin $PREFIX/$LIB_DIR
+cp -ra $UNZIP_DIR/share $PREFIX/$LIB_DIR
+cp -ar $UNZIP_DIR/include $PREFIX/$LIB_DIR
+cp -ar $UNZIP_DIR/docs $PREFIX/$DOC_DIR
+
+cp -ar $UNZIP_DIR/../libyarn/* $PREFIX/$LIB_DIR
+
+#sed -i -e 's#source %s/greenplum_path.sh" % hawq_home#source /etc/default/hawq#' $PREFIX/$BIN_DIR/hawq
+ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/config
+## HAWQ-422 configs are expected to sit in this hardcoded locations ;(
+ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/etc
+# HAWQ-421
+ln -s /etc/default/hawq $PREFIX/$LIB_DIR/greenplum_path.sh
+
+## Put a wrapper to source the defaults and then run init script for particular object
+wrapper=$PREFIX/usr/lib/hawq/bin/lib/run-init.sh
+mkdir -p `dirname $wrapper`
+cat > $wrapper <<EOF
+#!/bin/bash
+
+BIGTOP_DEFAULTS_DIR=\${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hawq ] && . \${BIGTOP_DEFAULTS_DIR}/hawq
+
+bash -x /usr/lib/hawq/bin/lib/hawqinit.sh $* $HAWQ_HOME
+
+EOF
+chmod 755 $wrapper
+
+install -d -m 0755 $PREFIX/usr/bin
+
+install -d -m 0755 $PREFIX/$LIB_DIR/pxf
+UNZIP_DIR=pxf
+PXF_MODULES="pxf-hive pxf-hdfs pxf-service pxf-hbase pxf-api"
+for mod in $PXF_MODULES; do
+ tar zxf $UNZIP_DIR/$mod/build/distributions/*tar.gz -C $PREFIX/$LIB_DIR/pxf/
+done
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/compat
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/compat b/bigtop-packages/src/deb/hawq/compat
new file mode 100644
index 0000000..ec63514
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/compat
@@ -0,0 +1 @@
+9
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/control
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/control b/bigtop-packages/src/deb/hawq/control
new file mode 100644
index 0000000..269a809
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/control
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+Source: hawq
+Section: misc
+Priority: extra
+Maintainer: Bigtop <de...@bigtop.apache.org>
+Build-Depends: debhelper (>= 9)
+Standards-Version: 3.9.4
+Homepage: http://hawq.incubator.apache.org/
+
+Package: hawq
+Architecture: all
+Depends: adduser, hadoop-hdfs, bigtop-utils (>= 1.1.0), bigtop-tomcat
+Description: HAWQ (incubating) is an advanced analytics MPP database
+
+Package: hawq-doc
+Architecture: all
+Section: doc
+Description: HAWQ Documentation
+ Documentation for Apache HAWQ (incubating)
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/copyright
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/copyright b/bigtop-packages/src/deb/hawq/copyright
new file mode 100644
index 0000000..89a5120
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/copyright
@@ -0,0 +1,15 @@
+Format: http://dep.debian.net/deps/dep5
+Source: https://git-wip-us.apache.org/repos/asf/incubator-ignite
+Upstream-Name: none
+
+Files: *
+Copyright: 2014 - 2015, The Apache Software Foundation
+License: Apache-2.0
+
+Files debian/*
+Copyright: 2011, The Apache Software Foundation
+License: Apache-2.0
+
+License: Apache-2.0
+ On Debian systems, the complete text of the Apache 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/hawq-doc.dirs
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq-doc.dirs b/bigtop-packages/src/deb/hawq/hawq-doc.dirs
new file mode 100644
index 0000000..61fe83a
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq-doc.dirs
@@ -0,0 +1 @@
+/usr/share/doc/hawq-docs
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/hawq-doc.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq-doc.install b/bigtop-packages/src/deb/hawq/hawq-doc.install
new file mode 100644
index 0000000..99376a2
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq-doc.install
@@ -0,0 +1 @@
+/usr/share/doc/hawq-doc
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/hawq.dirs
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.dirs b/bigtop-packages/src/deb/hawq/hawq.dirs
new file mode 100644
index 0000000..3487c0d
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.dirs
@@ -0,0 +1,4 @@
+/usr/bin
+/usr/lib/hawq
+/var/log/hawq
+/etc/default
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/hawq.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.install b/bigtop-packages/src/deb/hawq/hawq.install
new file mode 100644
index 0000000..59d4005
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.install
@@ -0,0 +1,3 @@
+/usr/lib/hawq
+/usr/bin/hawq
+/etc/hawq
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/hawq.manpages
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.manpages b/bigtop-packages/src/deb/hawq/hawq.manpages
new file mode 100644
index 0000000..e5d94ec
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.manpages
@@ -0,0 +1 @@
+debian/hawq.1
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/hawq.postinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.postinst b/bigtop-packages/src/deb/hawq/hawq.postinst
new file mode 100644
index 0000000..916a8a4
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.postinst
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# postinst script for hawq
+
+hawq_pkg_name=hawq
+
+set -e
+
+case "$1" in
+ configure)
+ # Install config alternatives
+ update-alternatives --install /etc/${hawq_pkg_name}/conf ${hawq_pkg_name}-conf \
+ /etc/${hawq_pkg_name}/conf.dist 30
+
+ mkdir -p /var/log/${hawq_pkg_name} /var/run/${hawq_pkg_name} /var/run/${hawq_pkg_name}/work || :
+ chown hawq:hawq /var/log/${hawq_pkg_name} /var/run/${hawq_pkg_name}
+ chmod g+w /var/log/${hawq_pkg_name} /var/run/${hawq_pkg_name}
+ chown hawq:hawq /var/run/${hawq_pkg_name} /var/run/${hawq_pkg_name}/work
+ chmod 0755 /var/run/${hawq_pkg_name}
+ chmod 1777 /var/run/${hawq_pkg_name}/work
+ ;;
+
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/hawq.preinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.preinst b/bigtop-packages/src/deb/hawq/hawq.preinst
new file mode 100644
index 0000000..0b36a7a
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.preinst
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# preinst script for hawq
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <new-preinst> `install'
+# * <new-preinst> `install' <old-version>
+# * <new-preinst> `upgrade' <old-version>
+# * <old-preinst> `abort-upgrade' <new-version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ install|upgrade)
+ if ! getent passwd hawq >/dev/null; then
+ # Adding system user: hawq
+ adduser \
+ --system \
+ --group \
+ --home /var/run/hawq \
+ --gecos "HAWQ User" \
+ --shell /bin/bash \
+ hawq >/dev/null
+ fi
+ ;;
+
+ abort-upgrade)
+ ;;
+
+ *)
+ echo "preinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/hawq.prerm
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.prerm b/bigtop-packages/src/deb/hawq/hawq.prerm
new file mode 100644
index 0000000..0bcaf3c
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.prerm
@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# prerm script for hawq
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <prerm> `remove'
+# * <old-prerm> `upgrade' <new-version>
+# * <new-prerm> `failed-upgrade' <old-version>
+# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
+# * <deconfigured's-prerm> `deconfigure' `in-favour'
+# <package-being-installed> <version> `removing'
+# <conflicting-package> <version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ remove|upgrade|deconfigure)
+ update-alternatives --remove hawq-conf /etc/hawq/conf.dist || :
+ ;;
+
+ failed-upgrade)
+ ;;
+
+ *)
+ echo "prerm called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/rules b/bigtop-packages/src/deb/hawq/rules
new file mode 100755
index 0000000..b4a3c9a
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/rules
@@ -0,0 +1,55 @@
+#!/usr/bin/make -f
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -*- makefile -*-
+SHELL=/bin/bash
+
+# Uncomment this to turn on verbose mode.
+export DH_VERBOSE=1
+
+# This has to be exported to make some magic below work.
+export DH_OPTIONS
+
+hawq_pkg_name=hawq
+
+%:
+ dh $@
+
+override_dh_auto_build:
+ bash debian/do-component-build
+
+
+override_dh_auto_install:
+ cp debian/hawq.default debian/${hawq_pkg_name}/etc/default/${hawq_pkg_name}
+ env HADOOP_VERSION=${HAWQ_BASE_VERSION} \
+ bash -x debian/install_hawq.sh \
+ --build-dir=target/bin \
+ --conf-dir=/etc/${hawq_pkg_name}/conf.dist \
+ --doc-dir=usr/share/doc/${hawq_pkg_name}-doc \
+ --prefix=debian/tmp
+ ln -s /var/log/${hawq_pkg_name} debian/tmp/usr/lib/${hawq_pkg_name}/logs
+ ln -s /var/run/${hawq_pkg_name} debian/tmp/usr/lib/${hawq_pkg_name}/pids
+ bash debian/init.d.tmpl debian/hawq-master.svc deb debian/${hawq_pkg_name}-master-service.init
+ bash debian/init.d.tmpl debian/hawq-segment.svc deb debian/${hawq_pkg_name}-segment-service.init
+
+## Let's override the auto_configure and auto_clean to make sure existing
+## top-level Makefile doesn't interfere with the package creation
+override_dh_auto_configure:
+
+override_dh_auto_clean:
+
+override_dh_auto_test:
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/deb/hawq/source/format
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/source/format b/bigtop-packages/src/deb/hawq/source/format
new file mode 100644
index 0000000..163aaf8
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/rpm/hawq/BUILD/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/BUILD/.gitignore b/bigtop-packages/src/rpm/hawq/BUILD/.gitignore
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/rpm/hawq/RPMS/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/RPMS/.gitignore b/bigtop-packages/src/rpm/hawq/RPMS/.gitignore
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/rpm/hawq/SOURCES/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/SOURCES/.gitignore b/bigtop-packages/src/rpm/hawq/SOURCES/.gitignore
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/rpm/hawq/SPECS/hawq.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/SPECS/hawq.spec b/bigtop-packages/src/rpm/hawq/SPECS/hawq.spec
new file mode 100644
index 0000000..0b17e84
--- /dev/null
+++ b/bigtop-packages/src/rpm/hawq/SPECS/hawq.spec
@@ -0,0 +1,188 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+%define etc_hawq_conf %{_sysconfdir}/%{name}/conf
+%define etc_hawq_conf_dist %{etc_hawq_conf}.dist
+%define hawq_home /usr/lib/%{name}
+%define bin_hawq %{hawq_home}/bin
+%define lib_hawq %{hawq_home}/lib
+%define conf_hawq %{hawq_home}/config
+%define logs_hawq %{hawq_home}/logs
+%define pids_hawq %{hawq_home}/pids
+%define man_dir %{_mandir}
+%define hawq_username hawq
+%define vcs_tag incubator-%{name}-HAWQ-307
+
+%define pxf %{hawq_home}/pxf
+
+%if %{?suse_version:1}0
+
+# Only tested on openSUSE 11.4. le'ts update it for previous release when confirmed
+%if 0%{suse_version} > 1130
+%define suse_check \# Define an empty suse_check for compatibility with older sles
+%endif
+
+# SLES is more strict anc check all symlinks point to valid path
+# But we do point to a hadoop jar which is not there at build time
+# (but would be at install time).
+# Since our package build system does not handle dependencies,
+# these symlink checks are deactivated
+%define __os_install_post \
+ %{suse_check} ; \
+ /usr/lib/rpm/brp-compress ; \
+ %{nil}
+
+%define doc_hawq %{_docdir}/%{name}
+%global initd_dir %{_sysconfdir}/rc.d
+%define alternatives_cmd update-alternatives
+
+%else
+
+# CentOS 5 does not have any dist macro
+# So I will suppose anything that is not Mageia or a SUSE will be a RHEL/CentOS/Fedora
+%if %{!?mgaversion:1}0
+
+# FIXME: brp-repack-jars uses unzip to expand jar files
+# Unfortunately guice-2.0.jar pulled by ivy contains some files and directories without any read permission
+# and make whole process to fail.
+# So for now brp-repack-jars is being deactivated until this is fixed.
+# See BIGTOP-294
+#%define __os_install_post \
+# %{_rpmconfigdir}/brp-compress ; \
+# %{_rpmconfigdir}/brp-strip-static-archive %{__strip} ; \
+# %{_rpmconfigdir}/brp-strip-comment-note %{__strip} %{__objdump} ; \
+# /usr/lib/rpm/brp-python-bytecompile ; \
+# %{nil}
+%endif
+
+%define doc_hawq %{_docdir}/%{name}-%{hawq_version}
+%global initd_dir %{_sysconfdir}/rc.d/init.d
+%define alternatives_cmd alternatives
+
+%endif
+
+
+Name: hawq
+Version: %{hawq_version}
+Release: %{hawq_release}
+Summary: Apache Hawq (incubating) is an advanced analytics MPP database
+URL: http://hawq.incubator.apache.org/
+Group: Development/Libraries
+Buildroot: %{_topdir}/INSTALL/%{name}-%{version}
+License: APL2
+Source0: %{vcs_tag}.tar.gz
+Source1: do-component-build
+Source2: install_hawq.sh
+Source3: hawq-master.svc
+Source4: init.d.tmpl
+Source5: hawq.default
+Source6: hawq-segment.svc
+Requires: coreutils, /usr/sbin/useradd, /sbin/chkconfig, /sbin/service
+Requires: hadoop-hdfs, bigtop-utils >= 1.0, bigtop-tomcat
+
+%if 0%{?mgaversion}
+Requires: bsh-utils
+%else
+Requires: sh-utils
+%endif
+
+%description
+Hawq is an open-source, distributed, MPP database engine
+
+%package doc
+Summary: Hawq Documentation
+Group: Documentation
+BuildArch: noarch
+
+%description doc
+Documentation for Hawq platform
+
+%prep
+%setup -n %{vcs_tag}
+
+%build
+bash %{SOURCE1}
+
+%install
+%__rm -rf $RPM_BUILD_ROOT
+env HAWQ_VERSION=%{version} bash %{SOURCE2} \
+ --build-dir=target/bin \
+ --doc-dir=%{doc_hawq} \
+ --conf-dir=%{etc_hawq_conf_dist} \
+ --prefix=$RPM_BUILD_ROOT
+
+%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/
+
+%__install -d -m 0755 $RPM_BUILD_ROOT/etc/default/
+%__install -m 0644 %{SOURCE5} $RPM_BUILD_ROOT/etc/default/%{name}
+
+%__install -d -m 0755 %{buildroot}/%{_localstatedir}/log/%{name}
+
+ln -s %{_localstatedir}/log/%{name} %{buildroot}/%{logs_hawq}
+
+%__install -d -m 0755 %{buildroot}/%{_localstatedir}/run/%{name}
+ln -s %{_localstatedir}/run/%{name} %{buildroot}/%{pids_hawq}
+
+master_init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{name}
+segment_init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{name}-segment
+bash %{SOURCE4} ${RPM_SOURCE_DIR}/%{SOURCE3} rpm $master_init_file
+bash %{SOURCE4} ${RPM_SOURCE_DIR}/%{SOURCE6} rpm $segment_init_file
+chmod 755 $master_init_file $segment_init_file
+
+%__install -d -m 0755 $RPM_BUILD_ROOT/usr/bin
+
+%pre
+getent group hawq 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hawq
+getent passwd hawq 2>&1 > /dev/null || /usr/sbin/useradd -c "hawq" -s /sbin/nologin -g hawq -r -d /var/run/hawq hawq 2> /dev/null || :
+
+%post
+%{alternatives_cmd} --install %{etc_hawq_conf} %{name}-conf %{etc_hawq_conf_dist} 30
+chkconfig --add %{name}
+
+%preun
+if [ "$1" = 0 ]; then
+ %{alternatives_cmd} --remove %{name}-conf %{etc_hawq_conf_dist} || :
+fi
+
+if [ $1 = 0 ] ; then
+ service %{name} stop > /dev/null 2>&1
+ chkconfig --del %{name}
+fi
+
+%postun
+if [ $1 -ge 1 ]; then
+ service %{name} condrestart >/dev/null 2>&1
+fi
+
+
+#######################
+#### FILES SECTION ####
+#######################
+%files
+%defattr(-,hawq,hawq)
+%attr(0755,root,root)/%{initd_dir}/%{name}
+%attr(0755,root,root)/%{initd_dir}/%{name}-segment
+%dir %{_localstatedir}/log/%{name}
+%dir %{_localstatedir}/run/%{name}
+
+%defattr(-,root,root)
+%config(noreplace) %{_sysconfdir}/default/%{name}
+%{hawq_home}
+%config(noreplace) %{etc_hawq_conf_dist}
+
+%files doc
+%defattr(-,root,root)
+%doc %{doc_hawq}
+
+%clean
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop-packages/src/rpm/hawq/SRPMS/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/SRPMS/.gitignore b/bigtop-packages/src/rpm/hawq/SRPMS/.gitignore
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8a946b70/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index 2a709bf..f8a6878 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -431,5 +431,15 @@ bigtop {
url { site = "https://github.com/quantcast/qfs/archive"
archive = site }
}
+ 'hawq' {
+ name = 'hawq'
+ relNotes = 'Apache HAWQ (incubating)'
+ version { base = '2.0.0'; pkg = base; release = 1 }
+ tarball { source = "HAWQ-307.tar.gz"
+ destination = "incubator-${name}-HAWQ-307.tar.gz" }
+ url { download_path = "/apache/incubator-$name/archive"
+ site = "https://github.com/${download_path}"
+ archive = site }
+ }
}
}
[13/13] bigtop git commit: BIGTOP-2325. Deployment recipes for HAWQ
Posted by co...@apache.org.
BIGTOP-2325. Deployment recipes for HAWQ
Initial draft for puppet recipes
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/18370b22
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/18370b22
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/18370b22
Branch: refs/heads/BIGTOP-2320
Commit: 18370b22df9a761287fe4bdbfba5eade90c73455
Parents: b8ab393
Author: Konstantin Boudnik <co...@apache.org>
Authored: Tue Feb 16 21:27:27 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Mar 17 19:20:01 2016 -0700
----------------------------------------------------------------------
.../puppet/hieradata/bigtop/cluster.yaml | 6 +
bigtop-deploy/puppet/manifests/cluster.pp | 4 +
.../puppet/modules/hawq/templates/gpcheck.cnf | 58 ++++
.../puppet/modules/hawq/templates/hawq-site.xml | 158 +++++++++
.../puppet/modules/hawq/templates/hawq.default | 40 +++
.../modules/hawq/templates/hdfs-client.xml | 331 +++++++++++++++++++
.../puppet/modules/hawq/templates/sysctl.conf | 24 ++
.../modules/hawq/templates/yarn-client.xml | 123 +++++++
bigtop-deploy/puppet/modules/hawq/tests/init.pp | 16 +
.../src/common/hadoop/init-hcfs.json | 1 +
bigtop-packages/src/common/hawq/hawq-master.svc | 4 +-
bigtop-packages/src/common/hawq/hawq.default | 10 +
bigtop-packages/src/deb/hawq/rules | 4 +-
bigtop.bom | 2 +-
14 files changed, 776 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index de98502..cf8ffcb 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -174,3 +174,9 @@ zeppelin::server::spark_master_url: "yarn-client"
zeppelin::server::hiveserver2_url: "jdbc:hive2://%{hiera('hadoop-hive::common::hiveserver2_host')}:%{hiera('hadoop-hive::common::hiveserver2_port')}"
zeppelin::server::hiveserver2_user: "%{hiera('bigtop::hiveserver2_user')}"
zeppelin::server::hiveserver2_password: "%{hiera('bigtop::hiveserver2_password')}"
+
+# hawq
+bigtop::hawq_master_node: "%{hiera('bigtop::hawq_master_node')}"
+bigtop::hawq_master_port: "5432"
+bigtop::hawq_master_datadir: "/var/run/hawq/work/masterdd"
+bigtop::hawq_segment_datadir: "/var/run/hawq/work/segmendd"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index a0be567..f80ef5a 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -105,6 +105,9 @@ $roles_map = {
zeppelin => {
master => ["zeppelin-server"],
},
+ hawq => {
+ master => ["hawq"],
+ },
}
class hadoop_cluster_node (
@@ -159,6 +162,7 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_
"hadoop_hive",
"hadoop_oozie",
"hadoop_pig",
+ "hawq",
"sqoop2",
"hadoop_zookeeper",
"hcatalog",
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf b/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf
new file mode 100644
index 0000000..11ae02f
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf
@@ -0,0 +1,58 @@
+[global]
+configfile_version = 4
+
+[linux.mount]
+mount.points = /
+
+[linux.sysctl]
+sysctl.kernel.shmmax = 500000000
+sysctl.kernel.shmmni = 4096
+sysctl.kernel.shmall = 4000000000
+sysctl.kernel.sem = 250 512000 100 2048
+sysctl.kernel.sysrq = 1
+sysctl.kernel.core_uses_pid = 1
+sysctl.kernel.msgmnb = 65536
+sysctl.kernel.msgmax = 65536
+sysctl.kernel.msgmni = 2048
+sysctl.net.ipv4.tcp_syncookies = 0
+sysctl.net.ipv4.ip_forward = 0
+sysctl.net.ipv4.conf.default.accept_source_route = 0
+sysctl.net.ipv4.tcp_tw_recycle = 1
+sysctl.net.ipv4.tcp_max_syn_backlog = 200000
+sysctl.net.ipv4.conf.all.arp_filter = 1
+sysctl.net.ipv4.ip_local_port_range = 1281 65535
+sysctl.net.core.netdev_max_backlog = 200000
+sysctl.vm.overcommit_memory = 2
+sysctl.fs.nr_open = 3000000
+sysctl.kernel.threads-max = 798720
+sysctl.kernel.pid_max = 798720
+# increase network
+sysctl.net.core.rmem_max = 2097152
+sysctl.net.core.wmem_max = 2097152
+
+[linux.limits]
+soft.nofile = 2900000
+hard.nofile = 2900000
+soft.nproc = 131072
+hard.nproc = 131072
+
+[linux.diskusage]
+diskusage.monitor.mounts = /
+diskusage.monitor.usagemax = 90%
+
+[hdfs]
+dfs.mem.namenode.heap = 40960
+dfs.mem.datanode.heap = 6144
+# in hdfs-site.xml
+dfs.support.append = true
+dfs.client.enable.read.from.local = true
+dfs.block.local-path-access.user = gpadmin
+dfs.datanode.max.transfer.threads = 40960
+dfs.client.socket-timeout = 300000000
+dfs.datanode.socket.write.timeout = 7200000
+dfs.namenode.handler.count = 60
+ipc.server.handler.queue.size = 3300
+dfs.datanode.handler.count = 60
+ipc.client.connection.maxidletime = 3600000
+dfs.namenode.accesstime.precision = -1
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml b/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml
new file mode 100644
index 0000000..713fa40
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml
@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<configuration>
+ <property>
+ <name>hawq_master_address_host</name>
+ <value><%= @hawq_head %></value>
+ <description>The host name of hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_address_port</name>
+ <value><%= @hawq_head_port %></value>
+ <description>The port of hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_standby_address_host</name>
+ <value>none</value>
+ <description>The host name of hawq standby master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_address_port</name>
+ <value>40000</value>
+ <description>The port of hawq segment.</description>
+ </property>
+
+ <property>
+ <name>hawq_dfs_url</name>
+ <value><%= @hadoop_head_node %>:<%= @hadoop_namenode_port %>/hawq_default</value>
+ <description>URL for accessing HDFS.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_directory</name>
+ <value><% @hawq_masterdata_dir %></value>
+ <description>The directory of hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_directory</name>
+ <value><% @hawq_segmentdata_dir %>d</value>
+ <description>The directory of hawq segment.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_temp_directory</name>
+ <value>/tmp</value>
+ <description>The temporary directory reserved for hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_temp_directory</name>
+ <value>/tmp</value>
+ <description>The temporary directory reserved for hawq segment.</description>
+ </property>
+
+ <!-- HAWQ resource manager parameters -->
+ <property>
+ <name>hawq_global_rm_type</name>
+ <value>none</value>
+ <description>The resource manager type to start for allocating resource.
+ 'none' means hawq resource manager exclusively uses whole
+ cluster; 'yarn' means hawq resource manager contacts YARN
+ resource manager to negotiate resource.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_memory_limit_perseg</name>
+ <value>64GB</value>
+ <description>The limit of memory usage in a hawq segment when
+ hawq_global_rm_type is set 'none'.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_nvcore_limit_perseg</name>
+ <value>16</value>
+ <description>The limit of virtual core usage in a hawq segment when
+ hawq_global_rm_type is set 'none'.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_address</name>
+ <value><%= @hawq_yarn_rm_host %>:<%= @hawq_yarn_rm_port %></value>
+ <description>The address of YARN resource manager server.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_scheduler_address</name>
+ <value>localhost:8030</value>
+ <description>The address of YARN scheduler server.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_queue_name</name>
+ <value>default</value>
+ <description>The YARN queue name to register hawq resource manager.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_app_name</name>
+ <value>hawq</value>
+ <description>The application name to register hawq resource manager in YARN.</description>
+ </property>
+ <!-- HAWQ resource manager parameters end here. -->
+
+ <!-- HAWQ resource enforcement parameters -->
+ <property>
+ <name>hawq_re_cpu_enable</name>
+ <value>false</value>
+ <description>The control to enable/disable CPU resource enforcement.</description>
+ </property>
+
+ <property>
+ <name>hawq_re_cgroup_mount_point</name>
+ <value>/sys/fs/cgroup</value>
+ <description>The mount point of CGroup file system for resource enforcement.
+ For example, /sys/fs/cgroup/cpu/hawq for CPU sub-system.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_re_cgroup_hierarchy_name</name>
+ <value>hawq</value>
+ <description>The name of the hierarchy to accomodate CGroup directories/files for resource enforcement.
+ For example, /sys/fs/cgroup/cpu/hawq for CPU sub-system.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_nvseg_perquery_perseg_limit</name>
+ <value>8</value>
+ <description>This is something that init tries to push in</description>
+ </property>
+ <!-- HAWQ resource enforcement parameters end here. -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hawq.default b/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
new file mode 100644
index 0000000..8190260
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export HAWQ_HOME="/usr/lib/hawq"
+## Due to some weird scripting, hawq doesn't work without GPHOME
+export GPHOME=$HAWQ_HOME
+export HAWQ_CONF_DIR="/etc/hawq/conf"
+
+export HAWQ_PID_DIR="/var/run/hawq"
+export HAWQ_LOG_DIR="/var/log/hawq"
+export HAWQ_WORK_DIR="/var/run/hawq/work"
+export HAWQ_IDENT_STRING=hawq
+
+export PATH=$HAWQ_HOME/bin:$HAWQ_HOME/ext/python/bin:$PATH
+export LD_LIBRARY_PATH=$HAWQ_HOME/lib:$HAWQ_HOME/ext/python/lib:$LD_LIBRARY_PATH
+export PYTHONPATH=$HAWQ_HOME/lib/python:$HAWQ_HOME/lib/python/pygresql:$PYTHONPATH
+export OPENSSL_CONF=$HAWQ_CONF_DIR/openssl.cnf
+export LIBHDFS3_CONF=$HAWQ_CONF_DIR/hdfs-client.xml
+export LIBYARN_CONF=$HAWQ_CONF_DIR/yarn-client.xml
+export HAWQSITE_CONF=$HAWQ_CONF_DIR/hawq-site.xml
+
+export HAWQ_MASTER_PORT=5432
+export HAWQ_SEGMENT_PORT=40000
+export HAWQ_MASTERDATA_DIR=<%= @hawq_masterdata_dir %>
+export HAWQ_SEGMENTDATA_DIR=<%= @hawq_segmentdata_dir %>
+
+export HAWQ_TIMEOUT=30
+export HAWQ_SHUTDOWN_MODE=smart
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml b/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml
new file mode 100644
index 0000000..3f08696
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml
@@ -0,0 +1,331 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<configuration>
+
+ <!-- KDC
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+ KDC -->
+
+ <!-- HA
+ <property>
+ <name>dfs.nameservices</name>
+ <value>phdcluster</value>
+ </property>
+
+ <property>
+ <name>dfs.ha.namenodes.phdcluster</name>
+ <value>nn1,nn2</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.rpc-address.phdcluster.nn1</name>
+ <value>mdw:9000</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.rpc-address.phdcluster.nn2</name>
+ <value>smdw:9000</value>
+ </property>
+
+<property>
+<name>dfs.namenode.http-address.phdcluster.nn1</name>
+<value>mdw:50070</value>
+</property>
+
+<property>
+<name>dfs.namenode.http-address.phdcluster.nn2</name>
+<value>smdw:50070</value>
+</property>
+
+HA -->
+
+ <!-- RPC client configuration -->
+ <property>
+ <name>rpc.client.timeout</name>
+ <value>3600000</value>
+ <description>
+ timeout interval of a RPC invocation in millisecond. default is 3600000.
+ </description>
+ </property>
+ <property>
+ <name>rpc.client.connect.tcpnodelay</name>
+ <value>true</value>
+ <description>
+ whether set socket TCP_NODELAY to true when connect to RPC server. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.max.idle</name>
+ <value>10000</value>
+ <description>
+ the max idle time of a RPC connection in millisecond. default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.ping.interval</name>
+ <value>10000</value>
+ <description>
+ the interval which the RPC client send a heart beat to server. 0 means disable, default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to setup the connection. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.retry</name>
+ <value>10</value>
+ <description>
+ the max retry times if the RPC client fail to setup the connection to server. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to read from server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to write to server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.socket.linger.timeout</name>
+ <value>-1</value>
+ <description>
+ set value to socket SO_LINGER when connect to RPC server. -1 means default OS value. default is -1.
+ </description>
+ </property>
+
+ <!-- dfs client configuration -->
+ <property>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>true</value>
+ <description>
+ whether reading block file bypass datanode if the block and the client are on the same node. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.default.replica</name>
+ <value>3</value>
+ <description>
+ the default number of replica. default is 3.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.prefetchsize</name>
+ <value>10</value>
+ <description>
+ the default number of blocks which information will be prefetched. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.failover.max.attempts</name>
+ <value>15</value>
+ <description>
+ if multiply namenodes are configured, it is the max retry times when the dfs client try to issue a RPC call. default is 15.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.default.blocksize</name>
+ <value>134217728</value>
+ <description>
+ default block size. default is 134217728.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.log.severity</name>
+ <value>INFO</value>
+ <description>
+ the minimal log severity level, valid values include FATAL, ERROR, INFO, DEBUG1, DEBUG2, DEBUG3. default is INFO.
+ </description>
+ </property>
+
+ <!-- input client configuration -->
+ <property>
+ <name>input.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the input stream is trying to setup the connection to datanode. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the input stream is trying to read from datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the input stream is trying to write to datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.localread.default.buffersize</name>
+ <value>2097152</value>
+ <description>
+ number of bytes of the buffer which is used to hold the data from block file and verify checksum.
+ it is only used when "dfs.client.read.shortcircuit" is set to true. default is 1048576.
+ </description>
+ </property>
+
+ <property>
+ <name>input.localread.blockinfo.cachesize</name>
+ <value>1000</value>
+ <description>
+ the size of block file path information cache. default is 1000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.read.getblockinfo.retry</name>
+ <value>3</value>
+ <description>
+ the max retry times when the client fail to get block information from namenode. default is 3.
+ </description>
+ </property>
+
+ <!-- output client configuration -->
+ <property>
+ <name>output.replace-datanode-on-failure</name>
+ <value>false</value>
+ <description>
+ whether the client add new datanode into pipeline if the number of nodes in pipeline is less the specified number of replicas. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>output.default.chunksize</name>
+ <value>512</value>
+ <description>
+ the number of bytes of a chunk in pipeline. default is 512.
+ </description>
+ </property>
+
+ <property>
+ <name>output.default.packetsize</name>
+ <value>65536</value>
+ <description>
+ the number of bytes of a packet in pipeline. default is 65536.
+ </description>
+ </property>
+
+ <property>
+ <name>output.default.write.retry</name>
+ <value>10</value>
+ <description>
+ the max retry times when the client fail to setup the pipeline. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>output.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the output stream is trying to setup the connection to datanode. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>output.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the output stream is trying to read from datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>output.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the output stream is trying to write to datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>output.packetpool.size</name>
+ <value>1024</value>
+ <description>
+ the max number of packets in a file's packet pool. default is 1024.
+ </description>
+ </property>
+
+ <property>
+ <name>output.close.timeout</name>
+ <value>900000</value>
+ <description>
+ the timeout interval in millisecond when close an output stream. default is 900000.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ <description>
+ Optional. This is a path to a UNIX domain socket that will be used for
+ communication between the DataNode and local HDFS clients.
+ If the string "_PORT" is present in this path, it will be replaced by the
+ TCP port of the DataNode.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.use.legacy.blockreader.local</name>
+ <value>false</value>
+ <description>
+ Legacy short-circuit reader implementation based on HDFS-2246 is used
+ if this configuration parameter is true.
+ This is for the platforms other than Linux
+ where the new implementation based on HDFS-347 is not available.
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-deploy/puppet/modules/hawq/templates/sysctl.conf
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/sysctl.conf b/bigtop-deploy/puppet/modules/hawq/templates/sysctl.conf
new file mode 100644
index 0000000..2ac1598
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/sysctl.conf
@@ -0,0 +1,24 @@
+kernel.shmmax = 1000000000
+kernel.shmmni = 4096
+kernel.shmall = 4000000000
+kernel.sem = 250 512000 100 2048
+kernel.sysrq = 1
+kernel.core_uses_pid = 1
+kernel.msgmnb = 65536
+kernel.msgmax = 65536
+kernel.msgmni = 2048
+net.ipv4.tcp_syncookies = 0
+net.ipv4.ip_forward = 0
+net.ipv4.conf.default.accept_source_route = 0
+net.ipv4.tcp_tw_recycle = 1
+net.ipv4.tcp_max_syn_backlog = 200000
+net.ipv4.conf.all.arp_filter = 1
+net.ipv4.ip_local_port_range = 1281 65535
+net.core.netdev_max_backlog = 200000
+vm.overcommit_memory = 2
+fs.nr_open = 3000000
+kernel.threads-max = 798720
+kernel.pid_max = 798720
+# increase network
+net.core.rmem_max=2097152
+net.core.wmem_max=2097152
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml b/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml
new file mode 100644
index 0000000..a5f9df7
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<configuration>
+
+ <!-- KDC
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+ KDC -->
+
+ <!-- HA
+ <property>
+ <name>yarn.resourcemanager.ha</name>
+ <value>%RESOURCEMANAGER%:8032,%RESOURCEMANAGER2%:8032</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.ha</name>
+ <value>%RESOURCEMANAGER%:8030,%RESOURCEMANAGER2%:8030</value>
+ </property>
+ HA -->
+
+ <!-- RPC client configuration -->
+ <property>
+ <name>rpc.client.timeout</name>
+ <value>3600000</value>
+ <description>
+ timeout interval of a RPC invocation in millisecond. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.tcpnodelay</name>
+ <value>true</value>
+ <description>
+ whether set socket TCP_NODELAY to true when connect to RPC server. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.max.idle</name>
+ <value>10000</value>
+ <description>
+ the max idle time of a RPC connection in millisecond. default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.ping.interval</name>
+ <value>10000</value>
+ <description>
+ the interval which the RPC client send a heart beat to server. 0 means disable, default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to setup the connection. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.retry</name>
+ <value>10</value>
+ <description>
+ the max retry times if the RPC client fail to setup the connection to server. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to read from server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to write to server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.socket.linger.timeout</name>
+ <value>-1</value>
+ <description>
+ set value to socket SO_LINGER when connect to RPC server. -1 means default OS value. default is -1.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.client.failover.max.attempts</name>
+ <value>15</value>
+ <description>
+ if multiply resource managers are configured, it is the max retry times when the yarn client try to issue a RPC call. default is 15.
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-deploy/puppet/modules/hawq/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/tests/init.pp b/bigtop-deploy/puppet/modules/hawq/tests/init.pp
new file mode 100644
index 0000000..37a5a08
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/tests/init.pp
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hawq::cluster_node { "test-hawq-node": }
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-packages/src/common/hadoop/init-hcfs.json
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.json b/bigtop-packages/src/common/hadoop/init-hcfs.json
index f71f385..dea5f90 100644
--- a/bigtop-packages/src/common/hadoop/init-hcfs.json
+++ b/bigtop-packages/src/common/hadoop/init-hcfs.json
@@ -64,6 +64,7 @@
["/tmp/hadoop-yarn","777","mapred","mapred"],
["/var/log/hadoop-yarn/apps","1777","yarn","mapred"],
["/hbase",null,"hbase","hbase"],
+ ["/hawq_default","755","hawq","hawq"],
["/solr",null,"solr","solr"],
["/benchmarks","777",null,null],
["/user","755","HCFS_SUPER_USER",null],
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-packages/src/common/hawq/hawq-master.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq-master.svc b/bigtop-packages/src/common/hawq/hawq-master.svc
index 4fa5097..e5aca0d 100644
--- a/bigtop-packages/src/common/hawq/hawq-master.svc
+++ b/bigtop-packages/src/common/hawq/hawq-master.svc
@@ -34,7 +34,7 @@ start() {
exit 0
fi
- su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start $SVC_OPTS"
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start master $SVC_OPTS"
checkstatusofproc
RETVAL=$?
@@ -51,7 +51,7 @@ cat <<'__EOT__'
stop() {
log_success_msg "Stopping $DESC (${DAEMON}): "
- su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop $SVC_OPTS"
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop master $SVC_OPTS"
sleep 3
RETVAL=$?
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-packages/src/common/hawq/hawq.default
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq.default b/bigtop-packages/src/common/hawq/hawq.default
index f8cf6e1..574e845 100644
--- a/bigtop-packages/src/common/hawq/hawq.default
+++ b/bigtop-packages/src/common/hawq/hawq.default
@@ -21,6 +21,8 @@ export HAWQ_LOG_DIR="/var/log/hawq"
export HAWQ_WORK_DIR="/var/run/hawq/work"
export HAWQ_IDENT_STRING=hawq
+export HAWQ_DATA_DIR=$HAWQ_WORK_DIR/hawq-data-directory
+
export PATH=$HAWQ_HOME/bin:$HAWQ_HOME/ext/python/bin:$PATH
export LD_LIBRARY_PATH=$HAWQ_HOME/lib:$HAWQ_HOME/ext/python/lib:$LD_LIBRARY_PATH
export PYTHONPATH=$HAWQ_HOME/lib/python:$HAWQ_HOME/lib/python/pygresql:$PYTHONPATH
@@ -28,3 +30,11 @@ export OPENSSL_CONF=$HAWQ_CONF_DIR/openssl.cnf
export LIBHDFS3_CONF=$HAWQ_CONF_DIR/hdfs-client.xml
export LIBYARN_CONF=$HAWQ_CONF_DIR/yarn-client.xml
export HAWQSITE_CONF=$HAWQ_CONF_DIR/hawq-site.xml
+
+export HAWQ_MASTER_PORT=5432
+export HAWQ_SEGMENT_PORT=40000
+export HAWQ_MASTERDATA_DIR=$HAWQ_DATA_DIR/masterdd
+export HAWQ_SEGMENTDATA_DIR=$HAWQ_DATA_DIR/segmentdd
+
+export HAWQ_TIMEOUT=30
+export HAWQ_SHUTDOWN_MODE=smart
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop-packages/src/deb/hawq/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/rules b/bigtop-packages/src/deb/hawq/rules
index b4a3c9a..1e256a1 100755
--- a/bigtop-packages/src/deb/hawq/rules
+++ b/bigtop-packages/src/deb/hawq/rules
@@ -30,7 +30,7 @@ hawq_pkg_name=hawq
dh $@
override_dh_auto_build:
- bash debian/do-component-build
+ bash debian/do-component-build
override_dh_auto_install:
@@ -46,7 +46,7 @@ override_dh_auto_install:
bash debian/init.d.tmpl debian/hawq-master.svc deb debian/${hawq_pkg_name}-master-service.init
bash debian/init.d.tmpl debian/hawq-segment.svc deb debian/${hawq_pkg_name}-segment-service.init
-## Let's override the auto_configure and auto_clean to make sure existing
+## Let's override the auto_configure and auto_clean to make sure existing
## top-level Makefile doesn't interfere with the package creation
override_dh_auto_configure:
http://git-wip-us.apache.org/repos/asf/bigtop/blob/18370b22/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index f8a6878..5670546 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -90,7 +90,7 @@ bigtop {
zookeeper:['hadoop', 'hbase'],
hadoop:['ignite-hadoop', 'hbase', 'crunch', 'pig', 'hive', 'tez', 'sqoop', 'sqoop2',
'oozie', 'mahout', 'flume', 'giraph', 'solr', 'crunch', 'spark',
- 'phoenix', 'tachyon', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin',
+ 'phoenix', 'tachyon', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin', 'hawq',
'tajo', 'apex'
],
hbase:['phoenix','giraph','ycsb'],
[08/13] bigtop git commit: BIGTOP-2350: Fix the location of the
HCAT_PREFIX variable as well as remove an incorrect & unecessary define of
HADOOP_PREFIX
Posted by co...@apache.org.
BIGTOP-2350: Fix the location of the HCAT_PREFIX variable as well as remove an incorrect & unecessary define of HADOOP_PREFIX
Signed-off-by: Youngwoo Kim <yw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/16068110
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/16068110
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/16068110
Branch: refs/heads/BIGTOP-2320
Commit: 160681109cc7f1d464fddc0b3609f3e7de0f8186
Parents: 78ca808
Author: Rob Leidle <le...@amazon.com>
Authored: Fri Jan 15 09:54:56 2016 -0800
Committer: Youngwoo Kim <yw...@apache.org>
Committed: Mon Mar 14 02:42:27 2016 +0900
----------------------------------------------------------------------
bigtop-packages/src/common/hive/hive-webhcat-server.default | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/16068110/bigtop-packages/src/common/hive/hive-webhcat-server.default
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hive/hive-webhcat-server.default b/bigtop-packages/src/common/hive/hive-webhcat-server.default
index f9cd31c..9ab1d2e 100644
--- a/bigtop-packages/src/common/hive/hive-webhcat-server.default
+++ b/bigtop-packages/src/common/hive/hive-webhcat-server.default
@@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-export HCAT_PREFIX=/usr
-export HADOOP_PREFIX=/usr
+export HCAT_PREFIX=/usr/lib/hive-hcatalog
export WEBHCAT_LOG_DIR=/var/log/hive-hcatalog
export WEBHCAT_PID_DIR=/var/lib/hive-hcatalog
# FIXME: HCATALOG-636
[06/13] bigtop git commit: BIGTOP-2353: fix qfs suse build
Posted by co...@apache.org.
BIGTOP-2353: fix qfs suse build
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/72ee14cd
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/72ee14cd
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/72ee14cd
Branch: refs/heads/BIGTOP-2320
Commit: 72ee14cdb8565eb2749092cf70d3ba107f924ade
Parents: 3ed2eee
Author: Olaf Flebbe <of...@ofleb.be>
Authored: Sun Mar 6 11:23:59 2016 +0100
Committer: Olaf Flebbe <of...@fleb.be>
Committed: Sun Mar 6 18:51:02 2016 +0100
----------------------------------------------------------------------
.../src/common/qfs/patch1-findsusekrb5config.diff | 11 +++++++++++
bigtop-packages/src/rpm/qfs/SPECS/qfs.spec | 3 ++-
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/72ee14cd/bigtop-packages/src/common/qfs/patch1-findsusekrb5config.diff
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/qfs/patch1-findsusekrb5config.diff b/bigtop-packages/src/common/qfs/patch1-findsusekrb5config.diff
new file mode 100644
index 0000000..d1e12dc
--- /dev/null
+++ b/bigtop-packages/src/common/qfs/patch1-findsusekrb5config.diff
@@ -0,0 +1,11 @@
+--- qfs-1.1.4/cmake/Modules/FindKrb5.cmake.old 2016-03-06 11:17:38.041431320 +0100
++++ qfs-1.1.4/cmake/Modules/FindKrb5.cmake 2016-03-06 11:18:38.165433125 +0100
+@@ -25,7 +25,7 @@
+ NO_DEFAULT_PATH
+ )
+ ENDIF(KRB5_PREFIX)
+-FIND_PROGRAM(KRB5_C_CONFIG NAMES krb5-config)
++FIND_PROGRAM(KRB5_C_CONFIG NAMES krb5-config PATHS /usr/lib/mit/bin)
+
+ MESSAGE(STATUS "found krb5-config here ${KRB5_C_CONFIG}")
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/72ee14cd/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec b/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec
index 599c4ed..2001d46 100644
--- a/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec
+++ b/bigtop-packages/src/rpm/qfs/SPECS/qfs.spec
@@ -74,6 +74,7 @@ Requires: qfs-python
Requires: qfs-webui
Source0: %{qfs_name}-%{qfs_base_version}.tar.gz
+#BIGTOP_PATCH_FILES
%description
Quantcast File System (QFS) is a high-performance, fault-tolerant, distributed
@@ -168,8 +169,8 @@ Web UI for viewing Quantcast File Server chunkserver and metaserver status.
echo $RPM_SOURCE_DIR
echo $RPM_BUILD_ROOT
-exit
+#BIGTOP_PATCH_COMMANDS
%build
%if 0%{?with_python3}
PTYHON3_PATH=%{__python3}
[09/13] bigtop git commit: BIGTOP-2358. Update the URL for
Bigtop-trunk-packages job on README
Posted by co...@apache.org.
BIGTOP-2358. Update the URL for Bigtop-trunk-packages job on README
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/2805e17e
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/2805e17e
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/2805e17e
Branch: refs/heads/BIGTOP-2320
Commit: 2805e17e077b7b6d8f894ba18da734e812ffd509
Parents: 1606811
Author: Youngwoo Kim <yw...@apache.org>
Authored: Sun Mar 13 23:54:05 2016 +0900
Committer: Youngwoo Kim <yw...@apache.org>
Committed: Mon Mar 14 20:37:33 2016 +0900
----------------------------------------------------------------------
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/2805e17e/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 51ec455..13e2f5a 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,7 @@ rules will be used for the CTR process:
there're doubts in the approach taken, design decision, or implementation
details
* a committer should keep an eye on the official CI builds at
- http://ci.bigtop.apache.org/job/Bigtop-trunk/ (Bigtop-trunk-packages builds)
+ https://ci.bigtop.apache.org/view/Packages/job/Bigtop-trunk-packages/ (Bigtop-trunk-packages builds)
to make sure that committed changes haven't break anything. In
which case the committer should take a timely effort to resolve the issues
and unblock the others in the community
[03/13] bigtop git commit: BIGTOP-2351. Vagrant VM is missing
/user/vagrant HDFS directory
Posted by co...@apache.org.
BIGTOP-2351. Vagrant VM is missing /user/vagrant HDFS directory
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/8f0645d1
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/8f0645d1
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/8f0645d1
Branch: refs/heads/BIGTOP-2320
Commit: 8f0645d1138ae39f1ebe08d8653ce8c577a9cc25
Parents: 027037e
Author: Konstantin Boudnik <co...@boudnik.org>
Authored: Wed Mar 2 16:49:24 2016 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Mar 3 10:13:57 2016 -0800
----------------------------------------------------------------------
bigtop-packages/src/common/hadoop/init-hcfs.json | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8f0645d1/bigtop-packages/src/common/hadoop/init-hcfs.json
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.json b/bigtop-packages/src/common/hadoop/init-hcfs.json
index 7f9b6cc..f71f385 100644
--- a/bigtop-packages/src/common/hadoop/init-hcfs.json
+++ b/bigtop-packages/src/common/hadoop/init-hcfs.json
@@ -69,6 +69,7 @@
["/user","755","HCFS_SUPER_USER",null],
["/user/history","755","mapred","mapred"],
["/user/jenkins","777","jenkins",null],
+ ["/user/vagrant","777","vagrant",null],
["/user/hive","777",null,null],
["/user/root","777","root",null],
["/user/hue","777","hue","hue"],
[02/13] bigtop git commit: BIGTOP-1843: Upgrade Sqoop to 1.4.6
Posted by co...@apache.org.
BIGTOP-1843: Upgrade Sqoop to 1.4.6
Signed-off-by: Youngwoo Kim <yw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/027037e6
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/027037e6
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/027037e6
Branch: refs/heads/BIGTOP-2320
Commit: 027037e6330c8269ca20d1424ff3d02967218c03
Parents: 03a3f34
Author: Craig Foster <fo...@amazon.com>
Authored: Wed Feb 10 10:32:27 2016 -0800
Committer: Youngwoo Kim <yw...@apache.org>
Committed: Wed Mar 2 20:45:23 2016 +0900
----------------------------------------------------------------------
bigtop.bom | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/027037e6/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index 363b3fc..64e6687 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -185,7 +185,7 @@ bigtop {
'sqoop' {
name = 'sqoop'
relNotes = 'Apache Sqoop v1'
- version { base = '1.4.5'; pkg = base; release = 1 }
+ version { base = '1.4.6'; pkg = base; release = 1 }
tarball { destination = "${name}-${version.base}.tar.gz"
source = destination }
url { download_path = "/$name/${version.base}/"