You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2016/03/09 02:31:41 UTC
[1/3] bigtop git commit: BIGTOP-2321. Add packaging code for HAWQ
[Forced Update!]
Repository: bigtop
Updated Branches:
refs/heads/BIGTOP-2320 403e61db9 -> 7b1aed9d5 (forced update)
BIGTOP-2321. Add packaging code for HAWQ
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b909af42
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b909af42
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b909af42
Branch: refs/heads/BIGTOP-2320
Commit: b909af4213fb6935d14c7ce4c6dc08673a63aec3
Parents: 1e97841
Author: Konstantin Boudnik <co...@apache.org>
Authored: Tue Feb 9 08:27:17 2016 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Mon Mar 7 19:58:52 2016 -0800
----------------------------------------------------------------------
.../src/common/hawq/do-component-build | 58 ++++++
bigtop-packages/src/common/hawq/hawq-master.svc | 63 +++++++
.../src/common/hawq/hawq-segment.svc | 64 +++++++
bigtop-packages/src/common/hawq/hawq.1 | 59 ++++++
bigtop-packages/src/common/hawq/hawq.default | 30 +++
bigtop-packages/src/common/hawq/install_hawq.sh | 155 +++++++++++++++
bigtop-packages/src/deb/hawq/compat | 1 +
bigtop-packages/src/deb/hawq/control | 32 ++++
bigtop-packages/src/deb/hawq/copyright | 15 ++
bigtop-packages/src/deb/hawq/hawq-doc.dirs | 1 +
bigtop-packages/src/deb/hawq/hawq-doc.install | 1 +
bigtop-packages/src/deb/hawq/hawq.dirs | 4 +
bigtop-packages/src/deb/hawq/hawq.install | 3 +
bigtop-packages/src/deb/hawq/hawq.manpages | 1 +
bigtop-packages/src/deb/hawq/hawq.postinst | 47 +++++
bigtop-packages/src/deb/hawq/hawq.preinst | 61 ++++++
bigtop-packages/src/deb/hawq/hawq.prerm | 55 ++++++
bigtop-packages/src/deb/hawq/rules | 55 ++++++
bigtop-packages/src/deb/hawq/source/format | 1 +
bigtop-packages/src/rpm/hawq/BUILD/.gitignore | 0
bigtop-packages/src/rpm/hawq/RPMS/.gitignore | 0
bigtop-packages/src/rpm/hawq/SOURCES/.gitignore | 0
bigtop-packages/src/rpm/hawq/SPECS/hawq.spec | 189 +++++++++++++++++++
bigtop-packages/src/rpm/hawq/SRPMS/.gitignore | 0
bigtop.bom | 10 +
25 files changed, 905 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/common/hawq/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/do-component-build b/bigtop-packages/src/common/hawq/do-component-build
new file mode 100755
index 0000000..ce2d23a
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/do-component-build
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+. `dirname $0`/bigtop.bom
+CWD=`pwd`
+
+# Why? A single script with ksh, which isn't even ksh specific?
+sed -i -e 's#/bin/ksh#/bin/bash#' tools/bin/diskqual.ksh
+
+#docker run --rm=true -v `pwd`:/data -u root rlei/mydocker:latest /bin/sh -c "
+## build libyarn first
+cd depends/libyarn
+mkdir -p build
+cd build
+../bootstrap --prefix=${CWD}/libyarn
+make -j8
+make install
+cd ${CWD}
+ldconfig $CWD/libyarn/lib
+## libyarn should be available in the system at this point
+
+echo "Running the native build"
+date;
+./configure --prefix=${CWD}/hawq \
+ --with-libs=$CWD/libyarn/lib \
+ --with-includes=$CWD/libyarn/include
+if [ $? != 0 ]; then
+ echo HAWQ configure failed.
+ exit 1
+fi
+make -j8
+if [ $? != 0 ]; then
+ echo HAWQ compile failed.
+ exit 1
+fi
+make install
+if [ $? != 0 ]; then
+ echo HAWQ make install failed.
+ exit 1
+fi
+
+cd pxf
+./gradlew distTar
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/common/hawq/hawq-master.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq-master.svc b/bigtop-packages/src/common/hawq/hawq-master.svc
new file mode 100644
index 0000000..c306a7f
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/hawq-master.svc
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DAEMON="hawq master"
+DESC="hawq master daemon"
+EXEC_PATH="\${HAWQ_HOME}/bin/hawq"
+SVC_USER="hawq"
+PIDFILE="\${HAWQ_PID_DIR}/${DAEMON}.pid"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+ [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+ log_success_msg "Starting $DESC (${DAEMON}): "
+
+ checkstatusofproc
+ status=$?
+ if [ "$status" -eq "$STATUS_RUNNING" ]; then
+ log_success_msg "${DESC} is running"
+ exit 0
+ fi
+
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start ${DAEMON}"
+
+ checkstatusofproc
+ RETVAL=$?
+ [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
+ return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+ log_success_msg "Stopping $DESC (${DAEMON}): "
+
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop ${DAEMON}"
+
+ sleep 3
+ RETVAL=$?
+
+ [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
+ return $RETVAL
+}
+__EOT__
+
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/common/hawq/hawq-segment.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq-segment.svc b/bigtop-packages/src/common/hawq/hawq-segment.svc
new file mode 100644
index 0000000..0b4acab
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/hawq-segment.svc
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DAEMON="hawq segment"
+DESC="hawq segment daemon"
+EXEC_PATH="\${HAWQ_HOME}/bin/hawq"
+SVC_USER="hawq"
+PIDFILE="\${HAWQ_PID_DIR}/${DAEMON}.pid"
+SVC_OPTS="-l \${HAWQ_LOG_DIR}"
+
+generate_start() {
+
+cat <<'__EOT__'
+start() {
+ [ -x $EXEC_PATH ] || exit $ERROR_PROGRAM_NOT_INSTALLED
+ log_success_msg "Starting $DESC (${DAEMON}): "
+
+ checkstatusofproc
+ status=$?
+ if [ "$status" -eq "$STATUS_RUNNING" ]; then
+ log_success_msg "${DESC} is running"
+ exit 0
+ fi
+
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start segment $SVC_OPTS"
+
+ checkstatusofproc
+ RETVAL=$?
+ [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
+ return $RETVAL
+}
+__EOT__
+
+}
+
+generate_stop() {
+
+cat <<'__EOT__'
+stop() {
+ log_success_msg "Stopping $DESC (${DAEMON}): "
+
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop segment $SVC_OPTS"
+
+ sleep 3
+ RETVAL=$?
+
+ [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
+ return $RETVAL
+}
+__EOT__
+
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/common/hawq/hawq.1
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq.1 b/bigtop-packages/src/common/hawq/hawq.1
new file mode 100644
index 0000000..96c62c0
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/hawq.1
@@ -0,0 +1,59 @@
+.\" Licensed to the Apache Software Foundation (ASF) under one or more
+.\" contributor license agreements. See the NOTICE file distributed with
+.\" this work for additional information regarding copyright ownership.
+.\" The ASF licenses this file to You under the Apache License, Version 2.0
+.\" (the "License"); you may not use this file except in compliance with
+.\" the License. You may obtain a copy of the License at
+.\"
+.\" http://www.apache.org/licenses/LICENSE-2.0
+.\"
+.\" Unless required by applicable law or agreed to in writing, software
+.\" distributed under the License is distributed on an "AS IS" BASIS,
+.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.\" See the License for the specific language governing permissions and
+.\" limitations under the License.
+.\"
+.\" Process this file with
+.\" groff -man -Tascii ignite.1
+.\"
+.TH ignite 1 "Mar 2015 " Linux "User Manuals"
+
+.SH NAME
+Ignite \- Ignite is a filesystem and MR accelerator for Hadoop.
+
+.SH SYNOPSIS
+
+.B ignite
+\fICOMMAND\fR
+
+.SH DESCRIPTION
+
+Ignite is a filesystem and MR accelerator for Hadoop. Use it when you
+you need predictable file system caching, fast MR processing...
+
+Ignite is an open-source, distributed blah-blah
+
+For more information about Ignite, see http://ignite.apache.org/
+
+\fICOMMAND\fR may be one of the following:
+ something do something
+ else do something else
+
+Most commands print help when invoked w/o parameters or with --help.
+
+.SH ENVIRONMENT
+
+.IP JAVA_HOME
+The java implementation to use. Overrides JAVA_HOME.
+
+.IP IGNITE_HOME
+Extra Java CLASSPATH entries.
+
+.IP IGNITE_CONF_DIR
+Alternate conf dir. Default is ${IGNITE_HOME}/conf.
+
+.IP HADOOP_HOME
+Optionally, the Hadoop home to run with.
+
+.SH COPYRIGHT
+2014 - 2015, The Apache Software Foundation
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/common/hawq/hawq.default
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq.default b/bigtop-packages/src/common/hawq/hawq.default
new file mode 100644
index 0000000..f8cf6e1
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/hawq.default
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export HAWQ_HOME="/usr/lib/hawq"
+export HAWQ_CONF_DIR="/etc/hawq/conf"
+
+export HAWQ_PID_DIR="/var/run/hawq"
+export HAWQ_LOG_DIR="/var/log/hawq"
+export HAWQ_WORK_DIR="/var/run/hawq/work"
+export HAWQ_IDENT_STRING=hawq
+
+export PATH=$HAWQ_HOME/bin:$HAWQ_HOME/ext/python/bin:$PATH
+export LD_LIBRARY_PATH=$HAWQ_HOME/lib:$HAWQ_HOME/ext/python/lib:$LD_LIBRARY_PATH
+export PYTHONPATH=$HAWQ_HOME/lib/python:$HAWQ_HOME/lib/python/pygresql:$PYTHONPATH
+export OPENSSL_CONF=$HAWQ_CONF_DIR/openssl.cnf
+export LIBHDFS3_CONF=$HAWQ_CONF_DIR/hdfs-client.xml
+export LIBYARN_CONF=$HAWQ_CONF_DIR/yarn-client.xml
+export HAWQSITE_CONF=$HAWQ_CONF_DIR/hawq-site.xml
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/common/hawq/install_hawq.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/install_hawq.sh b/bigtop-packages/src/common/hawq/install_hawq.sh
new file mode 100755
index 0000000..de0a88c
--- /dev/null
+++ b/bigtop-packages/src/common/hawq/install_hawq.sh
@@ -0,0 +1,155 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --build-dir=DIR path to hawq dist.dir
+ --prefix=PREFIX path to install into
+
+ Optional options:
+ --doc-dir=DIR path to install docs into [/usr/share/doc/hawq]
+ --lib-dir=DIR path to install hawq home [/usr/lib/hawq]
+ --installed-lib-dir=DIR path where lib-dir will end up on target system
+ --bin-dir=DIR path to install bins [/usr/bin]
+ --examples-dir=DIR path to install examples [doc-dir/examples]
+ ... [ see source for more similar options ]
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'prefix:' \
+ -l 'doc-dir:' \
+ -l 'lib-dir:' \
+ -l 'installed-lib-dir:' \
+ -l 'bin-dir:' \
+ -l 'examples-dir:' \
+ -l 'conf-dir:' \
+ -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+ case "$1" in
+ --prefix)
+ PREFIX=$2 ; shift 2
+ ;;
+ --build-dir)
+ BUILD_DIR=$2 ; shift 2
+ ;;
+ --doc-dir)
+ DOC_DIR=$2 ; shift 2
+ ;;
+ --lib-dir)
+ LIB_DIR=$2 ; shift 2
+ ;;
+ --installed-lib-dir)
+ INSTALLED_LIB_DIR=$2 ; shift 2
+ ;;
+ --bin-dir)
+ BIN_DIR=$2 ; shift 2
+ ;;
+ --examples-dir)
+ EXAMPLES_DIR=$2 ; shift 2
+ ;;
+ --conf-dir)
+ CONF_DIR=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+for var in PREFIX BUILD_DIR ; do
+ if [ -z "$(eval "echo \$$var")" ]; then
+ echo Missing param: $var
+ usage
+ fi
+done
+
+MAN_DIR=${MAN_DIR:-/usr/share/man/man1}
+DOC_DIR=${DOC_DIR:-/usr/share/doc/hawq}
+LIB_DIR=${LIB_DIR:-/usr/lib/hawq}
+BIN_DIR=${BIN_DIR:-/usr/lib/hawq/bin}
+ETC_DIR=${ETC_DIR:-/etc/hawq}
+CONF_DIR=${CONF_DIR:-${ETC_DIR}/conf.dist}
+
+install -d -m 0755 $PREFIX/$LIB_DIR
+install -d -m 0755 $PREFIX/$LIB_DIR/lib
+install -d -m 0755 $PREFIX/$LIB_DIR/include
+install -d -m 0755 $PREFIX/$LIB_DIR/sbin
+install -d -m 0755 $PREFIX/$LIB_DIR/share
+install -d -m 0755 $PREFIX/$DOC_DIR
+install -d -m 0755 $PREFIX/$BIN_DIR
+install -d -m 0755 $PREFIX/$ETC_DIR
+install -d -m 0755 $PREFIX/$CONF_DIR
+install -d -m 0755 $PREFIX/$MAN_DIR
+install -d -m 0755 $PREFIX/var/run/hawq/work/
+
+UNZIP_DIR=hawq
+cp -ar $UNZIP_DIR/lib $PREFIX/$LIB_DIR
+cp -a $UNZIP_DIR/etc/* $PREFIX/$CONF_DIR
+cp -ra $UNZIP_DIR/bin/* $PREFIX/$BIN_DIR
+cp -ra $UNZIP_DIR/sbin $PREFIX/$LIB_DIR
+cp -ra $UNZIP_DIR/share $PREFIX/$LIB_DIR
+cp -ar $UNZIP_DIR/include $PREFIX/$LIB_DIR
+cp -ar $UNZIP_DIR/docs $PREFIX/$DOC_DIR
+
+cp -ar $UNZIP_DIR/../libyarn/* $PREFIX/$LIB_DIR
+
+ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/config
+## HAWQ-422 configs are expected to sit in this hardcoded locations ;(
+ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/etc
+# HAWQ-421
+ln -s $ETC_DIR/default/hawq $PREFIX/$LIB_DIR/greenplum_path.sh
+
+wrapper=$PREFIX/usr/bin/hawq
+mkdir -p `dirname $wrapper`
+cat > $wrapper <<EOF
+#!/bin/bash
+
+BIGTOP_DEFAULTS_DIR=\${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hadoop ] && . \${BIGTOP_DEFAULTS_DIR}/hadoop
+[ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hawq ] && . \${BIGTOP_DEFAULTS_DIR}/hawq
+
+exec /usr/lib/hawq/bin/hawq \$1 hawq
+EOF
+chmod 755 $wrapper
+
+install -d -m 0755 $PREFIX/usr/bin
+
+install -d -m 0755 $PREFIX/$LIB_DIR/pxf
+UNZIP_DIR=pxf
+PXF_MODULES="pxf-hive pxf-hdfs pxf-service pxf-hbase pxf-api"
+for mod in $PXF_MODULES; do
+ tar zxf $UNZIP_DIR/$mod/build/distributions/*tar.gz -C $PREFIX/$LIB_DIR/pxf/
+done
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/compat
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/compat b/bigtop-packages/src/deb/hawq/compat
new file mode 100644
index 0000000..ec63514
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/compat
@@ -0,0 +1 @@
+9
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/control
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/control b/bigtop-packages/src/deb/hawq/control
new file mode 100644
index 0000000..269a809
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/control
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+Source: hawq
+Section: misc
+Priority: extra
+Maintainer: Bigtop <de...@bigtop.apache.org>
+Build-Depends: debhelper (>= 9)
+Standards-Version: 3.9.4
+Homepage: http://hawq.incubator.apache.org/
+
+Package: hawq
+Architecture: all
+Depends: adduser, hadoop-hdfs, bigtop-utils (>= 1.1.0), bigtop-tomcat
+Description: HAWQ (incubating) is an advanced analytics MPP database
+
+Package: hawq-doc
+Architecture: all
+Section: doc
+Description: HAWQ Documentation
+ Documentation for Apache HAWQ (incubating)
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/copyright
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/copyright b/bigtop-packages/src/deb/hawq/copyright
new file mode 100644
index 0000000..89a5120
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/copyright
@@ -0,0 +1,15 @@
+Format: http://dep.debian.net/deps/dep5
+Source: https://git-wip-us.apache.org/repos/asf/incubator-ignite
+Upstream-Name: none
+
+Files: *
+Copyright: 2014 - 2015, The Apache Software Foundation
+License: Apache-2.0
+
+Files debian/*
+Copyright: 2011, The Apache Software Foundation
+License: Apache-2.0
+
+License: Apache-2.0
+ On Debian systems, the complete text of the Apache 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/hawq-doc.dirs
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq-doc.dirs b/bigtop-packages/src/deb/hawq/hawq-doc.dirs
new file mode 100644
index 0000000..61fe83a
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq-doc.dirs
@@ -0,0 +1 @@
+/usr/share/doc/hawq-docs
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/hawq-doc.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq-doc.install b/bigtop-packages/src/deb/hawq/hawq-doc.install
new file mode 100644
index 0000000..99376a2
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq-doc.install
@@ -0,0 +1 @@
+/usr/share/doc/hawq-doc
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/hawq.dirs
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.dirs b/bigtop-packages/src/deb/hawq/hawq.dirs
new file mode 100644
index 0000000..3487c0d
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.dirs
@@ -0,0 +1,4 @@
+/usr/bin
+/usr/lib/hawq
+/var/log/hawq
+/etc/default
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/hawq.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.install b/bigtop-packages/src/deb/hawq/hawq.install
new file mode 100644
index 0000000..59d4005
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.install
@@ -0,0 +1,3 @@
+/usr/lib/hawq
+/usr/bin/hawq
+/etc/hawq
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/hawq.manpages
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.manpages b/bigtop-packages/src/deb/hawq/hawq.manpages
new file mode 100644
index 0000000..e5d94ec
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.manpages
@@ -0,0 +1 @@
+debian/hawq.1
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/hawq.postinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.postinst b/bigtop-packages/src/deb/hawq/hawq.postinst
new file mode 100644
index 0000000..916a8a4
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.postinst
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# postinst script for hawq
+
+hawq_pkg_name=hawq
+
+set -e
+
+case "$1" in
+ configure)
+ # Install config alternatives
+ update-alternatives --install /etc/${hawq_pkg_name}/conf ${hawq_pkg_name}-conf \
+ /etc/${hawq_pkg_name}/conf.dist 30
+
+ mkdir -p /var/log/${hawq_pkg_name} /var/run/${hawq_pkg_name} /var/run/${hawq_pkg_name}/work || :
+ chown hawq:hawq /var/log/${hawq_pkg_name} /var/run/${hawq_pkg_name}
+ chmod g+w /var/log/${hawq_pkg_name} /var/run/${hawq_pkg_name}
+ chown hawq:hawq /var/run/${hawq_pkg_name} /var/run/${hawq_pkg_name}/work
+ chmod 0755 /var/run/${hawq_pkg_name}
+ chmod 1777 /var/run/${hawq_pkg_name}/work
+ ;;
+
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+#DEBHELPER#
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/hawq.preinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.preinst b/bigtop-packages/src/deb/hawq/hawq.preinst
new file mode 100644
index 0000000..0b36a7a
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.preinst
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# preinst script for hawq
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <new-preinst> `install'
+# * <new-preinst> `install' <old-version>
+# * <new-preinst> `upgrade' <old-version>
+# * <old-preinst> `abort-upgrade' <new-version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ install|upgrade)
+ if ! getent passwd hawq >/dev/null; then
+ # Adding system user: hawq
+ adduser \
+ --system \
+ --group \
+ --home /var/run/hawq \
+ --gecos "HAWQ User" \
+ --shell /bin/bash \
+ hawq >/dev/null
+ fi
+ ;;
+
+ abort-upgrade)
+ ;;
+
+ *)
+ echo "preinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/hawq.prerm
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/hawq.prerm b/bigtop-packages/src/deb/hawq/hawq.prerm
new file mode 100644
index 0000000..0bcaf3c
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/hawq.prerm
@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# prerm script for hawq
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <prerm> `remove'
+# * <old-prerm> `upgrade' <new-version>
+# * <new-prerm> `failed-upgrade' <old-version>
+# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
+# * <deconfigured's-prerm> `deconfigure' `in-favour'
+# <package-being-installed> <version> `removing'
+# <conflicting-package> <version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ remove|upgrade|deconfigure)
+ update-alternatives --remove hawq-conf /etc/hawq/conf.dist || :
+ ;;
+
+ failed-upgrade)
+ ;;
+
+ *)
+ echo "prerm called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/rules b/bigtop-packages/src/deb/hawq/rules
new file mode 100755
index 0000000..b4a3c9a
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/rules
@@ -0,0 +1,55 @@
+#!/usr/bin/make -f
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -*- makefile -*-
+SHELL=/bin/bash
+
+# Uncomment this to turn on verbose mode.
+export DH_VERBOSE=1
+
+# This has to be exported to make some magic below work.
+export DH_OPTIONS
+
+hawq_pkg_name=hawq
+
+%:
+ dh $@
+
+override_dh_auto_build:
+ bash debian/do-component-build
+
+
+override_dh_auto_install:
+ cp debian/hawq.default debian/${hawq_pkg_name}/etc/default/${hawq_pkg_name}
+ env HADOOP_VERSION=${HAWQ_BASE_VERSION} \
+ bash -x debian/install_hawq.sh \
+ --build-dir=target/bin \
+ --conf-dir=/etc/${hawq_pkg_name}/conf.dist \
+ --doc-dir=usr/share/doc/${hawq_pkg_name}-doc \
+ --prefix=debian/tmp
+ ln -s /var/log/${hawq_pkg_name} debian/tmp/usr/lib/${hawq_pkg_name}/logs
+ ln -s /var/run/${hawq_pkg_name} debian/tmp/usr/lib/${hawq_pkg_name}/pids
+ bash debian/init.d.tmpl debian/hawq-master.svc deb debian/${hawq_pkg_name}-master-service.init
+ bash debian/init.d.tmpl debian/hawq-segment.svc deb debian/${hawq_pkg_name}-segment-service.init
+
+## Let's override the auto_configure and auto_clean to make sure existing
+## top-level Makefile doesn't interfere with the package creation
+override_dh_auto_configure:
+
+override_dh_auto_clean:
+
+override_dh_auto_test:
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/deb/hawq/source/format
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/source/format b/bigtop-packages/src/deb/hawq/source/format
new file mode 100644
index 0000000..163aaf8
--- /dev/null
+++ b/bigtop-packages/src/deb/hawq/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/rpm/hawq/BUILD/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/BUILD/.gitignore b/bigtop-packages/src/rpm/hawq/BUILD/.gitignore
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/rpm/hawq/RPMS/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/RPMS/.gitignore b/bigtop-packages/src/rpm/hawq/RPMS/.gitignore
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/rpm/hawq/SOURCES/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/SOURCES/.gitignore b/bigtop-packages/src/rpm/hawq/SOURCES/.gitignore
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/rpm/hawq/SPECS/hawq.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/SPECS/hawq.spec b/bigtop-packages/src/rpm/hawq/SPECS/hawq.spec
new file mode 100644
index 0000000..606b047
--- /dev/null
+++ b/bigtop-packages/src/rpm/hawq/SPECS/hawq.spec
@@ -0,0 +1,189 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+%define etc_hawq_conf %{_sysconfdir}/%{name}/conf
+%define etc_hawq_conf_dist %{etc_hawq_conf}.dist
+%define hawq_home /usr/lib/%{name}
+%define bin_hawq %{hawq_home}/bin
+%define lib_hawq %{hawq_home}/lib
+%define conf_hawq %{hawq_home}/config
+%define logs_hawq %{hawq_home}/logs
+%define pids_hawq %{hawq_home}/pids
+%define man_dir %{_mandir}
+%define hawq_username hawq
+%define vcs_tag incubator-%{name}-%{hawq_version}-beta-incubating-RC4
+
+%define pxf %{hawq_home}/pxf
+
+%if %{?suse_version:1}0
+
+# Only tested on openSUSE 11.4. le'ts update it for previous release when confirmed
+%if 0%{suse_version} > 1130
+%define suse_check \# Define an empty suse_check for compatibility with older sles
+%endif
+
+# SLES is more strict anc check all symlinks point to valid path
+# But we do point to a hadoop jar which is not there at build time
+# (but would be at install time).
+# Since our package build system does not handle dependencies,
+# these symlink checks are deactivated
+%define __os_install_post \
+ %{suse_check} ; \
+ /usr/lib/rpm/brp-compress ; \
+ %{nil}
+
+%define doc_hawq %{_docdir}/%{name}
+%global initd_dir %{_sysconfdir}/rc.d
+%define alternatives_cmd update-alternatives
+
+%else
+
+# CentOS 5 does not have any dist macro
+# So I will suppose anything that is not Mageia or a SUSE will be a RHEL/CentOS/Fedora
+%if %{!?mgaversion:1}0
+
+# FIXME: brp-repack-jars uses unzip to expand jar files
+# Unfortunately guice-2.0.jar pulled by ivy contains some files and directories without any read permission
+# and make whole process to fail.
+# So for now brp-repack-jars is being deactivated until this is fixed.
+# See BIGTOP-294
+#%define __os_install_post \
+# %{_rpmconfigdir}/brp-compress ; \
+# %{_rpmconfigdir}/brp-strip-static-archive %{__strip} ; \
+# %{_rpmconfigdir}/brp-strip-comment-note %{__strip} %{__objdump} ; \
+# /usr/lib/rpm/brp-python-bytecompile ; \
+# %{nil}
+%endif
+
+%define doc_hawq %{_docdir}/%{name}-%{hawq_version}
+%global initd_dir %{_sysconfdir}/rc.d/init.d
+%define alternatives_cmd alternatives
+
+%endif
+
+
+Name: hawq
+Version: %{hawq_version}
+Release: %{hawq_release}
+Summary: Apache Hawq (incubating) is an advanced analytics MPP database
+URL: http://hawq.incubator.apache.org/
+Group: Development/Libraries
+Buildroot: %{_topdir}/INSTALL/%{name}-%{version}
+License: APL2
+Source0: %{vcs_tag}.tar.gz
+Source1: do-component-build
+Source2: install_hawq.sh
+Source3: hawq-master.svc
+Source4: init.d.tmpl
+Source5: hawq.default
+Source6: hawq-segment.svc
+Requires: coreutils, /usr/sbin/useradd, /sbin/chkconfig, /sbin/service
+Requires: hadoop-hdfs, bigtop-utils >= 1.0, bigtop-tomcat
+
+%if 0%{?mgaversion}
+Requires: bsh-utils
+%else
+Requires: sh-utils
+%endif
+
+%description
+Hawq is an open-source, distributed, MPP database engine
+
+%package doc
+Summary: Hawq Documentation
+Group: Documentation
+BuildArch: noarch
+
+%description doc
+Documentation for Hawq platform
+
+%prep
+%setup -n %{vcs_tag}
+
+%build
+bash %{SOURCE1}
+
+%install
+%__rm -rf $RPM_BUILD_ROOT
+env HAWQ_VERSION=%{version} bash %{SOURCE2} \
+ --build-dir=target/bin \
+ --doc-dir=%{doc_hawq} \
+ --conf-dir=%{etc_hawq_conf_dist} \
+ --prefix=$RPM_BUILD_ROOT
+
+%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/
+
+%__install -d -m 0755 $RPM_BUILD_ROOT/etc/default/
+%__install -m 0644 %{SOURCE5} $RPM_BUILD_ROOT/etc/default/%{name}
+
+%__install -d -m 0755 %{buildroot}/%{_localstatedir}/log/%{name}
+
+ln -s %{_localstatedir}/log/%{name} %{buildroot}/%{logs_hawq}
+
+%__install -d -m 0755 %{buildroot}/%{_localstatedir}/run/%{name}
+ln -s %{_localstatedir}/run/%{name} %{buildroot}/%{pids_hawq}
+
+master_init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{name}
+segment_init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{name}-segment
+bash %{SOURCE4} ${RPM_SOURCE_DIR}/%{SOURCE3} rpm $master_init_file
+bash %{SOURCE4} ${RPM_SOURCE_DIR}/%{SOURCE6} rpm $segment_init_file
+chmod 755 $master_init_file $segment_init_file
+
+%__install -d -m 0755 $RPM_BUILD_ROOT/usr/bin
+
+%pre
+getent group hawq 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hawq
+getent passwd hawq 2>&1 > /dev/null || /usr/sbin/useradd -c "hawq" -s /sbin/nologin -g hawq -r -d /var/run/hawq hawq 2> /dev/null || :
+
+%post
+%{alternatives_cmd} --install %{etc_hawq_conf} %{name}-conf %{etc_hawq_conf_dist} 30
+chkconfig --add %{name}
+
+%preun
+if [ "$1" = 0 ]; then
+ %{alternatives_cmd} --remove %{name}-conf %{etc_hawq_conf_dist} || :
+fi
+
+if [ $1 = 0 ] ; then
+ service %{name} stop > /dev/null 2>&1
+ chkconfig --del %{name}
+fi
+
+%postun
+if [ $1 -ge 1 ]; then
+ service %{name} condrestart >/dev/null 2>&1
+fi
+
+
+#######################
+#### FILES SECTION ####
+#######################
+%files
+%defattr(-,hawq,hawq)
+%attr(0755,root,root)/%{initd_dir}/%{name}
+%attr(0755,root,root)/%{initd_dir}/%{name}-segment
+%dir %{_localstatedir}/log/%{name}
+%dir %{_localstatedir}/run/%{name}
+
+%defattr(-,root,root)
+%config(noreplace) %{_sysconfdir}/default/%{name}
+%{hawq_home}
+/usr/bin/%{name}
+%config(noreplace) %{etc_hawq_conf_dist}
+
+%files doc
+%defattr(-,root,root)
+%doc %{doc_hawq}
+
+%clean
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop-packages/src/rpm/hawq/SRPMS/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hawq/SRPMS/.gitignore b/bigtop-packages/src/rpm/hawq/SRPMS/.gitignore
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b909af42/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index 363b3fc..59fdfbc 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -422,5 +422,15 @@ bigtop {
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
}
+ 'hawq' {
+ name = 'hawq'
+ relNotes = 'Apache HAWQ (incubating)'
+ version { base = '2.0.0'; pkg = base; release = 1 }
+ tarball { source = "${version.base}-beta-incubating-RC4.tar.gz"
+ destination = "incubator-${name}-${version.base}-beta-incubating-RC4.tar.gz" }
+ url { download_path = "/apache/incubator-$name/archive"
+ site = "https://github.com/${download_path}"
+ archive = site }
+ }
}
}
[3/3] bigtop git commit: BIGTOP-2325. Deployment recipes for HAWQ
Posted by co...@apache.org.
BIGTOP-2325. Deployment recipes for HAWQ
Initial draft for puppet recipes
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/7b1aed9d
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/7b1aed9d
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/7b1aed9d
Branch: refs/heads/BIGTOP-2320
Commit: 7b1aed9d5c56e49d9875d1603758eb9dcfbbbbe1
Parents: b79eb5f
Author: Konstantin Boudnik <co...@apache.org>
Authored: Tue Feb 16 21:27:27 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Mon Mar 7 20:01:55 2016 -0800
----------------------------------------------------------------------
.../puppet/hieradata/bigtop/cluster.yaml | 4 +
bigtop-deploy/puppet/manifests/cluster.pp | 4 +
.../puppet/modules/hawq/manifests/init.pp | 115 +++++++
.../puppet/modules/hawq/templates/gpcheck.cnf | 58 ++++
.../puppet/modules/hawq/templates/hawq-site.xml | 158 +++++++++
.../puppet/modules/hawq/templates/hawq.default | 32 ++
.../modules/hawq/templates/hdfs-client.xml | 331 +++++++++++++++++++
.../modules/hawq/templates/yarn-client.xml | 123 +++++++
bigtop-deploy/puppet/modules/hawq/tests/init.pp | 16 +
.../src/common/hadoop/init-hcfs.json | 1 +
bigtop-packages/src/common/hawq/hawq-master.svc | 5 +-
bigtop-packages/src/deb/hawq/rules | 4 +-
bigtop.bom | 2 +-
13 files changed, 848 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index de98502..eb604ca 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -174,3 +174,7 @@ zeppelin::server::spark_master_url: "yarn-client"
zeppelin::server::hiveserver2_url: "jdbc:hive2://%{hiera('hadoop-hive::common::hiveserver2_host')}:%{hiera('hadoop-hive::common::hiveserver2_port')}"
zeppelin::server::hiveserver2_user: "%{hiera('bigtop::hiveserver2_user')}"
zeppelin::server::hiveserver2_password: "%{hiera('bigtop::hiveserver2_password')}"
+
+# hawq
+bigtop::hawq_master_node: "%{hiera('bigtop::hawq_master_node')}"
+bigtop::hawq_master_port: "5432"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index a0be567..f80ef5a 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -105,6 +105,9 @@ $roles_map = {
zeppelin => {
master => ["zeppelin-server"],
},
+ hawq => {
+ master => ["hawq"],
+ },
}
class hadoop_cluster_node (
@@ -159,6 +162,7 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_
"hadoop_hive",
"hadoop_oozie",
"hadoop_pig",
+ "hawq",
"sqoop2",
"hadoop_zookeeper",
"hcatalog",
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/manifests/init.pp b/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
new file mode 100644
index 0000000..b932632
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hawq {
+ class deploy ($roles) {
+ if ("hawq" in $roles) {
+ hawq::cluster_node { "hawq-node": }
+ }
+ }
+
+ define cluster_node() {
+ $hadoop_head_node = hiera("bigtop::hadoop_head_node")
+ $hadoop_namenode_port = hiera("hadoop::common_hdfs::hadoop_namenode_port", "8020")
+ $hawq_head = hiera("bigtop::hawq_master_node", "localhost")
+ $hawq_head_port = hiera('bigtop::hawq_master_port', "5432")
+ $hawq_yarn_rm_host = hiera('hadoop::common_yarn::hadoop_rm_host')
+ $hawq_yarn_rm_port = hiera('hadoop::common_yarn::hadoop_rm_port')
+
+ package { "hawq":
+ ensure => latest,
+ require => Package["libhdfs3-devel"],
+ ## require => for centos this crap needs epel-release
+ }
+
+ file { "/etc/default/hawq":
+ content => template("hawq/hawq.default"),
+ require => Package["hawq"],
+ }
+
+ file { "/etc/hawq/conf":
+ ensure => directory,
+ owner => 'hawq',
+ group => 'hawq',
+ mode => '0755',
+ require => Package["hawq"],
+ }
+ file { "/etc/hawq/conf/hawq-site.xml":
+ content => template('hawq/hawq-site.xml'),
+ require => [File["/etc/hawq/conf"]],
+ owner => 'hawq',
+ group => 'hawq',
+ mode => '0755',
+ }
+ file { "/etc/hawq/conf/gpcheck.cnf":
+ content => template('hawq/gpcheck.cnf'),
+ require => [File["/etc/hawq/conf"]],
+ }
+ file { "/etc/hawq/conf/hdfs-client.xml":
+ content => template('hawq/hdfs-client.xml'),
+ require => [File["/etc/hawq/conf"]],
+ }
+ file { "/etc/hawq/conf/yarn-client.xml":
+ content => template('hawq/yarn-client.xml'),
+ require => [File["/etc/hawq/conf"]],
+ }
+ file { "/etc/hawq/conf/slaves":
+ ensure => file,
+ content => "localhost", ## TODO - this has to be dynamic
+ }
+
+ exec { "install pygresql modules1":
+ path => ['/usr/bin'],
+ command => 'pip --retries=50 --timeout=300 install pg8000 simplejson unittest2 pycrypto pygresql pyyaml lockfile paramiko psi',
+ require => [ Package['python-pip', 'postgresql-devel'] ],
+ }
+ exec { "install pygresql modules2":
+ path => ['/usr/bin'],
+ command => 'pip --retries=50 --timeout=300 install http://darcs.idyll.org/~t/projects/figleaf-0.6.1.tar.gz',
+ require => [ Package['python-pip', 'pychecker'], Exec ['install pygresql modules1'] ],
+ ## HAWQ install instructions are suggesting to
+ ## uninstall postgresql postgresql-libs postgresql-devel at this point
+ ## but I don't think it matter, and for sure looks ugly
+ }
+
+ package { "python-pip":
+ ensure => latest,
+ }
+ package { "pychecker":
+ ensure => latest,
+ }
+ package { "postgresql-devel":
+ ensure => latest,
+ }
+ package { "libhdfs3-devel":
+ ensure => latest,
+ }
+
+### TODO init require hdfs to be running. Need to test this
+ exec { "hawk init master":
+ path => ['/usr/bin'],
+ # Silly init will ask if I am really sure I want to init the cluster
+ command => 'echo y | bash -x /usr/bin/hawq init master',
+ require => [ Package['hawq'], Exec ['install pygresql modules2'] ],
+ }
+
+## TODO The expectation is that init will start the service. I don't think so...
+ service { "hawq":
+ ensure => running,
+ require => [ Package["hawq"], File["/etc/default/hawq"], Exec["hawk init"] ],
+ subscribe => [ Package["hawq"], File["/etc/default/hawq", "/etc/hawq/conf/hawq-site.xml"] ]
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf b/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf
new file mode 100644
index 0000000..11ae02f
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf
@@ -0,0 +1,58 @@
+[global]
+configfile_version = 4
+
+[linux.mount]
+mount.points = /
+
+[linux.sysctl]
+sysctl.kernel.shmmax = 500000000
+sysctl.kernel.shmmni = 4096
+sysctl.kernel.shmall = 4000000000
+sysctl.kernel.sem = 250 512000 100 2048
+sysctl.kernel.sysrq = 1
+sysctl.kernel.core_uses_pid = 1
+sysctl.kernel.msgmnb = 65536
+sysctl.kernel.msgmax = 65536
+sysctl.kernel.msgmni = 2048
+sysctl.net.ipv4.tcp_syncookies = 0
+sysctl.net.ipv4.ip_forward = 0
+sysctl.net.ipv4.conf.default.accept_source_route = 0
+sysctl.net.ipv4.tcp_tw_recycle = 1
+sysctl.net.ipv4.tcp_max_syn_backlog = 200000
+sysctl.net.ipv4.conf.all.arp_filter = 1
+sysctl.net.ipv4.ip_local_port_range = 1281 65535
+sysctl.net.core.netdev_max_backlog = 200000
+sysctl.vm.overcommit_memory = 2
+sysctl.fs.nr_open = 3000000
+sysctl.kernel.threads-max = 798720
+sysctl.kernel.pid_max = 798720
+# increase network
+sysctl.net.core.rmem_max = 2097152
+sysctl.net.core.wmem_max = 2097152
+
+[linux.limits]
+soft.nofile = 2900000
+hard.nofile = 2900000
+soft.nproc = 131072
+hard.nproc = 131072
+
+[linux.diskusage]
+diskusage.monitor.mounts = /
+diskusage.monitor.usagemax = 90%
+
+[hdfs]
+dfs.mem.namenode.heap = 40960
+dfs.mem.datanode.heap = 6144
+# in hdfs-site.xml
+dfs.support.append = true
+dfs.client.enable.read.from.local = true
+dfs.block.local-path-access.user = gpadmin
+dfs.datanode.max.transfer.threads = 40960
+dfs.client.socket-timeout = 300000000
+dfs.datanode.socket.write.timeout = 7200000
+dfs.namenode.handler.count = 60
+ipc.server.handler.queue.size = 3300
+dfs.datanode.handler.count = 60
+ipc.client.connection.maxidletime = 3600000
+dfs.namenode.accesstime.precision = -1
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml b/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml
new file mode 100644
index 0000000..f3ec4ec
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml
@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<configuration>
+ <property>
+ <name>hawq_master_address_host</name>
+ <value><%= @hawq_head %></value>
+ <description>The host name of hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_address_port</name>
+ <value><%= @hawq_head_port %></value>
+ <description>The port of hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_standby_address_host</name>
+ <value>none</value>
+ <description>The host name of hawq standby master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_address_port</name>
+ <value>40000</value>
+ <description>The port of hawq segment.</description>
+ </property>
+
+ <property>
+ <name>hawq_dfs_url</name>
+ <value><%= @hadoop_head_node %>:<%= @hadoop_namenode_port %>/hawq_default</value>
+ <description>URL for accessing HDFS.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_directory</name>
+ <value>~/hawq-data-directory/masterdd</value>
+ <description>The directory of hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_directory</name>
+ <value>~/hawq-data-directory/segmentdd</value>
+ <description>The directory of hawq segment.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_temp_directory</name>
+ <value>/tmp</value>
+ <description>The temporary directory reserved for hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_temp_directory</name>
+ <value>/tmp</value>
+ <description>The temporary directory reserved for hawq segment.</description>
+ </property>
+
+ <!-- HAWQ resource manager parameters -->
+ <property>
+ <name>hawq_global_rm_type</name>
+ <value>none</value>
+ <description>The resource manager type to start for allocating resource.
+ 'none' means hawq resource manager exclusively uses whole
+ cluster; 'yarn' means hawq resource manager contacts YARN
+ resource manager to negotiate resource.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_memory_limit_perseg</name>
+ <value>64GB</value>
+ <description>The limit of memory usage in a hawq segment when
+ hawq_global_rm_type is set 'none'.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_nvcore_limit_perseg</name>
+ <value>16</value>
+ <description>The limit of virtual core usage in a hawq segment when
+ hawq_global_rm_type is set 'none'.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_address</name>
+ <value><%= @hawq_yarn_rm_host %>:<%= @hawq_yarn_rm_port %></value>
+ <description>The address of YARN resource manager server.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_scheduler_address</name>
+ <value>localhost:8030</value>
+ <description>The address of YARN scheduler server.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_queue_name</name>
+ <value>default</value>
+ <description>The YARN queue name to register hawq resource manager.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_app_name</name>
+ <value>hawq</value>
+ <description>The application name to register hawq resource manager in YARN.</description>
+ </property>
+ <!-- HAWQ resource manager parameters end here. -->
+
+ <!-- HAWQ resource enforcement parameters -->
+ <property>
+ <name>hawq_re_cpu_enable</name>
+ <value>false</value>
+ <description>The control to enable/disable CPU resource enforcement.</description>
+ </property>
+
+ <property>
+ <name>hawq_re_cgroup_mount_point</name>
+ <value>/sys/fs/cgroup</value>
+ <description>The mount point of CGroup file system for resource enforcement.
+ For example, /sys/fs/cgroup/cpu/hawq for CPU sub-system.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_re_cgroup_hierarchy_name</name>
+ <value>hawq</value>
+ <description>The name of the hierarchy to accomodate CGroup directories/files for resource enforcement.
+ For example, /sys/fs/cgroup/cpu/hawq for CPU sub-system.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_nvseg_perquery_perseg_limit</name>
+ <value>8</value>
+ <description>This is something that init tries to push in</description>
+ </property>
+ <!-- HAWQ resource enforcement parameters end here. -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hawq.default b/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
new file mode 100644
index 0000000..19da99e
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export HAWQ_HOME="/usr/lib/hawq"
+## Due to some weird scripting, hawq doesn't work without GPHOME
+export GPHOME=$HAWQ_HOME
+export HAWQ_CONF_DIR="/etc/hawq/conf"
+
+export HAWQ_PID_DIR="/var/run/hawq"
+export HAWQ_LOG_DIR="/var/log/hawq"
+export HAWQ_WORK_DIR="/var/run/hawq/work"
+export HAWQ_IDENT_STRING=hawq
+
+export PATH=$HAWQ_HOME/bin:$HAWQ_HOME/ext/python/bin:$PATH
+export LD_LIBRARY_PATH=$HAWQ_HOME/lib:$HAWQ_HOME/ext/python/lib:$LD_LIBRARY_PATH
+export PYTHONPATH=$HAWQ_HOME/lib/python:$HAWQ_HOME/lib/python/pygresql:$PYTHONPATH
+export OPENSSL_CONF=$HAWQ_CONF_DIR/openssl.cnf
+export LIBHDFS3_CONF=$HAWQ_CONF_DIR/hdfs-client.xml
+export LIBYARN_CONF=$HAWQ_CONF_DIR/yarn-client.xml
+export HAWQSITE_CONF=$HAWQ_CONF_DIR/hawq-site.xml
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml b/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml
new file mode 100644
index 0000000..3f08696
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml
@@ -0,0 +1,331 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<configuration>
+
+ <!-- KDC
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+ KDC -->
+
+ <!-- HA
+ <property>
+ <name>dfs.nameservices</name>
+ <value>phdcluster</value>
+ </property>
+
+ <property>
+ <name>dfs.ha.namenodes.phdcluster</name>
+ <value>nn1,nn2</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.rpc-address.phdcluster.nn1</name>
+ <value>mdw:9000</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.rpc-address.phdcluster.nn2</name>
+ <value>smdw:9000</value>
+ </property>
+
+<property>
+<name>dfs.namenode.http-address.phdcluster.nn1</name>
+<value>mdw:50070</value>
+</property>
+
+<property>
+<name>dfs.namenode.http-address.phdcluster.nn2</name>
+<value>smdw:50070</value>
+</property>
+
+HA -->
+
+ <!-- RPC client configuration -->
+ <property>
+ <name>rpc.client.timeout</name>
+ <value>3600000</value>
+ <description>
+ timeout interval of a RPC invocation in millisecond. default is 3600000.
+ </description>
+ </property>
+ <property>
+ <name>rpc.client.connect.tcpnodelay</name>
+ <value>true</value>
+ <description>
+ whether set socket TCP_NODELAY to true when connect to RPC server. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.max.idle</name>
+ <value>10000</value>
+ <description>
+ the max idle time of a RPC connection in millisecond. default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.ping.interval</name>
+ <value>10000</value>
+ <description>
+ the interval which the RPC client send a heart beat to server. 0 means disable, default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to setup the connection. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.retry</name>
+ <value>10</value>
+ <description>
+ the max retry times if the RPC client fail to setup the connection to server. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to read from server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to write to server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.socket.linger.timeout</name>
+ <value>-1</value>
+ <description>
+ set value to socket SO_LINGER when connect to RPC server. -1 means default OS value. default is -1.
+ </description>
+ </property>
+
+ <!-- dfs client configuration -->
+ <property>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>true</value>
+ <description>
+ whether reading block file bypass datanode if the block and the client are on the same node. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.default.replica</name>
+ <value>3</value>
+ <description>
+ the default number of replica. default is 3.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.prefetchsize</name>
+ <value>10</value>
+ <description>
+ the default number of blocks which information will be prefetched. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.failover.max.attempts</name>
+ <value>15</value>
+ <description>
+ if multiply namenodes are configured, it is the max retry times when the dfs client try to issue a RPC call. default is 15.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.default.blocksize</name>
+ <value>134217728</value>
+ <description>
+ default block size. default is 134217728.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.log.severity</name>
+ <value>INFO</value>
+ <description>
+ the minimal log severity level, valid values include FATAL, ERROR, INFO, DEBUG1, DEBUG2, DEBUG3. default is INFO.
+ </description>
+ </property>
+
+ <!-- input client configuration -->
+ <property>
+ <name>input.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the input stream is trying to setup the connection to datanode. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the input stream is trying to read from datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the input stream is trying to write to datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.localread.default.buffersize</name>
+ <value>2097152</value>
+ <description>
+ number of bytes of the buffer which is used to hold the data from block file and verify checksum.
+ it is only used when "dfs.client.read.shortcircuit" is set to true. default is 1048576.
+ </description>
+ </property>
+
+ <property>
+ <name>input.localread.blockinfo.cachesize</name>
+ <value>1000</value>
+ <description>
+ the size of block file path information cache. default is 1000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.read.getblockinfo.retry</name>
+ <value>3</value>
+ <description>
+ the max retry times when the client fail to get block information from namenode. default is 3.
+ </description>
+ </property>
+
+ <!-- output client configuration -->
+ <property>
+ <name>output.replace-datanode-on-failure</name>
+ <value>false</value>
+ <description>
+ whether the client add new datanode into pipeline if the number of nodes in pipeline is less the specified number of replicas. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>output.default.chunksize</name>
+ <value>512</value>
+ <description>
+ the number of bytes of a chunk in pipeline. default is 512.
+ </description>
+ </property>
+
+ <property>
+ <name>output.default.packetsize</name>
+ <value>65536</value>
+ <description>
+ the number of bytes of a packet in pipeline. default is 65536.
+ </description>
+ </property>
+
+ <property>
+ <name>output.default.write.retry</name>
+ <value>10</value>
+ <description>
+ the max retry times when the client fail to setup the pipeline. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>output.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the output stream is trying to setup the connection to datanode. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>output.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the output stream is trying to read from datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>output.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the output stream is trying to write to datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>output.packetpool.size</name>
+ <value>1024</value>
+ <description>
+ the max number of packets in a file's packet pool. default is 1024.
+ </description>
+ </property>
+
+ <property>
+ <name>output.close.timeout</name>
+ <value>900000</value>
+ <description>
+ the timeout interval in millisecond when close an output stream. default is 900000.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ <description>
+ Optional. This is a path to a UNIX domain socket that will be used for
+ communication between the DataNode and local HDFS clients.
+ If the string "_PORT" is present in this path, it will be replaced by the
+ TCP port of the DataNode.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.use.legacy.blockreader.local</name>
+ <value>false</value>
+ <description>
+ Legacy short-circuit reader implementation based on HDFS-2246 is used
+ if this configuration parameter is true.
+ This is for the platforms other than Linux
+ where the new implementation based on HDFS-347 is not available.
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml b/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml
new file mode 100644
index 0000000..a5f9df7
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<configuration>
+
+ <!-- KDC
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+ KDC -->
+
+ <!-- HA
+ <property>
+ <name>yarn.resourcemanager.ha</name>
+ <value>%RESOURCEMANAGER%:8032,%RESOURCEMANAGER2%:8032</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.ha</name>
+ <value>%RESOURCEMANAGER%:8030,%RESOURCEMANAGER2%:8030</value>
+ </property>
+ HA -->
+
+ <!-- RPC client configuration -->
+ <property>
+ <name>rpc.client.timeout</name>
+ <value>3600000</value>
+ <description>
+ timeout interval of a RPC invocation in millisecond. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.tcpnodelay</name>
+ <value>true</value>
+ <description>
+ whether set socket TCP_NODELAY to true when connect to RPC server. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.max.idle</name>
+ <value>10000</value>
+ <description>
+ the max idle time of a RPC connection in millisecond. default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.ping.interval</name>
+ <value>10000</value>
+ <description>
+ the interval which the RPC client send a heart beat to server. 0 means disable, default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to setup the connection. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.retry</name>
+ <value>10</value>
+ <description>
+ the max retry times if the RPC client fail to setup the connection to server. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to read from server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to write to server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.socket.linger.timeout</name>
+ <value>-1</value>
+ <description>
+ set value to socket SO_LINGER when connect to RPC server. -1 means default OS value. default is -1.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.client.failover.max.attempts</name>
+ <value>15</value>
+ <description>
+ if multiply resource managers are configured, it is the max retry times when the yarn client try to issue a RPC call. default is 15.
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-deploy/puppet/modules/hawq/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/tests/init.pp b/bigtop-deploy/puppet/modules/hawq/tests/init.pp
new file mode 100644
index 0000000..37a5a08
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/tests/init.pp
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hawq::cluster_node { "test-hawq-node": }
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-packages/src/common/hadoop/init-hcfs.json
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.json b/bigtop-packages/src/common/hadoop/init-hcfs.json
index 7f9b6cc..a9e0674 100644
--- a/bigtop-packages/src/common/hadoop/init-hcfs.json
+++ b/bigtop-packages/src/common/hadoop/init-hcfs.json
@@ -64,6 +64,7 @@
["/tmp/hadoop-yarn","777","mapred","mapred"],
["/var/log/hadoop-yarn/apps","1777","yarn","mapred"],
["/hbase",null,"hbase","hbase"],
+ ["/hawq_default","755","hawq","hawq"],
["/solr",null,"solr","solr"],
["/benchmarks","777",null,null],
["/user","755","HCFS_SUPER_USER",null],
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-packages/src/common/hawq/hawq-master.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq-master.svc b/bigtop-packages/src/common/hawq/hawq-master.svc
index c306a7f..bd60980 100644
--- a/bigtop-packages/src/common/hawq/hawq-master.svc
+++ b/bigtop-packages/src/common/hawq/hawq-master.svc
@@ -18,6 +18,7 @@ DESC="hawq master daemon"
EXEC_PATH="\${HAWQ_HOME}/bin/hawq"
SVC_USER="hawq"
PIDFILE="\${HAWQ_PID_DIR}/${DAEMON}.pid"
+SVC_OPTS="-l \${HAWQ_LOG_DIR}"
generate_start() {
@@ -33,7 +34,7 @@ start() {
exit 0
fi
- su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start ${DAEMON}"
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start master $SVC_OPTS"
checkstatusofproc
RETVAL=$?
@@ -50,7 +51,7 @@ cat <<'__EOT__'
stop() {
log_success_msg "Stopping $DESC (${DAEMON}): "
- su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop ${DAEMON}"
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop master $SVC_OPTS"
sleep 3
RETVAL=$?
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop-packages/src/deb/hawq/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/rules b/bigtop-packages/src/deb/hawq/rules
index b4a3c9a..1e256a1 100755
--- a/bigtop-packages/src/deb/hawq/rules
+++ b/bigtop-packages/src/deb/hawq/rules
@@ -30,7 +30,7 @@ hawq_pkg_name=hawq
dh $@
override_dh_auto_build:
- bash debian/do-component-build
+ bash debian/do-component-build
override_dh_auto_install:
@@ -46,7 +46,7 @@ override_dh_auto_install:
bash debian/init.d.tmpl debian/hawq-master.svc deb debian/${hawq_pkg_name}-master-service.init
bash debian/init.d.tmpl debian/hawq-segment.svc deb debian/${hawq_pkg_name}-segment-service.init
-## Let's override the auto_configure and auto_clean to make sure existing
+## Let's override the auto_configure and auto_clean to make sure existing
## top-level Makefile doesn't interfere with the package creation
override_dh_auto_configure:
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7b1aed9d/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index 59fdfbc..f0066c3 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -90,7 +90,7 @@ bigtop {
zookeeper:['hadoop', 'hbase'],
hadoop:['ignite-hadoop', 'hbase', 'crunch', 'pig', 'hive', 'tez', 'sqoop', 'sqoop2',
'oozie', 'mahout', 'flume', 'giraph', 'solr', 'crunch', 'spark',
- 'phoenix', 'tachyon', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin',
+ 'phoenix', 'tachyon', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin', 'hawq',
'tajo', 'apex'
],
hbase:['phoenix','giraph','ycsb'],
[2/3] bigtop git commit: BIGTOP-2323. Build slaves toolchain needs to
be updated for HAWQ
Posted by co...@apache.org.
BIGTOP-2323. Build slaves toolchain needs to be updated for HAWQ
libhdfs3-devel
thrift-devel
epel-release is needed as well
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b79eb5fb
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b79eb5fb
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b79eb5fb
Branch: refs/heads/BIGTOP-2320
Commit: b79eb5fbcb43a7bdf54df9281cf0cd4c3a0384fe
Parents: b909af4
Author: Konstantin Boudnik <co...@apache.org>
Authored: Thu Feb 11 19:33:30 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Mon Mar 7 19:59:29 2016 -0800
----------------------------------------------------------------------
bigtop-packages/src/common/hawq/do-component-build | 14 ++++++++++++++
bigtop-packages/src/common/hawq/install_hawq.sh | 3 ++-
bigtop_toolchain/manifests/packages.pp | 8 ++++----
3 files changed, 20 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b79eb5fb/bigtop-packages/src/common/hawq/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/do-component-build b/bigtop-packages/src/common/hawq/do-component-build
index ce2d23a..5da8103 100755
--- a/bigtop-packages/src/common/hawq/do-component-build
+++ b/bigtop-packages/src/common/hawq/do-component-build
@@ -34,6 +34,20 @@ cd ${CWD}
ldconfig $CWD/libyarn/lib
## libyarn should be available in the system at this point
+## build libhdfs3
+wget https://github.com/PivotalRD/libhdfs3/archive/v2.2.31.tar.gz
+tar zxf v2.2.31.tar.gz
+cd libhdfs3-2.2.31
+mkdir -p build
+cd build
+../bootstrap
+make rpm-package
+## need to run diff target for deb and rpm. Figure out how
+cd ${CWD}
+# now something like
+# rpm -ivh rpms/RPMS/`uname -p`/*rpm
+## libhdfs3 libhdfs3-devel should be available now
+
echo "Running the native build"
date;
./configure --prefix=${CWD}/hawq \
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b79eb5fb/bigtop-packages/src/common/hawq/install_hawq.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/install_hawq.sh b/bigtop-packages/src/common/hawq/install_hawq.sh
index de0a88c..0b860be 100755
--- a/bigtop-packages/src/common/hawq/install_hawq.sh
+++ b/bigtop-packages/src/common/hawq/install_hawq.sh
@@ -126,6 +126,7 @@ cp -ar $UNZIP_DIR/docs $PREFIX/$DOC_DIR
cp -ar $UNZIP_DIR/../libyarn/* $PREFIX/$LIB_DIR
+sed -i -e 's#source %s/greenplum_path.sh" % hawq_home#source /etc/default/hawq#' $PREFIX/$BIN_DIR/hawq
ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/config
## HAWQ-422 configs are expected to sit in this hardcoded locations ;(
ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/etc
@@ -141,7 +142,7 @@ BIGTOP_DEFAULTS_DIR=\${BIGTOP_DEFAULTS_DIR-/etc/default}
[ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hadoop ] && . \${BIGTOP_DEFAULTS_DIR}/hadoop
[ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hawq ] && . \${BIGTOP_DEFAULTS_DIR}/hawq
-exec /usr/lib/hawq/bin/hawq \$1 hawq
+exec /usr/lib/hawq/bin/hawq \$@
EOF
chmod 755 $wrapper
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b79eb5fb/bigtop_toolchain/manifests/packages.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp
index f378530..12a2918 100644
--- a/bigtop_toolchain/manifests/packages.pp
+++ b/bigtop_toolchain/manifests/packages.pp
@@ -22,16 +22,16 @@ class bigtop_toolchain::packages {
} else {
$mysql_devel="mariadb-devel"
}
- $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel" ]
+ $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "readline-devel", "snappy-devel" ]
}
- /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel" ]
+ /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "readline-devel", "snappy-devel" ]
# fix package dependencies: BIGTOP-2120 and BIGTOP-2152
exec { '/usr/bin/zypper remove -y krb5-mini':
} -> exec {'/usr/bin/zypper install -y libopenssl-devel':
} -> Package <| |>
}
- Amazon: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel" ] }
- /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev" ]
+ Amazon: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "gperf", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "readline-devel", "snappy-devel" ] }
+ /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "bzip2-devel", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "gperf", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "readline-devel", "libsnappy-dev" ]
exec { "apt-update":
command => "/usr/bin/apt-get update"