You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2016/03/19 05:08:38 UTC
[1/2] bigtop git commit: BIGTOP-2323. Build slaves toolchain needs to
be updated for HAWQ [Forced Update!]
Repository: bigtop
Updated Branches:
refs/heads/BIGTOP-2320 7df3ee180 -> 10435d6e7 (forced update)
BIGTOP-2323. Build slaves toolchain needs to be updated for HAWQ
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/bb700d72
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/bb700d72
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/bb700d72
Branch: refs/heads/BIGTOP-2320
Commit: bb700d72976787bb498d50399b238c659afe7b8f
Parents: 1d68396
Author: Konstantin Boudnik <co...@apache.org>
Authored: Thu Feb 11 19:33:30 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Fri Mar 18 21:07:37 2016 -0700
----------------------------------------------------------------------
bigtop_toolchain/manifests/installer.pp | 3 ++
bigtop_toolchain/manifests/libhdfs.pp | 45 ++++++++++++++++++++++++++++
bigtop_toolchain/manifests/libthrift.pp | 37 +++++++++++++++++++++++
bigtop_toolchain/manifests/packages.pp | 10 ++++---
bigtop_toolchain/manifests/protobuf.pp | 11 ++++---
5 files changed, 98 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/bb700d72/bigtop_toolchain/manifests/installer.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/installer.pp b/bigtop_toolchain/manifests/installer.pp
index 246fcaf..d29a222 100644
--- a/bigtop_toolchain/manifests/installer.pp
+++ b/bigtop_toolchain/manifests/installer.pp
@@ -20,6 +20,9 @@ class bigtop_toolchain::installer {
include bigtop_toolchain::gradle
include bigtop_toolchain::protobuf
include bigtop_toolchain::packages
+## The hack workaround until such times when the lib is embedded into Hawq
+ include bigtop_toolchain::libhdfs
+ include bigtop_toolchain::libthrift
include bigtop_toolchain::env
include bigtop_toolchain::user
http://git-wip-us.apache.org/repos/asf/bigtop/blob/bb700d72/bigtop_toolchain/manifests/libhdfs.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/libhdfs.pp b/bigtop_toolchain/manifests/libhdfs.pp
new file mode 100644
index 0000000..4698108
--- /dev/null
+++ b/bigtop_toolchain/manifests/libhdfs.pp
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class bigtop_toolchain::libhdfs {
+
+ include bigtop_toolchain::packages
+
+
+ case $operatingsystem {
+ /(?i:(centos|fedora|amazon))/: {
+
+ $apache_prefix = nearest_apache_mirror()
+ $hackrepourl = "https://bintray.com/artifact/download/wangzw/rpm/centos7/x86_64"
+ $libhdfs = "libhdfs3-2.2.31-1.el7.centos.x86_64.rpm"
+ $libhdfs_dev = "libhdfs3-devel-2.2.31-1.el7.centos.x86_64.rpm"
+
+ exec {"download":
+ path => "/usr/bin",
+ cwd => "/usr/src",
+ command => "curl -L $hackrepourl/$libhdfs -o $libhdfs ; curl -L $hackrepourl/$libhdfs_dev -o $libhdfs_dev",
+ creates => "/usr/src/$libhdfs, /usr/src/$libhdfs_dev",
+ }
+
+ exec {"install":
+ path => "/usr/bin",
+ cwd => "/usr/src",
+ command => "yum install -y $libhdfs $libhdfs_dev",
+ require => [ Exec[ 'download' ], Package[ $packages::pkgs ] ],
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/bb700d72/bigtop_toolchain/manifests/libthrift.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/libthrift.pp b/bigtop_toolchain/manifests/libthrift.pp
new file mode 100644
index 0000000..bc1abf3
--- /dev/null
+++ b/bigtop_toolchain/manifests/libthrift.pp
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# eee the License for the specific language governing permissions and
+# limitations under the License.
+
+class bigtop_toolchain::libthrift {
+
+ include bigtop_toolchain::packages
+
+ case $operatingsystem {
+ /Ubuntu|Debian/: {
+ include apt
+ apt::conf { "disable_keys":
+ content => "APT::Get::AllowUnauthenticated 1;",
+ ensure => present
+ }
+ apt::source { "libthrift":
+ location => "http://dl.bintray.com/c0s/deb",
+ release => "trusty",
+ repos => "contrib",
+ ensure => present,
+ }
+ Apt::Source<||> -> Exec['apt_update'] -> Package<||>
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/bb700d72/bigtop_toolchain/manifests/packages.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp
index 26959e2..d80a3ed 100644
--- a/bigtop_toolchain/manifests/packages.pp
+++ b/bigtop_toolchain/manifests/packages.pp
@@ -22,16 +22,18 @@ class bigtop_toolchain::packages {
} else {
$mysql_devel="mariadb-devel"
}
- $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
+ $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "flex", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "readline-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel", "thrift-devel" ]
+ exec { '/usr/bin/yum install -y epel-release':
+ } -> Package <| |>
}
- /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
+ /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "flex", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "readline-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
# fix package dependencies: BIGTOP-2120 and BIGTOP-2152
exec { '/usr/bin/zypper remove -y krb5-mini':
} -> exec {'/usr/bin/zypper install -y libopenssl-devel':
} -> Package <| |>
}
- Amazon: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel" ] }
- /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev" ]
+ Amazon: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "flex", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "gperf", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "readline-devel", "snappy-devel" ] }
+ /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "libcurl4-openssl-dev", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "libapr1-dev", "flex", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libbz2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "gperf", "uuid", "libgsasl7-dev", "libevent-dev", "libjson-c-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libreadline-dev", "libsnappy-dev", "libboost-regex-dev", "libboost-system-dev", "libboost-thread-dev", "libthrift0", "libthrift-dev", "xfslibs-dev" ]
exec { "apt-update":
command => "/usr/bin/apt-get update"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/bb700d72/bigtop_toolchain/manifests/protobuf.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/protobuf.pp b/bigtop_toolchain/manifests/protobuf.pp
index c71dec2..53d539b 100644
--- a/bigtop_toolchain/manifests/protobuf.pp
+++ b/bigtop_toolchain/manifests/protobuf.pp
@@ -21,13 +21,15 @@ class bigtop_toolchain::protobuf {
/Ubuntu|Debian/: {
case $architecture {
'amd64' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5585371/+files/"
+ $url1= "http://launchpadlibrarian.net/166035876/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb"
+ $url2= "http://launchpadlibrarian.net/166035877/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb"
$arch= "amd64" }
'ppc64le' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5604345/+files"
$arch= "ppc64el" }
}
}
}
-
+
case $operatingsystem{
/Ubuntu|Debian/: {
$libprotobuf8 = "libprotobuf8_2.5.0-9ubuntu1_$arch.deb"
@@ -35,13 +37,14 @@ class bigtop_toolchain::protobuf {
$protobuf_compiler = "protobuf-compiler_2.5.0-9ubuntu1_$arch.deb"
exec { "download protobuf":
+ path => "/usr/bin",
cwd => "/usr/src",
- command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl -L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler",
- creates => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler" ]
+ command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl -L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler; curl -L $url1 -o libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb; curl -L $url2 -o libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb",
+ creates => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler", "/usr/src/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb", "/usr/src/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb" ]
}
exec { "install protobuf":
cwd => "/usr/src",
- command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler",
+ command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb",
require => EXEC["download protobuf"],
}
}
[2/2] bigtop git commit: BIGTOP-2325. Deployment recipes for HAWQ
Posted by co...@apache.org.
BIGTOP-2325. Deployment recipes for HAWQ
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/10435d6e
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/10435d6e
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/10435d6e
Branch: refs/heads/BIGTOP-2320
Commit: 10435d6e7b1db00e1a443894b33c18cab8001486
Parents: bb700d7
Author: Konstantin Boudnik <co...@apache.org>
Authored: Tue Feb 16 21:27:27 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Fri Mar 18 21:08:01 2016 -0700
----------------------------------------------------------------------
.../puppet/hieradata/bigtop/cluster.yaml | 6 +
bigtop-deploy/puppet/manifests/cluster.pp | 4 +
.../puppet/modules/hawq/templates/gpcheck.cnf | 58 ++++
.../puppet/modules/hawq/templates/hawq-site.xml | 158 +++++++++
.../puppet/modules/hawq/templates/hawq.default | 40 +++
.../modules/hawq/templates/hdfs-client.xml | 331 +++++++++++++++++++
.../puppet/modules/hawq/templates/sysctl.conf | 24 ++
.../modules/hawq/templates/yarn-client.xml | 123 +++++++
bigtop-deploy/puppet/modules/hawq/tests/init.pp | 16 +
.../src/common/hadoop/init-hcfs.json | 1 +
bigtop-packages/src/common/hawq/hawq-master.svc | 4 +-
bigtop-packages/src/common/hawq/hawq.default | 10 +
bigtop-packages/src/deb/hawq/rules | 4 +-
bigtop.bom | 2 +-
bigtop_toolchain/manifests/libhdfs.pp | 39 ++-
bigtop_toolchain/manifests/packages.pp | 2 +-
16 files changed, 809 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index de98502..cf8ffcb 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -174,3 +174,9 @@ zeppelin::server::spark_master_url: "yarn-client"
zeppelin::server::hiveserver2_url: "jdbc:hive2://%{hiera('hadoop-hive::common::hiveserver2_host')}:%{hiera('hadoop-hive::common::hiveserver2_port')}"
zeppelin::server::hiveserver2_user: "%{hiera('bigtop::hiveserver2_user')}"
zeppelin::server::hiveserver2_password: "%{hiera('bigtop::hiveserver2_password')}"
+
+# hawq
+bigtop::hawq_master_node: "%{hiera('bigtop::hawq_master_node')}"
+bigtop::hawq_master_port: "5432"
+bigtop::hawq_master_datadir: "/var/run/hawq/work/masterdd"
+bigtop::hawq_segment_datadir: "/var/run/hawq/work/segmendd"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index a0be567..f80ef5a 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -105,6 +105,9 @@ $roles_map = {
zeppelin => {
master => ["zeppelin-server"],
},
+ hawq => {
+ master => ["hawq"],
+ },
}
class hadoop_cluster_node (
@@ -159,6 +162,7 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_
"hadoop_hive",
"hadoop_oozie",
"hadoop_pig",
+ "hawq",
"sqoop2",
"hadoop_zookeeper",
"hcatalog",
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf b/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf
new file mode 100644
index 0000000..11ae02f
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/gpcheck.cnf
@@ -0,0 +1,58 @@
+[global]
+configfile_version = 4
+
+[linux.mount]
+mount.points = /
+
+[linux.sysctl]
+sysctl.kernel.shmmax = 500000000
+sysctl.kernel.shmmni = 4096
+sysctl.kernel.shmall = 4000000000
+sysctl.kernel.sem = 250 512000 100 2048
+sysctl.kernel.sysrq = 1
+sysctl.kernel.core_uses_pid = 1
+sysctl.kernel.msgmnb = 65536
+sysctl.kernel.msgmax = 65536
+sysctl.kernel.msgmni = 2048
+sysctl.net.ipv4.tcp_syncookies = 0
+sysctl.net.ipv4.ip_forward = 0
+sysctl.net.ipv4.conf.default.accept_source_route = 0
+sysctl.net.ipv4.tcp_tw_recycle = 1
+sysctl.net.ipv4.tcp_max_syn_backlog = 200000
+sysctl.net.ipv4.conf.all.arp_filter = 1
+sysctl.net.ipv4.ip_local_port_range = 1281 65535
+sysctl.net.core.netdev_max_backlog = 200000
+sysctl.vm.overcommit_memory = 2
+sysctl.fs.nr_open = 3000000
+sysctl.kernel.threads-max = 798720
+sysctl.kernel.pid_max = 798720
+# increase network
+sysctl.net.core.rmem_max = 2097152
+sysctl.net.core.wmem_max = 2097152
+
+[linux.limits]
+soft.nofile = 2900000
+hard.nofile = 2900000
+soft.nproc = 131072
+hard.nproc = 131072
+
+[linux.diskusage]
+diskusage.monitor.mounts = /
+diskusage.monitor.usagemax = 90%
+
+[hdfs]
+dfs.mem.namenode.heap = 40960
+dfs.mem.datanode.heap = 6144
+# in hdfs-site.xml
+dfs.support.append = true
+dfs.client.enable.read.from.local = true
+dfs.block.local-path-access.user = gpadmin
+dfs.datanode.max.transfer.threads = 40960
+dfs.client.socket-timeout = 300000000
+dfs.datanode.socket.write.timeout = 7200000
+dfs.namenode.handler.count = 60
+ipc.server.handler.queue.size = 3300
+dfs.datanode.handler.count = 60
+ipc.client.connection.maxidletime = 3600000
+dfs.namenode.accesstime.precision = -1
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml b/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml
new file mode 100644
index 0000000..713fa40
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hawq-site.xml
@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<configuration>
+ <property>
+ <name>hawq_master_address_host</name>
+ <value><%= @hawq_head %></value>
+ <description>The host name of hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_address_port</name>
+ <value><%= @hawq_head_port %></value>
+ <description>The port of hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_standby_address_host</name>
+ <value>none</value>
+ <description>The host name of hawq standby master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_address_port</name>
+ <value>40000</value>
+ <description>The port of hawq segment.</description>
+ </property>
+
+ <property>
+ <name>hawq_dfs_url</name>
+ <value><%= @hadoop_head_node %>:<%= @hadoop_namenode_port %>/hawq_default</value>
+ <description>URL for accessing HDFS.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_directory</name>
+ <value><% @hawq_masterdata_dir %></value>
+ <description>The directory of hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_directory</name>
+ <value><% @hawq_segmentdata_dir %>d</value>
+ <description>The directory of hawq segment.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_temp_directory</name>
+ <value>/tmp</value>
+ <description>The temporary directory reserved for hawq master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_temp_directory</name>
+ <value>/tmp</value>
+ <description>The temporary directory reserved for hawq segment.</description>
+ </property>
+
+ <!-- HAWQ resource manager parameters -->
+ <property>
+ <name>hawq_global_rm_type</name>
+ <value>none</value>
+ <description>The resource manager type to start for allocating resource.
+ 'none' means hawq resource manager exclusively uses whole
+ cluster; 'yarn' means hawq resource manager contacts YARN
+ resource manager to negotiate resource.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_memory_limit_perseg</name>
+ <value>64GB</value>
+ <description>The limit of memory usage in a hawq segment when
+ hawq_global_rm_type is set 'none'.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_nvcore_limit_perseg</name>
+ <value>16</value>
+ <description>The limit of virtual core usage in a hawq segment when
+ hawq_global_rm_type is set 'none'.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_address</name>
+ <value><%= @hawq_yarn_rm_host %>:<%= @hawq_yarn_rm_port %></value>
+ <description>The address of YARN resource manager server.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_scheduler_address</name>
+ <value>localhost:8030</value>
+ <description>The address of YARN scheduler server.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_queue_name</name>
+ <value>default</value>
+ <description>The YARN queue name to register hawq resource manager.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_app_name</name>
+ <value>hawq</value>
+ <description>The application name to register hawq resource manager in YARN.</description>
+ </property>
+ <!-- HAWQ resource manager parameters end here. -->
+
+ <!-- HAWQ resource enforcement parameters -->
+ <property>
+ <name>hawq_re_cpu_enable</name>
+ <value>false</value>
+ <description>The control to enable/disable CPU resource enforcement.</description>
+ </property>
+
+ <property>
+ <name>hawq_re_cgroup_mount_point</name>
+ <value>/sys/fs/cgroup</value>
+ <description>The mount point of CGroup file system for resource enforcement.
+ For example, /sys/fs/cgroup/cpu/hawq for CPU sub-system.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_re_cgroup_hierarchy_name</name>
+ <value>hawq</value>
+ <description>The name of the hierarchy to accomodate CGroup directories/files for resource enforcement.
+ For example, /sys/fs/cgroup/cpu/hawq for CPU sub-system.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_nvseg_perquery_perseg_limit</name>
+ <value>8</value>
+ <description>This is something that init tries to push in</description>
+ </property>
+ <!-- HAWQ resource enforcement parameters end here. -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hawq.default b/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
new file mode 100644
index 0000000..8190260
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export HAWQ_HOME="/usr/lib/hawq"
+## Due to some weird scripting, hawq doesn't work without GPHOME
+export GPHOME=$HAWQ_HOME
+export HAWQ_CONF_DIR="/etc/hawq/conf"
+
+export HAWQ_PID_DIR="/var/run/hawq"
+export HAWQ_LOG_DIR="/var/log/hawq"
+export HAWQ_WORK_DIR="/var/run/hawq/work"
+export HAWQ_IDENT_STRING=hawq
+
+export PATH=$HAWQ_HOME/bin:$HAWQ_HOME/ext/python/bin:$PATH
+export LD_LIBRARY_PATH=$HAWQ_HOME/lib:$HAWQ_HOME/ext/python/lib:$LD_LIBRARY_PATH
+export PYTHONPATH=$HAWQ_HOME/lib/python:$HAWQ_HOME/lib/python/pygresql:$PYTHONPATH
+export OPENSSL_CONF=$HAWQ_CONF_DIR/openssl.cnf
+export LIBHDFS3_CONF=$HAWQ_CONF_DIR/hdfs-client.xml
+export LIBYARN_CONF=$HAWQ_CONF_DIR/yarn-client.xml
+export HAWQSITE_CONF=$HAWQ_CONF_DIR/hawq-site.xml
+
+export HAWQ_MASTER_PORT=5432
+export HAWQ_SEGMENT_PORT=40000
+export HAWQ_MASTERDATA_DIR=<%= @hawq_masterdata_dir %>
+export HAWQ_SEGMENTDATA_DIR=<%= @hawq_segmentdata_dir %>
+
+export HAWQ_TIMEOUT=30
+export HAWQ_SHUTDOWN_MODE=smart
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml b/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml
new file mode 100644
index 0000000..3f08696
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hdfs-client.xml
@@ -0,0 +1,331 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<configuration>
+
+ <!-- KDC
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+ KDC -->
+
+ <!-- HA
+ <property>
+ <name>dfs.nameservices</name>
+ <value>phdcluster</value>
+ </property>
+
+ <property>
+ <name>dfs.ha.namenodes.phdcluster</name>
+ <value>nn1,nn2</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.rpc-address.phdcluster.nn1</name>
+ <value>mdw:9000</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.rpc-address.phdcluster.nn2</name>
+ <value>smdw:9000</value>
+ </property>
+
+<property>
+<name>dfs.namenode.http-address.phdcluster.nn1</name>
+<value>mdw:50070</value>
+</property>
+
+<property>
+<name>dfs.namenode.http-address.phdcluster.nn2</name>
+<value>smdw:50070</value>
+</property>
+
+HA -->
+
+ <!-- RPC client configuration -->
+ <property>
+ <name>rpc.client.timeout</name>
+ <value>3600000</value>
+ <description>
+ timeout interval of a RPC invocation in millisecond. default is 3600000.
+ </description>
+ </property>
+ <property>
+ <name>rpc.client.connect.tcpnodelay</name>
+ <value>true</value>
+ <description>
+ whether set socket TCP_NODELAY to true when connect to RPC server. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.max.idle</name>
+ <value>10000</value>
+ <description>
+ the max idle time of a RPC connection in millisecond. default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.ping.interval</name>
+ <value>10000</value>
+ <description>
+ the interval which the RPC client send a heart beat to server. 0 means disable, default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to setup the connection. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.retry</name>
+ <value>10</value>
+ <description>
+ the max retry times if the RPC client fail to setup the connection to server. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to read from server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to write to server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.socket.linger.timeout</name>
+ <value>-1</value>
+ <description>
+ set value to socket SO_LINGER when connect to RPC server. -1 means default OS value. default is -1.
+ </description>
+ </property>
+
+ <!-- dfs client configuration -->
+ <property>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>true</value>
+ <description>
+ whether reading block file bypass datanode if the block and the client are on the same node. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.default.replica</name>
+ <value>3</value>
+ <description>
+ the default number of replica. default is 3.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.prefetchsize</name>
+ <value>10</value>
+ <description>
+ the default number of blocks which information will be prefetched. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.failover.max.attempts</name>
+ <value>15</value>
+ <description>
+ if multiply namenodes are configured, it is the max retry times when the dfs client try to issue a RPC call. default is 15.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.default.blocksize</name>
+ <value>134217728</value>
+ <description>
+ default block size. default is 134217728.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.log.severity</name>
+ <value>INFO</value>
+ <description>
+ the minimal log severity level, valid values include FATAL, ERROR, INFO, DEBUG1, DEBUG2, DEBUG3. default is INFO.
+ </description>
+ </property>
+
+ <!-- input client configuration -->
+ <property>
+ <name>input.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the input stream is trying to setup the connection to datanode. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the input stream is trying to read from datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the input stream is trying to write to datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.localread.default.buffersize</name>
+ <value>2097152</value>
+ <description>
+ number of bytes of the buffer which is used to hold the data from block file and verify checksum.
+ it is only used when "dfs.client.read.shortcircuit" is set to true. default is 1048576.
+ </description>
+ </property>
+
+ <property>
+ <name>input.localread.blockinfo.cachesize</name>
+ <value>1000</value>
+ <description>
+ the size of block file path information cache. default is 1000.
+ </description>
+ </property>
+
+ <property>
+ <name>input.read.getblockinfo.retry</name>
+ <value>3</value>
+ <description>
+ the max retry times when the client fail to get block information from namenode. default is 3.
+ </description>
+ </property>
+
+ <!-- output client configuration -->
+ <property>
+ <name>output.replace-datanode-on-failure</name>
+ <value>false</value>
+ <description>
+ whether the client add new datanode into pipeline if the number of nodes in pipeline is less the specified number of replicas. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>output.default.chunksize</name>
+ <value>512</value>
+ <description>
+ the number of bytes of a chunk in pipeline. default is 512.
+ </description>
+ </property>
+
+ <property>
+ <name>output.default.packetsize</name>
+ <value>65536</value>
+ <description>
+ the number of bytes of a packet in pipeline. default is 65536.
+ </description>
+ </property>
+
+ <property>
+ <name>output.default.write.retry</name>
+ <value>10</value>
+ <description>
+ the max retry times when the client fail to setup the pipeline. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>output.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the output stream is trying to setup the connection to datanode. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>output.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the output stream is trying to read from datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>output.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the output stream is trying to write to datanode. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>output.packetpool.size</name>
+ <value>1024</value>
+ <description>
+ the max number of packets in a file's packet pool. default is 1024.
+ </description>
+ </property>
+
+ <property>
+ <name>output.close.timeout</name>
+ <value>900000</value>
+ <description>
+ the timeout interval in millisecond when close an output stream. default is 900000.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ <description>
+ Optional. This is a path to a UNIX domain socket that will be used for
+ communication between the DataNode and local HDFS clients.
+ If the string "_PORT" is present in this path, it will be replaced by the
+ TCP port of the DataNode.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.use.legacy.blockreader.local</name>
+ <value>false</value>
+ <description>
+ Legacy short-circuit reader implementation based on HDFS-2246 is used
+ if this configuration parameter is true.
+ This is for the platforms other than Linux
+ where the new implementation based on HDFS-347 is not available.
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-deploy/puppet/modules/hawq/templates/sysctl.conf
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/sysctl.conf b/bigtop-deploy/puppet/modules/hawq/templates/sysctl.conf
new file mode 100644
index 0000000..2ac1598
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/sysctl.conf
@@ -0,0 +1,24 @@
+kernel.shmmax = 1000000000
+kernel.shmmni = 4096
+kernel.shmall = 4000000000
+kernel.sem = 250 512000 100 2048
+kernel.sysrq = 1
+kernel.core_uses_pid = 1
+kernel.msgmnb = 65536
+kernel.msgmax = 65536
+kernel.msgmni = 2048
+net.ipv4.tcp_syncookies = 0
+net.ipv4.ip_forward = 0
+net.ipv4.conf.default.accept_source_route = 0
+net.ipv4.tcp_tw_recycle = 1
+net.ipv4.tcp_max_syn_backlog = 200000
+net.ipv4.conf.all.arp_filter = 1
+net.ipv4.ip_local_port_range = 1281 65535
+net.core.netdev_max_backlog = 200000
+vm.overcommit_memory = 2
+fs.nr_open = 3000000
+kernel.threads-max = 798720
+kernel.pid_max = 798720
+# increase network
+net.core.rmem_max=2097152
+net.core.wmem_max=2097152
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml b/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml
new file mode 100644
index 0000000..a5f9df7
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/templates/yarn-client.xml
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<configuration>
+
+ <!-- KDC
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ </property>
+ KDC -->
+
+ <!-- HA
+ <property>
+ <name>yarn.resourcemanager.ha</name>
+ <value>%RESOURCEMANAGER%:8032,%RESOURCEMANAGER2%:8032</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.ha</name>
+ <value>%RESOURCEMANAGER%:8030,%RESOURCEMANAGER2%:8030</value>
+ </property>
+ HA -->
+
+ <!-- RPC client configuration -->
+ <property>
+ <name>rpc.client.timeout</name>
+ <value>3600000</value>
+ <description>
+ timeout interval of a RPC invocation in millisecond. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.tcpnodelay</name>
+ <value>true</value>
+ <description>
+ whether set socket TCP_NODELAY to true when connect to RPC server. default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.max.idle</name>
+ <value>10000</value>
+ <description>
+ the max idle time of a RPC connection in millisecond. default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.ping.interval</name>
+ <value>10000</value>
+ <description>
+ the interval which the RPC client send a heart beat to server. 0 means disable, default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to setup the connection. default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.retry</name>
+ <value>10</value>
+ <description>
+ the max retry times if the RPC client fail to setup the connection to server. default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to read from server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ the timeout interval in millisecond when the RPC client is trying to write to server. default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.socket.linger.timeout</name>
+ <value>-1</value>
+ <description>
+ set value to socket SO_LINGER when connect to RPC server. -1 means default OS value. default is -1.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.client.failover.max.attempts</name>
+ <value>15</value>
+ <description>
+ if multiply resource managers are configured, it is the max retry times when the yarn client try to issue a RPC call. default is 15.
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-deploy/puppet/modules/hawq/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/tests/init.pp b/bigtop-deploy/puppet/modules/hawq/tests/init.pp
new file mode 100644
index 0000000..37a5a08
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hawq/tests/init.pp
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hawq::cluster_node { "test-hawq-node": }
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-packages/src/common/hadoop/init-hcfs.json
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.json b/bigtop-packages/src/common/hadoop/init-hcfs.json
index f71f385..dea5f90 100644
--- a/bigtop-packages/src/common/hadoop/init-hcfs.json
+++ b/bigtop-packages/src/common/hadoop/init-hcfs.json
@@ -64,6 +64,7 @@
["/tmp/hadoop-yarn","777","mapred","mapred"],
["/var/log/hadoop-yarn/apps","1777","yarn","mapred"],
["/hbase",null,"hbase","hbase"],
+ ["/hawq_default","755","hawq","hawq"],
["/solr",null,"solr","solr"],
["/benchmarks","777",null,null],
["/user","755","HCFS_SUPER_USER",null],
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-packages/src/common/hawq/hawq-master.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq-master.svc b/bigtop-packages/src/common/hawq/hawq-master.svc
index 4fa5097..e5aca0d 100644
--- a/bigtop-packages/src/common/hawq/hawq-master.svc
+++ b/bigtop-packages/src/common/hawq/hawq-master.svc
@@ -34,7 +34,7 @@ start() {
exit 0
fi
- su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start $SVC_OPTS"
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} start master $SVC_OPTS"
checkstatusofproc
RETVAL=$?
@@ -51,7 +51,7 @@ cat <<'__EOT__'
stop() {
log_success_msg "Stopping $DESC (${DAEMON}): "
- su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop $SVC_OPTS"
+ su -s /bin/bash ${SVC_USER} -c "${EXEC_PATH} stop master $SVC_OPTS"
sleep 3
RETVAL=$?
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-packages/src/common/hawq/hawq.default
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/hawq.default b/bigtop-packages/src/common/hawq/hawq.default
index f8cf6e1..574e845 100644
--- a/bigtop-packages/src/common/hawq/hawq.default
+++ b/bigtop-packages/src/common/hawq/hawq.default
@@ -21,6 +21,8 @@ export HAWQ_LOG_DIR="/var/log/hawq"
export HAWQ_WORK_DIR="/var/run/hawq/work"
export HAWQ_IDENT_STRING=hawq
+export HAWQ_DATA_DIR=$HAWQ_WORK_DIR/hawq-data-directory
+
export PATH=$HAWQ_HOME/bin:$HAWQ_HOME/ext/python/bin:$PATH
export LD_LIBRARY_PATH=$HAWQ_HOME/lib:$HAWQ_HOME/ext/python/lib:$LD_LIBRARY_PATH
export PYTHONPATH=$HAWQ_HOME/lib/python:$HAWQ_HOME/lib/python/pygresql:$PYTHONPATH
@@ -28,3 +30,11 @@ export OPENSSL_CONF=$HAWQ_CONF_DIR/openssl.cnf
export LIBHDFS3_CONF=$HAWQ_CONF_DIR/hdfs-client.xml
export LIBYARN_CONF=$HAWQ_CONF_DIR/yarn-client.xml
export HAWQSITE_CONF=$HAWQ_CONF_DIR/hawq-site.xml
+
+export HAWQ_MASTER_PORT=5432
+export HAWQ_SEGMENT_PORT=40000
+export HAWQ_MASTERDATA_DIR=$HAWQ_DATA_DIR/masterdd
+export HAWQ_SEGMENTDATA_DIR=$HAWQ_DATA_DIR/segmentdd
+
+export HAWQ_TIMEOUT=30
+export HAWQ_SHUTDOWN_MODE=smart
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop-packages/src/deb/hawq/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hawq/rules b/bigtop-packages/src/deb/hawq/rules
index b4a3c9a..1e256a1 100755
--- a/bigtop-packages/src/deb/hawq/rules
+++ b/bigtop-packages/src/deb/hawq/rules
@@ -30,7 +30,7 @@ hawq_pkg_name=hawq
dh $@
override_dh_auto_build:
- bash debian/do-component-build
+ bash debian/do-component-build
override_dh_auto_install:
@@ -46,7 +46,7 @@ override_dh_auto_install:
bash debian/init.d.tmpl debian/hawq-master.svc deb debian/${hawq_pkg_name}-master-service.init
bash debian/init.d.tmpl debian/hawq-segment.svc deb debian/${hawq_pkg_name}-segment-service.init
-## Let's override the auto_configure and auto_clean to make sure existing
+## Let's override the auto_configure and auto_clean to make sure existing
## top-level Makefile doesn't interfere with the package creation
override_dh_auto_configure:
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index f8a6878..5670546 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -90,7 +90,7 @@ bigtop {
zookeeper:['hadoop', 'hbase'],
hadoop:['ignite-hadoop', 'hbase', 'crunch', 'pig', 'hive', 'tez', 'sqoop', 'sqoop2',
'oozie', 'mahout', 'flume', 'giraph', 'solr', 'crunch', 'spark',
- 'phoenix', 'tachyon', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin',
+ 'phoenix', 'tachyon', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin', 'hawq',
'tajo', 'apex'
],
hbase:['phoenix','giraph','ycsb'],
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop_toolchain/manifests/libhdfs.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/libhdfs.pp b/bigtop_toolchain/manifests/libhdfs.pp
index 4698108..eef5dd9 100644
--- a/bigtop_toolchain/manifests/libhdfs.pp
+++ b/bigtop_toolchain/manifests/libhdfs.pp
@@ -18,8 +18,32 @@ class bigtop_toolchain::libhdfs {
include bigtop_toolchain::packages
-
case $operatingsystem {
+ /Ubuntu|Debian/: {
+ $hackrepourl = "https://bintray.com/artifact/download/wangzw/deb/dists/trusty/contrib/binary-amd64"
+ $libhdfs = "libhdfs3_2.2.31-1_amd64.deb"
+ $libhdfs_dev = "libhdfs3-dev_2.2.31-1_amd64.deb"
+
+ $bisonurl_dev = "http://launchpadlibrarian.net/140087283"
+ $bisonurl = "http://launchpadlibrarian.net/140087282"
+ $bison = "bison_2.7.1.dfsg-1_amd64.deb"
+ $bison_dev = "libbison-dev_2.7.1.dfsg-1_amd64.deb"
+
+ exec {"install":
+ path => "/usr/bin:/bin:/usr/sbin:/sbin",
+ cwd => "/usr/src",
+ command => "dpkg --install $libhdfs $libhdfs_dev $bison $bison_dev",
+ require => [ Exec[ 'download', 'download-bison' ], Package[ $packages::pkgs ] ],
+ }
+
+ exec {"download-bison":
+ path => "/usr/bin",
+ cwd => "/usr/src",
+ command => "curl -L $bisonurl/$bison -o $bison ; curl -L $bisonurl_dev/$bison_dev -o $bison_dev",
+ creates => "/usr/src/$bison, /usr/src/$bison_dev",
+ }
+ }
+
/(?i:(centos|fedora|amazon))/: {
$apache_prefix = nearest_apache_mirror()
@@ -27,12 +51,6 @@ class bigtop_toolchain::libhdfs {
$libhdfs = "libhdfs3-2.2.31-1.el7.centos.x86_64.rpm"
$libhdfs_dev = "libhdfs3-devel-2.2.31-1.el7.centos.x86_64.rpm"
- exec {"download":
- path => "/usr/bin",
- cwd => "/usr/src",
- command => "curl -L $hackrepourl/$libhdfs -o $libhdfs ; curl -L $hackrepourl/$libhdfs_dev -o $libhdfs_dev",
- creates => "/usr/src/$libhdfs, /usr/src/$libhdfs_dev",
- }
exec {"install":
path => "/usr/bin",
@@ -42,4 +60,11 @@ class bigtop_toolchain::libhdfs {
}
}
}
+
+ exec {"download":
+ path => "/usr/bin",
+ cwd => "/usr/src",
+ command => "curl -L $hackrepourl/$libhdfs -o $libhdfs ; curl -L $hackrepourl/$libhdfs_dev -o $libhdfs_dev",
+ creates => "/usr/src/$libhdfs, /usr/src/$libhdfs_dev",
+ }
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/10435d6e/bigtop_toolchain/manifests/packages.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp
index d80a3ed..817c89d 100644
--- a/bigtop_toolchain/manifests/packages.pp
+++ b/bigtop_toolchain/manifests/packages.pp
@@ -14,7 +14,7 @@
# limitations under the License.
class bigtop_toolchain::packages {
- case $operatingsystem{
+ case $operatingsystem {
/(?i:(centos|fedora))/: {
# Fedora 20 and CentOS 7 or above are using mariadb, while CentOS 6 is still mysql
if ($operatingsystem == "CentOS") and ($operatingsystemmajrelease <=6) {