You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2016/02/18 18:15:34 UTC
[1/3] bigtop git commit: BIGTOP-2325. More idiosyncratic stuff with
runtime requirements.
Repository: bigtop
Updated Branches:
refs/heads/BIGTOP-2320 7d92b2934 -> 6ac64fd9c
BIGTOP-2325. More idiosyncratic stuff with runtime requirements.
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ff2bdb60
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ff2bdb60
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ff2bdb60
Branch: refs/heads/BIGTOP-2320
Commit: ff2bdb60b8a1b90e788ecd50ac026052b460ca12
Parents: 7d92b29
Author: Konstantin Boudnik <co...@apache.org>
Authored: Thu Feb 18 20:13:10 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Feb 18 20:13:10 2016 +0300
----------------------------------------------------------------------
.../puppet/modules/hawq/manifests/init.pp | 39 +++++++++++++++++++-
.../puppet/modules/hawq/templates/hawq.default | 2 +
2 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ff2bdb60/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/manifests/init.pp b/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
index 6c45773..a8da991 100644
--- a/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hawq/manifests/init.pp
@@ -14,6 +14,8 @@
# limitations under the License.
class hawq {
+ include stdlib
+
class deploy ($roles) {
if ("hawq" in $roles) {
hawq::cluster_node { "hawq-node": }
@@ -61,10 +63,43 @@ class hawq {
content => template('hawq/yarn-client.xml'),
require => [File["/etc/hawq/conf"]],
}
+ file { "/etc/hawq/conf/slaves":
+ ensure => file,
+ content => $slaves_content,
+ }
+ $slaves_content = join (hiera('bigtop::hawq_slaves), ' ')
+
+ exec { "install pygresql modules1":
+ path => ['/usr/bin'],
+ command => 'pip --retries=50 --timeout=300 install pg8000 simplejson unittest2 pycrypto pygresql pyyaml lockfile paramiko psi',
+ require => [ Package['python-pip'] ],
+ }
+ exec { "install pygresql modules2":
+ path => ['/usr/bin'],
+ command => 'pip --retries=50 --timeout=300 install http://darcs.idyll.org/~t/projects/figleaf-0.6.1.tar.gz',
+ require => [ Package['python-pip'], Exec ['install pygresql modules1'] ],
+ }
+ exec { "install pygresql modules3":
+ path => ['/usr/bin'],
+ command => 'pip --retries=50 --timeout=300 install http://sourceforge.net/projects/pychecker/files/pychecker/0.8.19/pychecker-0.8.19.tar.gz/download',
+ require => [ Package['python-pip'], Exec ['install pygresql modules2'] ],
+ ## HAWQ install instructions are suggesting to
+ ## uninstall postgresql postgresql-libs postgresql-devel at this point
+ ## but I don't think it matter, and for sure looks ugly
+ }
+
+ package { "python-pip":
+ ensure => latest,
+ }
+ package { "postgresql":
+ ensure => latest,
+ }
+
exec { "hawk init":
path => ['/usr/bin'],
- command => 'bash -x /usr/bin/hawq init',
- require => Package['hawq'],
+ # Silly init will ask if I am really sure I want to init the cluster
+ command => 'echo y | bash -x /usr/bin/hawq init cluster',
+ require => [ Package['hawq', 'python-pip', 'postgresql'], Exec ['install pygresql modules3'] ],
}
service { "hawq":
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ff2bdb60/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hawq/templates/hawq.default b/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
index f8cf6e1..19da99e 100644
--- a/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
+++ b/bigtop-deploy/puppet/modules/hawq/templates/hawq.default
@@ -14,6 +14,8 @@
# limitations under the License.
export HAWQ_HOME="/usr/lib/hawq"
+## Due to some weird scripting, hawq doesn't work without GPHOME
+export GPHOME=$HAWQ_HOME
export HAWQ_CONF_DIR="/etc/hawq/conf"
export HAWQ_PID_DIR="/var/run/hawq"
[2/3] bigtop git commit: BIGTOP-2320. Need to add hadoop into hawq
deps
Posted by co...@apache.org.
BIGTOP-2320. Need to add hadoop into hawq deps
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/95b4b4c9
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/95b4b4c9
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/95b4b4c9
Branch: refs/heads/BIGTOP-2320
Commit: 95b4b4c9c9932b80f0a8d0f8fa94b654c921a268
Parents: ff2bdb6
Author: Konstantin Boudnik <co...@apache.org>
Authored: Thu Feb 18 20:14:22 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Feb 18 20:14:22 2016 +0300
----------------------------------------------------------------------
bigtop.bom | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/95b4b4c9/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index 213fad8..1a58df1 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -90,7 +90,7 @@ bigtop {
zookeeper:['hadoop', 'hbase'],
hadoop:['ignite-hadoop', 'hbase', 'crunch', 'pig', 'hive', 'tez', 'sqoop', 'sqoop2',
'oozie', 'mahout', 'flume', 'giraph', 'solr', 'crunch', 'spark',
- 'phoenix', 'tachyon', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin'
+ 'phoenix', 'tachyon', 'kafka', 'ycsb', 'kite', 'hama', 'zeppelin', 'hawq'
],
hbase:['phoenix','giraph','ycsb'],
pig:['datafu', 'oozie'],
[3/3] bigtop git commit: BIGTOP-2323. Workaround the issues with
hardcoded path to configs
Posted by co...@apache.org.
BIGTOP-2323. Workaround the issues with hardcoded path to configs
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6ac64fd9
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6ac64fd9
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6ac64fd9
Branch: refs/heads/BIGTOP-2320
Commit: 6ac64fd9c8f5ad272028cc45eb389598f04db8e2
Parents: 95b4b4c
Author: Konstantin Boudnik <co...@apache.org>
Authored: Thu Feb 18 20:14:55 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Feb 18 20:14:55 2016 +0300
----------------------------------------------------------------------
bigtop-packages/src/common/hawq/install_hawq.sh | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6ac64fd9/bigtop-packages/src/common/hawq/install_hawq.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hawq/install_hawq.sh b/bigtop-packages/src/common/hawq/install_hawq.sh
index df9945a..d1b7a59 100755
--- a/bigtop-packages/src/common/hawq/install_hawq.sh
+++ b/bigtop-packages/src/common/hawq/install_hawq.sh
@@ -126,7 +126,10 @@ cp -ar $UNZIP_DIR/docs $PREFIX/$DOC_DIR
cp -ar $UNZIP_DIR/../libyarn/* $PREFIX/$LIB_DIR
+sed -i -e 's#source %s/greenplum_path.sh" % hawq_home#source /etc/default/hawq#' $PREFIX/$BIN_DIR/hawq
ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/config
+## HAWQ-422 configs are expected to sit in this hardcoded locations ;(
+ln -s $PREFIX/$LIB_DIR/config $PREFIX/$LIB_DIR/etc
wrapper=$PREFIX/usr/bin/hawq
mkdir -p `dirname $wrapper`
@@ -137,7 +140,7 @@ BIGTOP_DEFAULTS_DIR=\${BIGTOP_DEFAULTS_DIR-/etc/default}
[ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hadoop ] && . \${BIGTOP_DEFAULTS_DIR}/hadoop
[ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hawq ] && . \${BIGTOP_DEFAULTS_DIR}/hawq
-exec /usr/lib/hawq/bin/hawq \$1 hawq
+exec /usr/lib/hawq/bin/hawq \$@
EOF
chmod 755 $wrapper