You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2016/03/18 03:22:44 UTC
[12/13] bigtop git commit: BIGTOP-2323. Build slaves toolchain needs
to be updated for HAWQ
BIGTOP-2323. Build slaves toolchain needs to be updated for HAWQ
thrift-devel is still an issue for Ubuntu
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b8ab3931
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b8ab3931
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b8ab3931
Branch: refs/heads/BIGTOP-2320
Commit: b8ab3931e7167af5a34d876717195e89e2812fa1
Parents: 8a946b7
Author: Konstantin Boudnik <co...@apache.org>
Authored: Thu Feb 11 19:33:30 2016 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Mar 17 19:20:01 2016 -0700
----------------------------------------------------------------------
bigtop_toolchain/manifests/installer.pp | 2 ++
bigtop_toolchain/manifests/libhdfs.pp | 45 ++++++++++++++++++++++++++++
bigtop_toolchain/manifests/packages.pp | 11 ++++---
bigtop_toolchain/manifests/protobuf.pp | 11 ++++---
4 files changed, 61 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/installer.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/installer.pp b/bigtop_toolchain/manifests/installer.pp
index 246fcaf..060d4bd 100644
--- a/bigtop_toolchain/manifests/installer.pp
+++ b/bigtop_toolchain/manifests/installer.pp
@@ -20,6 +20,8 @@ class bigtop_toolchain::installer {
include bigtop_toolchain::gradle
include bigtop_toolchain::protobuf
include bigtop_toolchain::packages
+## The hack workaround until such times when the lib is embedded into Hawq
+ include bigtop_toolchain::libhdfs
include bigtop_toolchain::env
include bigtop_toolchain::user
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/libhdfs.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/libhdfs.pp b/bigtop_toolchain/manifests/libhdfs.pp
new file mode 100644
index 0000000..4698108
--- /dev/null
+++ b/bigtop_toolchain/manifests/libhdfs.pp
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class bigtop_toolchain::libhdfs {
+
+ include bigtop_toolchain::packages
+
+
+ case $operatingsystem {
+ /(?i:(centos|fedora|amazon))/: {
+
+ $apache_prefix = nearest_apache_mirror()
+ $hackrepourl = "https://bintray.com/artifact/download/wangzw/rpm/centos7/x86_64"
+ $libhdfs = "libhdfs3-2.2.31-1.el7.centos.x86_64.rpm"
+ $libhdfs_dev = "libhdfs3-devel-2.2.31-1.el7.centos.x86_64.rpm"
+
+ exec {"download":
+ path => "/usr/bin",
+ cwd => "/usr/src",
+ command => "curl -L $hackrepourl/$libhdfs -o $libhdfs ; curl -L $hackrepourl/$libhdfs_dev -o $libhdfs_dev",
+ creates => "/usr/src/$libhdfs, /usr/src/$libhdfs_dev",
+ }
+
+ exec {"install":
+ path => "/usr/bin",
+ cwd => "/usr/src",
+ command => "yum install -y $libhdfs $libhdfs_dev",
+ require => [ Exec[ 'download' ], Package[ $packages::pkgs ] ],
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/packages.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp
index 26959e2..edd01da 100644
--- a/bigtop_toolchain/manifests/packages.pp
+++ b/bigtop_toolchain/manifests/packages.pp
@@ -22,16 +22,19 @@ class bigtop_toolchain::packages {
} else {
$mysql_devel="mariadb-devel"
}
- $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
}
- /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
+ $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "readline-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel", "thrift-devel" ]
+ exec { '/usr/bin/yum install -y epel-release':
+ } -> Package <| |>
+ }
+ /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "readline-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
# fix package dependencies: BIGTOP-2120 and BIGTOP-2152
exec { '/usr/bin/zypper remove -y krb5-mini':
} -> exec {'/usr/bin/zypper install -y libopenssl-devel':
} -> Package <| |>
}
- Amazon: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel" ] }
- /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev" ]
+ Amazon: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "gperf", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "readline-devel", "snappy-devel" ] }
+ /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "libcurl4-openssl-dev", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "libapr1-dev", "bison", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libbz2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "gperf", "uuid", "libgsasl7-dev", "libevent-dev", "libjson-c-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libreadline-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev" ]
exec { "apt-update":
command => "/usr/bin/apt-get update"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/protobuf.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/protobuf.pp b/bigtop_toolchain/manifests/protobuf.pp
index c71dec2..53d539b 100644
--- a/bigtop_toolchain/manifests/protobuf.pp
+++ b/bigtop_toolchain/manifests/protobuf.pp
@@ -21,13 +21,15 @@ class bigtop_toolchain::protobuf {
/Ubuntu|Debian/: {
case $architecture {
'amd64' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5585371/+files/"
+ $url1= "http://launchpadlibrarian.net/166035876/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb"
+ $url2= "http://launchpadlibrarian.net/166035877/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb"
$arch= "amd64" }
'ppc64le' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5604345/+files"
$arch= "ppc64el" }
}
}
}
-
+
case $operatingsystem{
/Ubuntu|Debian/: {
$libprotobuf8 = "libprotobuf8_2.5.0-9ubuntu1_$arch.deb"
@@ -35,13 +37,14 @@ class bigtop_toolchain::protobuf {
$protobuf_compiler = "protobuf-compiler_2.5.0-9ubuntu1_$arch.deb"
exec { "download protobuf":
+ path => "/usr/bin",
cwd => "/usr/src",
- command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl -L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler",
- creates => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler" ]
+ command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl -L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler; curl -L $url1 -o libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb; curl -L $url2 -o libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb",
+ creates => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler", "/usr/src/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb", "/usr/src/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb" ]
}
exec { "install protobuf":
cwd => "/usr/src",
- command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler",
+ command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb",
require => EXEC["download protobuf"],
}
}