You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/10/05 04:58:12 UTC
git commit: BIGTOP-1041. Add Puppet support for Spark cluster
deployment
Updated Branches:
refs/heads/master 5de186bc9 -> 31d510605
BIGTOP-1041. Add Puppet support for Spark cluster deployment
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/31d51060
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/31d51060
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/31d51060
Branch: refs/heads/master
Commit: 31d510605682dabb054a92f0c2e80b12e3adb82d
Parents: 5de186b
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Fri Oct 4 19:48:09 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Fri Oct 4 19:48:09 2013 -0700
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 16 ++++++
.../puppet/modules/spark/manifests/init.pp | 51 ++++++++++++++++++++
.../puppet/modules/spark/templates/spark-env.sh | 38 +++++++++++++++
.../puppet/modules/spark/tests/init.pp | 25 ++++++++++
4 files changed, 130 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31d51060/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index cec9cec..b4bb36d 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -69,6 +69,8 @@ class hadoop_cluster_node {
$hadoop_rm_proxy_port = extlookup("hadoop_rm_proxy_port", "8088")
$hadoop_history_server_port = extlookup("hadoop_history_server_port", "19888")
$hbase_thrift_port = extlookup("hbase_thrift_port", "9090")
+ $spark_master_port = extlookup("spark_master_port", "7077")
+ $spark_master_ui_port = extlookup("spark_master_ui_port", "18080")
$hadoop_ha_zookeeper_quorum = "${hadoop_head_node}:${hadoop_zookeeper_port}"
$solrcloud_zk = "${hadoop_head_node}:${hadoop_zookeeper_port}"
@@ -95,6 +97,8 @@ class hadoop_cluster_node {
$giraph_zookeeper_quorum = $hadoop_head_node
+ $spark_master_host = $hadoop_head_node
+
$hadoop_zookeeper_ensemble = ["$hadoop_head_node:2888:3888"]
# Set from facter if available
@@ -171,6 +175,12 @@ class hadoop_worker_node inherits hadoop_cluster_node {
root_url => $hadoop_namenode_uri,
kerberos_realm => $kerberos_realm,
}
+
+ spark::worker { "spark worker":
+ master_host => $spark_master_host,
+ master_port => $spark_master_port,
+ master_ui_port => $spark_master_ui_port,
+ }
}
class hadoop_head_node inherits hadoop_worker_node {
@@ -239,6 +249,12 @@ class hadoop_head_node inherits hadoop_worker_node {
kerberos_realm => $kerberos_realm,
}
+ spark::master { "spark master":
+ master_host => $spark_master_host,
+ master_port => $spark_master_port,
+ master_ui_port => $spark_master_ui_port,
+ }
+
hadoop-zookeeper::server { "zookeeper":
myid => "0",
ensemble => $hadoop_zookeeper_ensemble,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31d51060/bigtop-deploy/puppet/modules/spark/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
new file mode 100644
index 0000000..f23387e
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class spark {
+ class common {
+ package { "spark":
+ ensure => latest,
+ }
+
+ file { "/etc/spark/conf/spark-env.sh":
+ content => template("spark/spark-env.sh"),
+ require => [Package["spark"]],
+ }
+ }
+
+ define master($master_host = $fqdn, $master_port = "7077", $master_ui_port = "18080") {
+ include common
+
+ service { "spark-master":
+ ensure => running,
+ require => [ Package["spark"], File["/etc/spark/conf/spark-env.sh"], ],
+ subscribe => [Package["spark"], File["/etc/spark/conf/spark-env.sh"] ],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ }
+
+ define worker($master_host = $fqdn, $master_port = "7077", $master_ui_port = "18080") {
+ include common
+
+ service { "spark-worker":
+ ensure => running,
+ require => [ Package["spark"], File["/etc/spark/conf/spark-env.sh"], ],
+ subscribe => [Package["spark"], File["/etc/spark/conf/spark-env.sh"] ],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31d51060/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
new file mode 100644
index 0000000..5f14cc0
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# This file contains environment variables required to run Spark. Copy it as
+# spark-env.sh and edit that to configure Spark for your site.
+#
+# The following variables can be set in this file:
+# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
+# - MESOS_NATIVE_LIBRARY, to point to your libmesos.so if you use Mesos
+# - SPARK_JAVA_OPTS, to set node-specific JVM options for Spark. Note that
+# we recommend setting app-wide options in the application's driver program.
+# Examples of node-specific options : -Dspark.local.dir, GC options
+# Examples of app-wide options : -Dspark.serializer
+#
+# If using the standalone deploy mode, you can also set variables for it here:
+# - SPARK_MASTER_IP, to bind the master to a different IP address or hostname
+# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports
+# - SPARK_WORKER_CORES, to set the number of cores to use on this machine
+# - SPARK_WORKER_MEMORY, to set how much memory to use (e.g. 1000m, 2g)
+# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT
+# - SPARK_WORKER_INSTANCES, to set the number of worker processes per node
+
+
+### Let's run everything with JVM runtime, instead of Scala
+export SPARK_LAUNCH_WITH_SCALA=0
+export SPARK_LIBRARY_PATH=${SPARK_HOME}/lib
+export SCALA_LIBRARY_PATH=${SPARK_HOME}/lib
+export SPARK_MASTER_WEBUI_PORT=<%= master_ui_port %>
+export SPARK_MASTER_PORT=<%= master_port %>
+
+### Comment above 2 lines and uncomment the following if
+### you want to run with scala version, that is included with the package
+#export SCALA_HOME=${SCALA_HOME:-/usr/lib/spark/scala}
+#export PATH=$PATH:$SCALA_HOME/bin
+
+### change the following to specify a real cluster's Master host
+export STANDALONE_SPARK_MASTER_HOST=<%= master_host %>
+
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31d51060/bigtop-deploy/puppet/modules/spark/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/spark/tests/init.pp b/bigtop-deploy/puppet/modules/spark/tests/init.pp
new file mode 100644
index 0000000..fab86f1
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/spark/tests/init.pp
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+spark::master { "spark master":
+ master_port => "1978",
+ master_ui_port => "1979",
+}
+
+spark::worker { "spark worker":
+ master_host => "somewhere.com",
+ master_port => "1978",
+ master_ui_port => "1979",
+}