You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2016/07/29 17:03:16 UTC
[35/51] [abbrv] bigtop git commit: BIGTOP-2357: Add Apache Flink to
Bigtop / Create puppet recipes
BIGTOP-2357: Add Apache Flink to Bigtop / Create puppet recipes
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/3e0de8c0
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/3e0de8c0
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/3e0de8c0
Branch: refs/heads/BIGTOP-2253
Commit: 3e0de8c04bc8f6cb5993d3c08d1f56a837958db1
Parents: b5d001d
Author: Robert Metzger <rm...@apache.org>
Authored: Tue May 24 14:51:48 2016 +0200
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Fri Jun 17 16:29:40 2016 -0700
----------------------------------------------------------------------
.../puppet/hieradata/bigtop/cluster.yaml | 9 ++-
bigtop-deploy/puppet/hieradata/site.yaml | 1 +
bigtop-deploy/puppet/manifests/cluster.pp | 5 ++
.../puppet/modules/flink/manifests/init.pp | 62 ++++++++++++++++++++
.../modules/flink/templates/flink-conf.yaml | 31 ++++++++++
bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile | 1 +
6 files changed, 108 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index f592f64..5c2c5f1 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -16,7 +16,7 @@
# "$components" list. If $components isn't set then everything in the stack will
# be installed as usual. Otherwise only a specified list will be set
# Possible elements:
-# hadoop,yarn,hbase,alluxio,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs,
+# hadoop,yarn,hbase,alluxio,flink,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs,
# hue,mahout,giraph,crunch,pig,hive,zookeeper,ycsb,qfs
# Example (to deploy only HDFS and YARN server and gateway parts)
# This can be a comma-separated list or an array.
@@ -191,3 +191,10 @@ zeppelin::server::spark_master_url: "yarn-client"
zeppelin::server::hiveserver2_url: "jdbc:hive2://%{hiera('hadoop-hive::common::hiveserver2_host')}:%{hiera('hadoop-hive::common::hiveserver2_port')}"
zeppelin::server::hiveserver2_user: "%{hiera('bigtop::hiveserver2_user')}"
zeppelin::server::hiveserver2_password: "%{hiera('bigtop::hiveserver2_password')}"
+
+# Flink
+flink::common::jobmanager_host: "%{hiera('bigtop::hadoop_head_node')}"
+flink::common::jobmanager_port: "6123"
+
+flink::common::ui_port: "8081"
+flink::common::storage_dirs: "%{hiera('hadoop::hadoop_storage_dirs')}"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/hieradata/site.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml
index d429269..b9ed6f9 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -15,6 +15,7 @@ hadoop::hadoop_storage_dirs:
# - alluxio
# - apex
# - crunch
+# - flink
# - flume
# - giraph
# - ignite_hadoop
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index ba980bd..9ff424c 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -56,6 +56,10 @@ $roles_map = {
master => ["alluxio-master"],
worker => ["alluxio-worker"],
},
+ flink => {
+ master => ["flink-jobmanager"],
+ worker => ["flink-taskmanager"],
+ },
flume => {
worker => ["flume-agent"],
},
@@ -161,6 +165,7 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_
"alluxio",
"apex",
"crunch",
+ "flink",
"giraph",
"hadoop",
"hadoop_hbase",
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/modules/flink/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/flink/manifests/init.pp b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
new file mode 100644
index 0000000..ae79fe4
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
@@ -0,0 +1,62 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+class flink {
+
+ class deploy ($roles) {
+ if ("flink-jobmanager" in $roles) {
+ include flink::jobmanager
+ }
+
+ if ("flink-taskmanager" in $roles) {
+ include flink::taskmanager
+ }
+ }
+
+ class common($jobmanager_host, $jobmanager_port, $ui_port, $storage_dirs) {
+ # make sure flink is installed
+ package { "flink":
+ ensure => latest
+ }
+
+ # set values in flink-conf.yaml
+ file { "/etc/flink/conf/flink-conf.yaml":
+ content => template("flink/flink-conf.yaml"),
+ require => Package["flink"]
+ }
+ }
+
+ class jobmanager {
+ include common
+
+ service { "flink-jobmanager":
+ ensure => running,
+ require => Package["flink"],
+ subscribe => File["/etc/flink/conf/flink-conf.yaml"],
+ hasrestart => true,
+ hasstatus => true
+
+ }
+
+ }
+
+ class taskmanager {
+ include common
+
+ service { "flink-taskmanager":
+ ensure => running,
+ require => Package["flink"],
+ subscribe => File["/etc/flink/conf/flink-conf.yaml"],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
new file mode 100644
index 0000000..14e8853
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
@@ -0,0 +1,31 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+# Configuration values managed by puppet:
+jobmanager.rpc.address: <%= @jobmanager_host %>
+jobmanager.rpc.port: <%= @jobmanager_port %>
+jobmanager.web.port: <%= @ui_port %>
+
+<% if defined?(storage_dirs) %>
+taskmanager.tmp.dirs: <%= @storage_dirs.join(":") %>
+<% end %>
+
+
+# For performance reasons its highly recommended to allocate as much memory to the
+# Flink TaskManager as possible by setting 'taskmanager.heap.mb'.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
index 63aff20..b4017d0 100755
--- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
@@ -67,6 +67,7 @@ bigtop_master = "bigtop1.vagrant"
$script = <<SCRIPT
service iptables stop
+service firewalld stop
chkconfig iptables off
# Remove 127.0.0.1 entry since vagrant's hostname setting will map it to FQDN,
# which miss leads some daemons to bind on 127.0.0.1 instead of public or private IP