You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2017/03/25 02:25:28 UTC
bigtop git commit: BIGTOP-2708. puppet code always installs hdfs by
default
Repository: bigtop
Updated Branches:
refs/heads/master 047e4ddd7 -> 31fa70d73
BIGTOP-2708. puppet code always installs hdfs by default
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/31fa70d7
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/31fa70d7
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/31fa70d7
Branch: refs/heads/master
Commit: 31fa70d73af117407f00cf2ba473a76074f472ba
Parents: 047e4dd
Author: Roman Shaposhnik <rv...@apache.org>
Authored: Sat Mar 25 02:21:38 2017 +0000
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Fri Mar 24 19:24:26 2017 -0700
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 31 +++++++++++++++++---
bigtop-deploy/puppet/manifests/site.pp | 5 +++-
.../puppet/modules/hadoop/manifests/init.pp | 4 +--
.../pseudo-cluster/config/hieradata/site.yaml | 3 +-
.../sandbox/site.yaml.template.centos-6_hadoop | 2 +-
.../sandbox/site.yaml.template.debian-8_hadoop | 2 +-
provisioner/docker/config_centos6.yaml | 4 +--
provisioner/docker/config_centos7.yaml | 4 +--
provisioner/docker/config_debian8.yaml | 4 +--
provisioner/docker/config_ubuntu_trusty.yaml | 25 ----------------
provisioner/docker/config_ubuntu_xenial.yaml | 25 ++++++++++++++++
provisioner/vagrant/vagrantconfig.yaml | 4 +--
12 files changed, 70 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index d9f6e01..1313082 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -13,6 +13,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# The following is a bit of a tricky map. The idea here is that the keys
+# correspond to anything that could be specified in
+# hadoop_cluster_node::cluster_components:
+# The values are maps from each role that a node can have in a cluster:
+# client, gateway_server, library, master, worker, standby
+# to a role recognized by each puppet module's deploy class.
+#
+# Note that the code here will pass all these roles to all the deploy
+# classes defined in every single Bigtop's puppet module. This is similar
+# to how a visitor pattern works in OOP. One subtle ramification of this
+# approach is that you should make sure that deploy classes from different
+# modules do NOT accept same strings for role types.
+#
+# And if that wasn't enough of a head scratcher -- you also need to keep
+# in mind that there's no hdfs key in the following map, even though it
+# is a perfectly legal value for hadoop_cluster_node::cluster_components:
+# The reason for this is that hdfs is treated as an alias for either
+# hdfs-non-ha or hdfs-ha depending on whether HA for HDFS is either enabled
+# or disabled.
+
$roles_map = {
apex => {
client => ["apex-client"],
@@ -217,14 +237,17 @@ class node_with_components inherits hadoop_cluster_node {
}
$given_components = $components_array[0] ? {
- "all" => delete(keys($roles_map), ["hdfs-non-ha", "hdfs-ha"]),
+ "all" => delete(keys($roles_map), ["hdfs-non-ha", "hdfs-ha"]) << "hdfs",
default => $components_array,
}
$ha_dependent_components = $ha_enabled ? {
- true => ["hdfs-ha"],
- default => ["hdfs-non-ha"],
+ true => "hdfs-ha",
+ default => "hdfs-non-ha",
+ }
+ $components = member($given_components, "hdfs") ? {
+ true => delete($given_components, "hdfs") << $ha_dependent_components,
+ default => $given_components
}
- $components = concat($given_components, $ha_dependent_components)
$master_role_types = ["master", "worker", "library"]
$standby_role_types = ["standby", "library"]
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/bigtop-deploy/puppet/manifests/site.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/site.pp b/bigtop-deploy/puppet/manifests/site.pp
index ce5857f..f35a4e0 100644
--- a/bigtop-deploy/puppet/manifests/site.pp
+++ b/bigtop-deploy/puppet/manifests/site.pp
@@ -69,7 +69,10 @@ if ($provision_repo) {
case $::operatingsystem {
/Debian/: {
require apt
- require apt::backports
+ class { 'apt::backports':
+ pin => 500,
+ }
+ Class['apt::backports'] -> Package <||>
package { "jdk":
name => $jdk_package_name,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 30a68e2..7d8e48f 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -45,7 +45,7 @@ class hadoop ($hadoop_security_authentication = "simple",
}
}
- if ($hadoop::common_hdfs::ha != "disabled" and "standby-namenode" in $roles) {
+ if ("standby-namenode" in $roles and $hadoop::common_hdfs::ha != "disabled") {
include hadoop::namenode
}
@@ -67,7 +67,7 @@ class hadoop ($hadoop_security_authentication = "simple",
}
}
- if ($hadoop::common_hdfs::ha == "disabled" and "secondarynamenode" in $roles) {
+ if ("secondarynamenode" in $roles and $hadoop::common_hdfs::ha == "disabled") {
include hadoop::secondarynamenode
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/docker/pseudo-cluster/config/hieradata/site.yaml
----------------------------------------------------------------------
diff --git a/docker/pseudo-cluster/config/hieradata/site.yaml b/docker/pseudo-cluster/config/hieradata/site.yaml
index a8ab29d..1d29859 100644
--- a/docker/pseudo-cluster/config/hieradata/site.yaml
+++ b/docker/pseudo-cluster/config/hieradata/site.yaml
@@ -3,8 +3,9 @@ hadoop::hadoop_storage_dirs:
- /data/1
- /data/2
hadoop_cluster_node::cluster_components:
- - hadoop
+ - hdfs
- yarn
+ - mapreduce
bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64
bigtop::jdk_package_name: openjdk-8-jdk
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/docker/sandbox/site.yaml.template.centos-6_hadoop
----------------------------------------------------------------------
diff --git a/docker/sandbox/site.yaml.template.centos-6_hadoop b/docker/sandbox/site.yaml.template.centos-6_hadoop
index f38c4fb..554431e 100644
--- a/docker/sandbox/site.yaml.template.centos-6_hadoop
+++ b/docker/sandbox/site.yaml.template.centos-6_hadoop
@@ -16,5 +16,5 @@
bigtop::hadoop_head_node: "head.node.fqdn"
hadoop::hadoop_storage_dirs: [/data/1, /data/2]
bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64
-hadoop_cluster_node::cluster_components: [hadoop, yarn]
+hadoop_cluster_node::cluster_components: [hdfs, yarn, mapreduce]
bigtop::jdk_package_name: java-1.8.0-openjdk-devel.x86_64
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/docker/sandbox/site.yaml.template.debian-8_hadoop
----------------------------------------------------------------------
diff --git a/docker/sandbox/site.yaml.template.debian-8_hadoop b/docker/sandbox/site.yaml.template.debian-8_hadoop
index 9a00c43..d9c7c7f 100644
--- a/docker/sandbox/site.yaml.template.debian-8_hadoop
+++ b/docker/sandbox/site.yaml.template.debian-8_hadoop
@@ -16,5 +16,5 @@
bigtop::hadoop_head_node: "head.node.fqdn"
hadoop::hadoop_storage_dirs: [/data/1, /data/2]
bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64
-hadoop_cluster_node::cluster_components: [hadoop, yarn]
+hadoop_cluster_node::cluster_components: [hdfs, yarn, mapreduce]
bigtop::jdk_package_name: openjdk-8-jdk
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_centos6.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_centos6.yaml b/provisioner/docker/config_centos6.yaml
index 1f06731..9b7b6ee 100644
--- a/provisioner/docker/config_centos6.yaml
+++ b/provisioner/docker/config_centos6.yaml
@@ -19,7 +19,7 @@ docker:
repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64"
distro: centos
-components: [hadoop, yarn]
+components: [hdfs, yarn, mapreduce]
enable_local_repo: false
-smoke_test_components: [mapreduce, pig]
+smoke_test_components: [hdfs, yarn, mapreduce]
jdk: "java-1.8.0-openjdk-devel.x86_64"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_centos7.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml
index 95724ff..3fe66e5 100644
--- a/provisioner/docker/config_centos7.yaml
+++ b/provisioner/docker/config_centos7.yaml
@@ -19,7 +19,7 @@ docker:
repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"
distro: centos
-components: [hadoop, yarn]
+components: [hdfs, yarn, mapreduce]
enable_local_repo: false
-smoke_test_components: [mapreduce, pig]
+smoke_test_components: [hdfs, yarn, mapreduce]
jdk: "java-1.8.0-openjdk-devel.x86_64"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_debian8.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_debian8.yaml b/provisioner/docker/config_debian8.yaml
index ea8d150..d3e2399 100644
--- a/provisioner/docker/config_debian8.yaml
+++ b/provisioner/docker/config_debian8.yaml
@@ -19,7 +19,7 @@ docker:
repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64"
distro: debian
-components: [hadoop, yarn]
+components: [hdfs, yarn, mapreduce]
enable_local_repo: false
-smoke_test_components: [mapreduce, pig]
+smoke_test_components: [hdfs, yarn, mapreduce]
jdk: "openjdk-8-jdk"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_ubuntu_trusty.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_ubuntu_trusty.yaml b/provisioner/docker/config_ubuntu_trusty.yaml
deleted file mode 100644
index a56e97a..0000000
--- a/provisioner/docker/config_ubuntu_trusty.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-docker:
- memory_limit: "4g"
- image: "bigtop/puppet:ubuntu-14.04"
-
-repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64"
-distro: debian
-components: [hadoop, yarn]
-enable_local_repo: false
-smoke_test_components: [mapreduce, pig]
-jdk: "openjdk-8-jdk"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/docker/config_ubuntu_xenial.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_ubuntu_xenial.yaml b/provisioner/docker/config_ubuntu_xenial.yaml
new file mode 100644
index 0000000..3b454fa
--- /dev/null
+++ b/provisioner/docker/config_ubuntu_xenial.yaml
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_limit: "4g"
+ image: "bigtop/puppet:ubuntu-16.04"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64"
+distro: debian
+components: [hdfs, yarn, mapreduce]
+enable_local_repo: false
+smoke_test_components: [hdfs, yarn, mapreduce]
+jdk: "openjdk-8-jdk"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/31fa70d7/provisioner/vagrant/vagrantconfig.yaml
----------------------------------------------------------------------
diff --git a/provisioner/vagrant/vagrantconfig.yaml b/provisioner/vagrant/vagrantconfig.yaml
index aabc7bd..4974104 100644
--- a/provisioner/vagrant/vagrantconfig.yaml
+++ b/provisioner/vagrant/vagrantconfig.yaml
@@ -19,8 +19,8 @@ box: "puppetlabs/centos-7.2-64-nocm"
repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"
num_instances: 1
distro: centos
-components: [hadoop, yarn, hbase]
+components: [hdfs, yarn, mapreduce]
enable_local_repo: false
run_smoke_tests: false
-smoke_test_components: [mapreduce, pig]
+smoke_test_components: [hdfs, yarn, mapreduce]
jdk: "java-1.8.0-openjdk-devel.x86_64"