You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2015/05/07 21:41:53 UTC
bigtop git commit: BIGTOP-1657. bigtop-deploy puppets to support tez
Repository: bigtop
Updated Branches:
refs/heads/master bc8a256df -> 0c0e2fca4
BIGTOP-1657. bigtop-deploy puppets to support tez
Signed-off-by: Olaf Flebbe <of...@oflebbe.de>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/0c0e2fca
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/0c0e2fca
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/0c0e2fca
Branch: refs/heads/master
Commit: 0c0e2fca427ec30952eccf2019a7b0efb6b68b45
Parents: bc8a256
Author: Evans Ye <ev...@apache.org>
Authored: Wed May 6 03:35:40 2015 +0000
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu May 7 21:41:00 2015 +0200
----------------------------------------------------------------------
.../puppet/hieradata/bigtop/cluster.yaml | 4 ++
bigtop-deploy/puppet/hieradata/site.yaml | 1 +
bigtop-deploy/puppet/manifests/cluster.pp | 8 ++++
.../modules/hadoop-hive/templates/hive-site.xml | 7 +++
.../puppet/modules/hadoop/manifests/init.pp | 2 +-
.../modules/hadoop/templates/hadoop-env.sh | 12 +++--
.../modules/hadoop/templates/mapred-site.xml | 17 +++++++
.../puppet/modules/tez/manifests/init.pp | 22 +++++++++
bigtop-deploy/puppet/modules/tez/tests/init.pp | 17 +++++++
.../src/common/bigtop-utils/init-hcfs.groovy | 48 ++++++++++++++++++--
.../src/common/hadoop/init-hcfs.json | 5 +-
11 files changed, 134 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 1ea6407..44cb1b7 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -137,3 +137,7 @@ hue::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
giraph::client::zookeeper_quorum: "%{hiera('bigtop::hadoop_head_node')}"
hadoop-hive::client::hbase_zookeeper_quorum: "%{hiera('hadoop-hbase::common_config::zookeeper_quorum')}"
+
+# tez
+hadoop::common::tez_conf_dir: "/etc/tez/conf"
+hadoop::common::tez_jars: "/usr/lib/tez"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/hieradata/site.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml
index 54060f8..00861c4 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -29,6 +29,7 @@ hadoop::hadoop_storage_dirs:
# - spark
# - sqoop
# - tachyon
+# - tez
# - yarn
# - zookeeper
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 43e090f..35ef195 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -147,6 +147,11 @@ if ($hadoop_security_authentication == "kerberos") {
include hadoop-zookeeper::server
}
+ if ($all or "tez" in $components) {
+ include tez::client
+ Class['tez::client'] -> Exec<| title == "init hdfs" |>
+ }
+
# class hadoop::rsync_hdfs isn't used anywhere
#Exec<| title == "init hdfs" |> -> Class['Hadoop::Rsync_hdfs']
}
@@ -203,4 +208,7 @@ class hadoop_gateway_node inherits hadoop_cluster_node {
if ($all or "zookeeper" in $components) {
include hadoop-zookeeper::client
}
+ if ($all or "tez" in $components) {
+ include tez::client
+ }
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
index 5861d97..4e33907 100644
--- a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
@@ -42,6 +42,13 @@
</property>
<% end %>
+<% if @all or @components.include? "tez" -%>
+<property>
+ <name>hive.execution.engine</name>
+ <value>tez</value>
+</property>
+<% end %>
+
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true</value>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 45168ba..749773d 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -294,7 +294,7 @@ class hadoop ($hadoop_security_authentication = "simple",
$mapreduce_jobhistory_host = undef,
$mapreduce_jobhistory_port = "10020",
$mapreduce_jobhistory_webapp_port = "19888",
- $mapreduce_framework_name = "yarn",
+ $mapreduce_framework_name = undef,
$mapred_data_dirs = suffix($hadoop::hadoop_storage_dirs, "/mapred"),
$mapreduce_cluster_temp_dir = "/mapred/system",
$yarn_app_mapreduce_am_staging_dir = "/user",
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
index 318589f..f959f5b 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
@@ -34,10 +34,6 @@
# The java implementation to use. Required.
<%= shell_config("JAVA_HOME", "hadoop_java_home") %>
-# tez environment, needed to enable tez
-<%= shell_config("TEZ_CONF_DIR") %>
-<%= shell_config("TEZ_JARS") %>
-
# Extra Java CLASSPATH elements. Optional.
<%= shell_config("HADOOP_CLASSPATH") %>
@@ -84,6 +80,14 @@
# The scheduling priority for daemon processes. See 'man nice'.
<%= shell_config("HADOOP_NICENESS") %>
+# tez environment, needed to enable tez
+<% if (@all or @components.include? "tez") -%>
+<%= shell_config("TEZ_CONF_DIR") %>
+<%= shell_config("TEZ_JARS") %>
+# Add tez into HADOOP_CLASSPATH
+export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*
+<% end -%>
+
### WARNING: the following is NOT really optional. It is a shame that stock Hadoop
### hadoop_env.sh doesn't make it clear -- you can NOT turn com.sun.management.jmxremote off
### and have a working Hadoop cluster.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
index 1c8f3fd..2723e82 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
@@ -86,11 +86,28 @@
</property>
<% end %>
+<% if defined? @mapreduce_framework_name -%>
<property>
<name>mapreduce.framework.name</name>
<value><%= @mapreduce_framework_name %></value>
</property>
+<!-- If not specified, we automatically choose the best framework for users -->
+<% else -%>
+<% if @all or @components.include? "tez" -%>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn-tez</value>
+ </property>
+
+<% else -%>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+
+<% end -%>
+<% end -%>
<property>
<name>mapreduce.cluster.local.dir</name>
<value><%= @mapred_data_dirs.join(",") %></value>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/tez/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/tez/manifests/init.pp b/bigtop-deploy/puppet/modules/tez/manifests/init.pp
new file mode 100644
index 0000000..6db2220
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/tez/manifests/init.pp
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class tez {
+ class client {
+ package { "tez":
+ ensure => latest,
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-deploy/puppet/modules/tez/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/tez/tests/init.pp b/bigtop-deploy/puppet/modules/tez/tests/init.pp
new file mode 100644
index 0000000..9ee0636
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/tez/tests/init.pp
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include tez
+tez::client { "test-tez": }
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy b/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy
index a224843..d07ef01 100644
--- a/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy
+++ b/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy
@@ -139,6 +139,9 @@ def final FileSystem fs = FileSystem.get(conf);
LOG.info("PROVISIONING WITH FILE SYSTEM : " + fs.getClass());
+// Longest back off time to check whether the file system is ready for write
+def final int maxBackOff = 64;
+
/**
* Make a directory. Note when providing input to this functino that if
* nulls are given, the commands will work but behaviour varies depending on
@@ -150,7 +153,19 @@ LOG.info("PROVISIONING WITH FILE SYSTEM : " + fs.getClass());
* @param group can be null,
*/
def mkdir = { FileSystem fsys, Path dname, FsPermission mode, String user, String group ->
- fsys.mkdirs(dname);
+ boolean success = false;
+ for(i = 1; i <= maxBackOff; i*=2) {
+ try {
+ success = fsys.mkdirs(dname)
+ break;
+ } catch(Exception e) {
+ LOG.info("Failed to create directory " + dname + "... Retry after " + i + " second(s)");
+ Thread.sleep(i*1000);
+ }
+ }
+ if (!success) {
+ LOG.info("Can not create directory " + dname + " on " + fsys.getClass());
+ }
if (user != null) {
fsys.setOwner(dname, user, group);
}
@@ -256,8 +271,21 @@ def copyJars = { FileSystem fsys, File input, String jarstr, Path target ->
return validJar && filename.contains(jarstr)
}
}).each({ jar_file ->
- copied++;
- fsys.copyFromLocalFile(new Path(jar_file.getAbsolutePath()), target)
+ boolean success = false;
+ for(i = 1; i <= maxBackOff; i*=2) {
+ try {
+ fsys.copyFromLocalFile(new Path(jar_file.getAbsolutePath()), target)
+ copied++;
+ success = true;
+ break;
+ } catch(Exception e) {
+ LOG.info("Failed to upload " + jar_file.getAbsolutePath() + " to " + target + "... Retry after " + i + " second(s)");
+ Thread.sleep(i*1000);
+ }
+ if (!success) {
+ LOG.info("Can not upload " + jar_file.getAbsolutePath() + " to " + target + " on " + fsys.getClass());
+ }
+ }
});
return copied;
}
@@ -298,4 +326,18 @@ total_jars += copyJars(fs,
new File(PIG_HOME), "",
new Path(OOZIE_SHARE, "pig"))
+LOG.info("Now copying Jars into the DFS for tez ");
+LOG.info("This might take a few seconds...");
+
+def final TEZ_APPS = "/apps";
+def final TEZ_HOME = "/usr/lib/tez/";
+
+total_jars += copyJars(fs,
+ new File(TEZ_HOME, "lib/"), "",
+ new Path(TEZ_APPS, "tez/lib"))
+
+total_jars += copyJars(fs,
+ new File(TEZ_HOME), "",
+ new Path(TEZ_APPS, "tez"))
+
LOG.info("Total jars copied into the DFS : " + total_jars);
http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c0e2fca/bigtop-packages/src/common/hadoop/init-hcfs.json
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.json b/bigtop-packages/src/common/hadoop/init-hcfs.json
index 88beb1a..7f9b6cc 100644
--- a/bigtop-packages/src/common/hadoop/init-hcfs.json
+++ b/bigtop-packages/src/common/hadoop/init-hcfs.json
@@ -88,7 +88,10 @@
["/user/oozie/share/lib/distcp", null, null, null],
["/user/oozie/share/lib/pig", null, null, null],
["/var/log/spark/apps","1777","spark","spark"],
- ["/user/hbase","755","hbase",null]
+ ["/user/hbase","755","hbase",null],
+ ["/apps",null,null,null],
+ ["/apps/tez",null,null,null],
+ ["/apps/tez/lib",null,null,null]
],
"user": [
["tom", "0755", null],