You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2013/08/06 21:31:47 UTC
git commit: AMBARI-2800. YARN and MRv2 should work in kerberos secure
environment. (jaimin)
Updated Branches:
refs/heads/trunk 76a83d5a8 -> 4ac483eb9
AMBARI-2800. YARN and MRv2 should work in kerberos secure environment. (jaimin)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/4ac483eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/4ac483eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/4ac483eb
Branch: refs/heads/trunk
Commit: 4ac483eb94bc4bd360bc6bb2f68c4087dda72f27
Parents: 76a83d5
Author: Jaimin Jetly <ja...@hortonworks.com>
Authored: Tue Aug 6 12:31:11 2013 -0700
Committer: Jaimin Jetly <ja...@hortonworks.com>
Committed: Tue Aug 6 12:31:17 2013 -0700
----------------------------------------------------------------------
.../modules/hdp-hadoop/manifests/namenode.pp | 6 +++++
.../modules/hdp-hadoop/manifests/params.pp | 7 ++++++
.../hdp-hadoop/templates/hadoop-env.sh.erb | 7 ++++--
.../puppet/modules/hdp-hbase/manifests/init.pp | 26 +++-----------------
.../puppet/modules/hdp-yarn/manifests/init.pp | 15 +++++++++++
.../puppet/modules/hdp-yarn/manifests/params.pp | 9 +++++--
.../modules/hdp-yarn/manifests/smoketest.pp | 18 +++++++++++---
.../templates/container-executor.cfg.erb | 22 +++++++++++++++++
.../main/puppet/modules/hdp/manifests/params.pp | 2 ++
.../services/HDFS/configuration/core-site.xml | 1 +
10 files changed, 83 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
index 3209a1f..c203cf3 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
@@ -140,6 +140,12 @@ define hdp-hadoop::namenode::create_app_directories($service_state)
owner => $hdp::params::hbase_user,
service_state => $service_state
}
+ $hbase_staging_dir = $hdp::params::hbase_staging_dir
+ hdp-hadoop::hdfs::directory { $hbase_staging_dir:
+ owner => $hdp::params::hbase_user,
+ service_state => $service_state,
+ mode => '711',
+ }
}
if ($hdp::params::hive_server_host != "") {
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
index f1b96fc..92c38f8 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
@@ -83,6 +83,13 @@ class hdp-hadoop::params(
$mapred_pid_dir_prefix = hdp_default("mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+ ### JSVC_HOME path is correct for AMD64 only, but can be changed through API
+ if ($hdp::params::hdp_os_type == "suse") {
+ $jsvc_path = hdp_default("jsvc_path","/usr/lib/hadoop/sbin/Linux-amd64-64/")
+ } else {
+ $jsvc_path = hdp_default("jsvc_path","/usr/libexec/bigtop-utils")
+ }
+
### compression related
if (($hdp::params::lzo_enabled == true) and ($hdp::params::snappy_enabled == true)) {
$mapred_compress_map_output = true
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb
index 0da12a8..14efb6c 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb
@@ -31,6 +31,9 @@ export HADOOP_HOME_WARN_SUPPRESS=1
#TODO: if env var set that can cause problems
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-<%=scope.function_hdp_template_var("conf_dir")%>}
+# Path to jsvc required by secure HDP 2.0 datanode
+export JSVC_HOME=<%=scope.function_hdp_template_var("jsvc_path")%>
+
# The maximum amount of heap to use, in MB. Default is 1000.
export HADOOP_HEAPSIZE="<%=scope.function_hdp_template_var("hadoop_heapsize")%>"
@@ -106,8 +109,8 @@ do
JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
done
#Add libraries required by nodemanager
-MAPREDUCE_LIBS=<%=scope.function_hdp_template_var("mapreduce_libs_path")%>
+MAPREDUCE_LIBS=<%=scope.function_hdp_template_var("mapreduce_libs_path")%>
export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
# Setting path to hdfs command line
-export HADOOP_LIBEXEC_DIR=<%=scope.function_hdp_template_var("hadoop_libexec_dir")%>
\ No newline at end of file
+export HADOOP_LIBEXEC_DIR=<%=scope.function_hdp_template_var("hadoop_libexec_dir")%>
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp b/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
index bdd7fd2..ff5f3a4 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp
@@ -97,35 +97,15 @@ class hdp-hbase(
hdp-hbase::configfile { 'regionservers':}
if ($security_enabled == true) {
- if ($type == 'master') {
+ if ($type == 'master' and $service_state == 'running') {
hdp-hbase::configfile { 'hbase_master_jaas.conf' : }
} elsif ($type == 'regionserver' and $service_state == 'running') {
-
- $hbase_grant_premissions_file = '/tmp/hbase_grant_permissions.sh'
-
- file { $hbase_grant_premissions_file:
- owner => $hbase_user,
- group => $hdp::params::user_group,
- mode => '0644',
- content => template('hdp-hbase/hbase_grant_permissions.erb')
- }
-
- hdp::exec { '${smokeuser}_grant_privileges' :
- command => "su - ${smoke_test_user} -c 'hbase --config $conf_dir shell ${hbase_grant_premissions_file}'",
- require => File[$hbase_grant_premissions_file]
- }
-
- Hdp-hbase::Configfile<||> -> File[$hbase_grant_premissions_file] ->
- Hdp::Exec['${smokeuser}_grant_privileges'] -> Anchor['hdp-hbase::end']
-
- } elsif ($type == 'regionserver') {
hdp-hbase::configfile { 'hbase_regionserver_jaas.conf' : }
- } else {
+ } elsif ($type == 'client') {
hdp-hbase::configfile { 'hbase_client_jaas.conf' : }
}
}
-
- Anchor['hdp-hbase::begin'] -> Hdp::Package['hbase'] -> Hdp::Directory[$config_dir] ->
+ Anchor['hdp-hbase::begin'] -> Hdp::Package['hbase'] -> Hdp::Directory[$config_dir] ->
Hdp-hbase::Configfile<||> -> Anchor['hdp-hbase::end']
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/init.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/init.pp b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/init.pp
index 20e83d4..bd9a06a 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/init.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/init.pp
@@ -101,4 +101,19 @@ define hdp-yarn::generate_common_configs() {
owner => $hdp-yarn::params::yarn_user,
mode => 755
}
+ if ($hdp::params::security_enabled == true) {
+ $container_executor = "${hdp::params::yarn_container_bin}/container-executor"
+ file { $container_executor:
+ ensure => present,
+ group => 'hadoop',
+ mode => 6050
+ }
+
+ hdp::configfile { "${yarn_config_dir}/container-executor.cfg" :
+ component => 'yarn',
+ owner => 'root',
+ group => $hdp::params::user_group,
+ mode => '0644'
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
index 9cb799f..b6e9781 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
@@ -22,9 +22,14 @@ class hdp-yarn::params(
) inherits hdp-hadoop::params
{
- $conf_dir = $hdp::params::yarn_conf_dir
+ $conf_dir = $hdp::params::yarn_conf_dir
$stack_version = $hdp::params::stack_version
-
+ $smoke_test_user = $hdp::params::smokeuser
+ ## security params
+ $security_enabled = $hdp::params::security_enabled
+ $smoke_user_keytab = $hdp::params::smokeuser_keytab
+ $kinit_cmd = "${hdp::params::kinit_path_local} -kt ${smoke_user_keytab} ${smoke_test_user};"
+
## yarn-env
$hadoop_libexec_dir = hdp_default("yarn/yarn-env/hadoop_libexec_dir","/usr/lib/hadoop/libexec")
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp
index 6ef7bf8..560a0f7 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp
@@ -39,12 +39,23 @@ define hdp-yarn::smoketest(
hdp_fail("Unsupported component name: $component_name")
}
+ $security_enabled = $hdp::params::security_enabled
+ $smoke_user_keytab = $hdp::params::smokeuser_keytab
$smoke_test_user = $hdp::params::smokeuser
+ $kinit_cmd = "${hdp::params::kinit_path_local} -kt ${smoke_user_keytab} ${smoke_test_user};"
+
$validateStatusFileName = "validateYarnComponentStatus.py"
$validateStatusFilePath = "/tmp/$validateStatusFileName"
- $validateStatusCmd = "su - ${smoke_test_user} -c 'python $validateStatusFilePath $component_type -p $component_address'"
+ $validateStatusCmd = "python $validateStatusFilePath $component_type -p $component_address"
+
+ if ($security_enabled == true) {
+ $smoke_cmd = "${kinit_cmd} $validateStatusCmd"
+ } else {
+ $smoke_cmd = $validateStatusCmd
+ }
+
file { $validateStatusFilePath:
ensure => present,
@@ -53,11 +64,12 @@ define hdp-yarn::smoketest(
}
exec { $validateStatusFilePath:
- command => $validateStatusCmd,
+ command => $smoke_cmd,
tries => 3,
try_sleep => 5,
path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- logoutput => "true"
+ logoutput => "true",
+ user => $smoke_test_user
}
anchor{"hdp-yarn::smoketest::begin":} -> File[$validateStatusFilePath] -> Exec[$validateStatusFilePath] -> anchor{"hdp-yarn::smoketest::end":}
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/container-executor.cfg.erb
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/container-executor.cfg.erb b/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/container-executor.cfg.erb
new file mode 100644
index 0000000..b14d4eb
--- /dev/null
+++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/container-executor.cfg.erb
@@ -0,0 +1,22 @@
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+yarn.nodemanager.local-dirs=<%=scope.function_hdp_default(["yarn-site/yarn.nodemanager.local-dirs","/hadoop/yarn"])%>
+yarn.nodemanager.log-dirs=<%=scope.function_hdp_default(["yarn-site/yarn.nodemanager.log-dirs","/var/log/hadoop/yarn"])%>
+yarn.nodemanager.linux-container-executor.group=<%=scope.function_hdp_default(["yarn-site/yarn.nodemanager.linux-container-executor.group","hadoop"])%>
+banned.users = hfds,yarn,mapred,bin
+min.user.id=1000
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
index b27bff5..ef9d4d4 100644
--- a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
@@ -231,6 +231,7 @@ class hdp::params()
$hive_apps_whs_dir = hdp_default("hive_apps_whs_dir", "/apps/hive/warehouse")
$webhcat_apps_dir = hdp_default("webhcat_apps_dir", "/apps/webhcat")
$hbase_hdfs_root_dir = hdp_default("hbase-site/hbase.hdfs.root.dir","/apps/hbase/data")
+ $hbase_staging_dir = hdp_default("hbase-site/hbase.bulkload.staging.dir","/apps/hbase/staging")
$yarn_nm_app_log_dir = hdp_default("yarn-site/yarn.nodemanager.remote-app-log-dir","/app-logs")
@@ -353,6 +354,7 @@ class hdp::params()
$hadoop_deps = ['hadoop','hadoop-libhdfs','hadoop-native','hadoop-pipes','hadoop-sbin','hadoop-lzo', 'hadoop-lzo-native']
}
$yarn_bin = "/usr/lib/hadoop-yarn/sbin"
+ $yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
$mapred_bin = "/usr/lib/hadoop-mapreduce/sbin"
$hadoop_conf_dir = "/etc/hadoop/conf"
$yarn_conf_dir = "/etc/hadoop/conf"
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4ac483eb/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/core-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/core-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/core-site.xml
index e646d5b..ed6523c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/core-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/core-site.xml
@@ -160,6 +160,7 @@
<name>hadoop.security.auth_to_local</name>
<value>
RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/
+ RULE:[2:$1@$0](jhs@.*)s/.*/mapred/
RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/
RULE:[2:$1@$0](hm@.*)s/.*/hbase/
RULE:[2:$1@$0](rs@.*)s/.*/hbase/