You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2013/08/21 20:41:25 UTC
git commit: AMBARI-2970. NameNode fails to come up in HA mode.
(swagle)
Updated Branches:
refs/heads/trunk 81cd63d9e -> 67fd6fb7c
AMBARI-2970. NameNode fails to come up in HA mode. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/67fd6fb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/67fd6fb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/67fd6fb7
Branch: refs/heads/trunk
Commit: 67fd6fb7c489b44b83f8552f61e35b28aab5cbc5
Parents: 81cd63d
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Wed Aug 21 11:36:12 2013 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Wed Aug 21 11:39:13 2013 -0700
----------------------------------------------------------------------
.../functions/hdp_hadoop_get_namenode_id.rb | 47 ++++++++++++++++++++
.../hdp-hadoop/manifests/hdfs/directory.pp | 21 ++++++---
.../puppet/modules/hdp-hadoop/manifests/init.pp | 13 ++++--
.../modules/hdp-hadoop/manifests/namenode.pp | 44 +++++++++++-------
.../modules/hdp-hadoop/manifests/params.pp | 5 +++
5 files changed, 103 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/lib/puppet/parser/functions/hdp_hadoop_get_namenode_id.rb
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/lib/puppet/parser/functions/hdp_hadoop_get_namenode_id.rb b/ambari-agent/src/main/puppet/modules/hdp-hadoop/lib/puppet/parser/functions/hdp_hadoop_get_namenode_id.rb
new file mode 100644
index 0000000..719d1e6
--- /dev/null
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/lib/puppet/parser/functions/hdp_hadoop_get_namenode_id.rb
@@ -0,0 +1,47 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+#to get namenode service id in HA setup
+
+module Puppet::Parser::Functions
+ newfunction(:hdp_hadoop_get_namenode_id, :type => :rvalue) do |args|
+ namenode_id = ""
+ if args.length > 1
+ # Get hdfs-site to lookup hostname properties
+ lookup_property = args[0]
+ siteName = args[1]
+ siteConfig = lookupvar("#{siteName}")
+ nn_ids_str = lookupvar("::hdp::params::dfs_ha_namenode_ids")
+ hostname = lookupvar("::hdp::params::hostname")
+ nn_ids = nn_ids_str.to_s.split(',')
+
+ if nn_ids.length > 1
+ nn_ids.each do |id|
+ lookup_key = lookup_property + "." + id.to_s.strip
+ property_val = siteConfig.fetch(lookup_key, "")
+ if property_val != "" and property_val.include? hostname
+ namenode_id = id
+ end
+ end
+ end
+ end
+ namenode_id.strip
+ end
+end
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp
index 453aeb3..78ac747 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp
@@ -32,22 +32,31 @@ define hdp-hadoop::hdfs::directory(
$namenode_safe_mode_off = "hadoop dfsadmin -safemode get|grep 'Safe mode is OFF'"
$tries = 30
$try_sleep = 10
-
- if ($service_state == 'running') {
+ if ($hdp::params::dfs_ha_enabled == true) {
+ $namenode_id = $hdp-hadoop::params::namenode_id
+ if (hdp_is_empty($namenode_id) == false) {
+ $dfs_check_nn_status_cmd = "hdfs haadmin -getServiceState $namenode_id | grep active > /dev/null"
+ }
+ } else {
+ $dfs_check_nn_status_cmd = "true"
+ }
+ if ($service_state == 'running') {
if (hdp_get_major_stack_version($hdp::params::stack_version) >= 2) {
$mkdir_cmd = "fs -mkdir -p ${name}"
} else {
$mkdir_cmd = "fs -mkdir ${name}"
}
+
hdp-hadoop::exec-hadoop { $mkdir_cmd:
command => $mkdir_cmd,
- unless => $dir_exists,
- onlyif => $namenode_safe_mode_off,
+ unless => "$dfs_check_nn_status_cmd && $dir_exists",
+ onlyif => "$dfs_check_nn_status_cmd && $namenode_safe_mode_off",
try_sleep => $try_sleep,
tries => $tries
}
+
if ($owner == unset) {
$chown = ""
} else {
@@ -67,7 +76,7 @@ define hdp-hadoop::hdfs::directory(
}
hdp-hadoop::exec-hadoop {$chown_cmd :
command => $chown_cmd,
- onlyif => "$namenode_safe_mode_off && $dir_exists",
+ onlyif => "$dfs_check_nn_status_cmd && $namenode_safe_mode_off && $dir_exists",
try_sleep => $try_sleep,
tries => $tries
}
@@ -83,7 +92,7 @@ define hdp-hadoop::hdfs::directory(
}
hdp-hadoop::exec-hadoop {$chmod_cmd :
command => $chmod_cmd,
- onlyif => "$namenode_safe_mode_off && $dir_exists",
+ onlyif => "$dfs_check_nn_status_cmd && $namenode_safe_mode_off && $dir_exists",
try_sleep => $try_sleep,
tries => $tries
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp
index bd63543..b9624c4 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/init.pp
@@ -408,7 +408,8 @@ define hdp-hadoop::exec-hadoop(
$try_sleep = undef,
$user = undef,
$logoutput = undef,
- $onlyif = undef
+ $onlyif = undef,
+ $path = undef
)
{
include hdp-hadoop::params
@@ -439,10 +440,14 @@ define hdp-hadoop::exec-hadoop(
$kinit_if_needed = ""
}
- if ($echo_yes == true) {
- $cmd = "yes Y | hadoop --config ${conf_dir} ${command}"
+ if ($path == undef) {
+ if ($echo_yes == true) {
+ $cmd = "yes Y | hadoop --config ${conf_dir} ${command}"
+ } else {
+ $cmd = "hadoop --config ${conf_dir} ${command}"
+ }
} else {
- $cmd = "hadoop --config ${conf_dir} ${command}"
+ $cmd = "${path} ${command}"
}
if ($kinit_if_needed != "") {
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
index c203cf3..4ae78ae 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp
@@ -63,7 +63,7 @@ class hdp-hadoop::namenode(
group => $hdp::params::user_group
}
}
-
+
hdp-hadoop::namenode::create_name_dirs { $dfs_name_dir:
service_state => $service_state
}
@@ -82,20 +82,26 @@ class hdp-hadoop::namenode(
}
hdp-hadoop::namenode::create_app_directories { 'create_app_directories' :
- service_state => $service_state
+ service_state => $service_state
}
hdp-hadoop::namenode::create_user_directories { 'create_user_directories' :
- service_state => $service_state
+ service_state => $service_state
}
- #top level does not need anchors
- Anchor['hdp-hadoop::begin'] -> Hdp-hadoop::Namenode::Create_name_dirs<||> -> Hdp-hadoop::Service['namenode'] ->
- # Now, creating directories inside HDFS
- Hdp-hadoop::Namenode::Create_app_directories<||> -> Hdp-hadoop::Namenode::Create_user_directories<||> -> Anchor['hdp-hadoop::end']
+ Anchor['hdp-hadoop::begin'] ->
+ Hdp-hadoop::Namenode::Create_name_dirs<||> ->
+ Hdp-hadoop::Service['namenode'] ->
+ Hdp-hadoop::Namenode::Create_app_directories<||> ->
+ Hdp-hadoop::Namenode::Create_user_directories<||> ->
+ Anchor['hdp-hadoop::end']
+
if ($service_state == 'running' and $format == true) {
- Anchor['hdp-hadoop::begin'] -> Hdp-hadoop::Namenode::Create_name_dirs<||> ->
- Class['hdp-hadoop::namenode::format'] -> Hdp-hadoop::Service['namenode'] -> Anchor['hdp-hadoop::end']
+ Anchor['hdp-hadoop::begin'] ->
+ Hdp-hadoop::Namenode::Create_name_dirs<||> ->
+ Class['hdp-hadoop::namenode::format'] ->
+ Hdp-hadoop::Service['namenode'] ->
+ Anchor['hdp-hadoop::end']
}
} else {
hdp_fail("TODO not implemented yet: service_state = ${service_state}")
@@ -128,10 +134,12 @@ define hdp-hadoop::namenode::create_app_directories($service_state)
service_state => $service_state,
owner => $hdp-hadoop::params::mapred_user
}
+
hdp-hadoop::hdfs::directory{ '/mapred/system' :
service_state => $service_state,
owner => $hdp-hadoop::params::mapred_user
}
+
Hdp-hadoop::Hdfs::Directory['/mapred'] -> Hdp-hadoop::Hdfs::Directory['/mapred/system']
if ($hdp::params::hbase_master_hosts != "") {
@@ -140,12 +148,13 @@ define hdp-hadoop::namenode::create_app_directories($service_state)
owner => $hdp::params::hbase_user,
service_state => $service_state
}
+
$hbase_staging_dir = $hdp::params::hbase_staging_dir
- hdp-hadoop::hdfs::directory { $hbase_staging_dir:
- owner => $hdp::params::hbase_user,
- service_state => $service_state,
- mode => '711',
- }
+ hdp-hadoop::hdfs::directory { $hbase_staging_dir:
+ owner => $hdp::params::hbase_user,
+ service_state => $service_state,
+ mode => '711'
+ }
}
if ($hdp::params::hive_server_host != "") {
@@ -224,7 +233,7 @@ define hdp-hadoop::namenode::create_user_directories($service_state)
$hive_hdfs_user_dir = $hdp::params::hive_hdfs_user_dir
$hive_dir_item="$hive_hdfs_user_dir,"
} else {
- $hive_dir_item=""
+ $hive_dir_item=""
}
if ($hdp::params::oozie_server != "") {
@@ -252,8 +261,9 @@ define hdp-hadoop::namenode::create_user_directories($service_state)
#Get unique users directories set
$users_dirs_set = hdp_set_from_comma_list($users_dir_list_comm_sep)
- hdp-hadoop::namenode::create_user_directory{$users_dirs_set:
- service_state => $service_state}
+ hdp-hadoop::namenode::create_user_directory{ $users_dirs_set:
+ service_state => $service_state
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/67fd6fb7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
index df3492e..3cc765c 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
@@ -202,4 +202,9 @@ class hdp-hadoop::params(
$ambari_db_rca_username = hdp_default("ambari_db_rca_username", "mapred")
$ambari_db_rca_password = hdp_default("ambari_db_rca_password", "mapred")
+ if ($hdp::params::dfs_ha_enabled == true) {
+ $nameservice = $hdp::params::dfs_ha_nameservices
+ $namenode_id = hdp_hadoop_get_namenode_id("dfs.namenode.rpc-address.${nameservice}", "hdfs-site")
+ }
+
}