You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2013/10/01 19:08:42 UTC
git commit: AMBARI-3368. NameNode start hangs with HA config'd. Stub
patch. (swagle)
Updated Branches:
refs/heads/branch-1.4.1 364619f2d -> 8c1490c78
AMBARI-3368. NameNode start hangs with HA config'd. Stub patch. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/8c1490c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/8c1490c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/8c1490c7
Branch: refs/heads/branch-1.4.1
Commit: 8c1490c786e718274d0c89c2a33ddf7b33582842
Parents: 364619f
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Mon Sep 30 15:20:46 2013 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Tue Oct 1 10:08:32 2013 -0700
----------------------------------------------------------------------
.../hdp-hadoop/manifests/hdfs/directory.pp | 35 +++++++++++++++-----
.../modules/hdp-hadoop/manifests/params.pp | 4 +++
2 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8c1490c7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp
index 0cddd58..f0852ae 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/directory.pp
@@ -30,6 +30,12 @@ define hdp-hadoop::hdfs::directory(
{
$dir_exists = "hadoop fs -ls ${name} >/dev/null 2>&1"
$namenode_safe_mode_off = "hadoop dfsadmin -safemode get|grep 'Safe mode is OFF'"
+
+ # Short circuit the expensive dfs client checks if directory was already created
+ $stub_dir = $hdp-hadoop::params::namenode_dirs_created_stub_dir
+ $stub_filename = $hdp-hadoop::params::namenode_dirs_stub_filename
+ $dir_absent_in_stub = "grep -q '^${name}$' ${stub_dir}/${stub_filename} > /dev/null 2>&1; test $? -ne 0"
+ $record_dir_in_stub = "echo '${name}' >> ${stub_dir}/${stub_filename}"
$tries = 30
$try_sleep = 10
@@ -42,7 +48,7 @@ define hdp-hadoop::hdfs::directory(
$dfs_check_nn_status_cmd = "true"
}
- if ($service_state == 'running' and $hdp::params::dfs_ha_enabled == false) {
+ if ($service_state == 'running') {
if (hdp_get_major_stack_version($hdp::params::stack_version) >= 2) {
$mkdir_cmd = "fs -mkdir -p ${name}"
} else {
@@ -51,12 +57,21 @@ define hdp-hadoop::hdfs::directory(
hdp-hadoop::exec-hadoop { $mkdir_cmd:
command => $mkdir_cmd,
- unless => "$dfs_check_nn_status_cmd && $dir_exists && ! $namenode_safe_mode_off",
- onlyif => "$dfs_check_nn_status_cmd && ! $dir_exists",
+ unless => "$dir_absent_in_stub && $dfs_check_nn_status_cmd && $dir_exists && ! $namenode_safe_mode_off",
+ onlyif => "$dir_absent_in_stub && $dfs_check_nn_status_cmd && ! $dir_exists",
try_sleep => $try_sleep,
tries => $tries
}
+ hdp::exec { $record_dir_in_stub:
+ command => $record_dir_in_stub,
+ user => $hdp-hadoop::params::hdfs_user,
+ onlyif => $dir_absent_in_stub
+ }
+
+ Hdp-hadoop::Exec-hadoop[$mkdir_cmd] ->
+ Hdp::Exec[$record_dir_in_stub]
+
if ($owner == unset) {
$chown = ""
} else {
@@ -76,11 +91,13 @@ define hdp-hadoop::hdfs::directory(
}
hdp-hadoop::exec-hadoop {$chown_cmd :
command => $chown_cmd,
- onlyif => "$dfs_check_nn_status_cmd && $namenode_safe_mode_off && $dir_exists",
+ onlyif => "$dir_absent_in_stub && $dfs_check_nn_status_cmd && $namenode_safe_mode_off && $dir_exists",
try_sleep => $try_sleep,
tries => $tries
}
- Hdp-hadoop::Exec-hadoop[$mkdir_cmd] -> Hdp-hadoop::Exec-hadoop[$chown_cmd]
+ Hdp-hadoop::Exec-hadoop[$mkdir_cmd] ->
+ Hdp-hadoop::Exec-hadoop[$chown_cmd] ->
+ Hdp::Exec[$record_dir_in_stub]
}
if ($mode != undef) {
@@ -90,13 +107,15 @@ define hdp-hadoop::hdfs::directory(
} else {
$chmod_cmd = "fs -chmod ${mode} ${name}"
}
- hdp-hadoop::exec-hadoop {$chmod_cmd :
+ hdp-hadoop::exec-hadoop { $chmod_cmd :
command => $chmod_cmd,
- onlyif => "$dfs_check_nn_status_cmd && $namenode_safe_mode_off && $dir_exists",
+ onlyif => "$dir_absent_in_stub && $dfs_check_nn_status_cmd && $namenode_safe_mode_off && $dir_exists",
try_sleep => $try_sleep,
tries => $tries
}
- Hdp-hadoop::Exec-hadoop[$mkdir_cmd] -> Hdp-hadoop::Exec-hadoop[$chmod_cmd]
+ Hdp-hadoop::Exec-hadoop[$mkdir_cmd] ->
+ Hdp-hadoop::Exec-hadoop[$chmod_cmd] ->
+ Hdp::Exec[$record_dir_in_stub]
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8c1490c7/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
index 03eb594..5b3d75e 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp
@@ -83,6 +83,10 @@ class hdp-hadoop::params(
$mapred_pid_dir_prefix = hdp_default("mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+ # Cannot create new dir in directory.pp, reusing existing path
+ $namenode_dirs_created_stub_dir = "${hdfs_log_dir_prefix}/${hdp::params::hdfs_user}"
+ $namenode_dirs_stub_filename = "namenode_dirs_created"
+
### JSVC_HOME path is correct for AMD64 only, but can be changed through API
if ($hdp::params::hdp_os_type == "suse") {
$jsvc_path = hdp_default("jsvc_path","/usr/lib/bigtop-utils")