You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2013/08/14 17:46:06 UTC
git commit: AMBARI-2897 Add component Zkfc to HDFS
Updated Branches:
refs/heads/trunk c244d4fc4 -> 8871874f0
AMBARI-2897 Add component Zkfc to HDFS
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/8871874f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/8871874f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/8871874f
Branch: refs/heads/trunk
Commit: 8871874f087b96cad2dcd6003779969ded2603a7
Parents: c244d4f
Author: Lisnichenko Dmitro <dl...@cybervisiontech.com>
Authored: Fri Aug 9 21:57:08 2013 +0300
Committer: Lisnichenko Dmitro <dl...@cybervisiontech.com>
Committed: Wed Aug 14 18:43:14 2013 +0300
----------------------------------------------------------------------
.../hdp-hadoop/manifests/hdfs/service_check.pp | 92 ++++++++++++++------
.../modules/hdp-hadoop/manifests/service.pp | 2 +
.../puppet/modules/hdp-hadoop/manifests/zkfc.pp | 51 +++++++++++
.../puppet/modules/hdp/manifests/configfile.pp | 2 +
.../main/puppet/modules/hdp/manifests/params.pp | 3 +
.../main/python/ambari_agent/AmbariConfig.py | 4 +-
.../src/main/python/ambari_agent/LiveStatus.py | 2 +
.../java/org/apache/ambari/server/Role.java | 3 +-
.../apache/ambari/server/utils/StageUtils.java | 1 +
.../stacks/HDP/2.0.5/services/HDFS/metainfo.xml | 5 ++
.../HDPLocal/2.0.5/services/HDFS/metainfo.xml | 5 ++
11 files changed, 140 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp
index 7b49ed5..37d0eea 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp
@@ -27,7 +27,7 @@ class hdp-hadoop::hdfs::service_check()
$safemode_command = "dfsadmin -safemode get | grep OFF"
$create_dir_cmd = "fs -mkdir ${dir} ; hadoop fs -chmod -R 777 ${dir}"
- $test_dir_exists = "hadoop fs -test -e ${dir}" #TODO: may fix up fact that test needs explicit hadoop while omamnd does not
+ $test_dir_exists = "hadoop fs -test -e ${dir}" #TODO: may fix up fact that test needs explicit hadoop while command does not
$cleanup_cmd = "fs -rm ${tmp_file}"
#cleanup put below to handle retries; if retrying there wil be a stale file that needs cleanup; exit code is fn of second command
$create_file_cmd = "${cleanup_cmd}; hadoop fs -put /etc/passwd ${tmp_file}" #TODO: inconsistent that second comamnd needs hadoop
@@ -63,7 +63,6 @@ class hdp-hadoop::hdfs::service_check()
}
-
#TODO: put in after testing
# hdp-hadoop::exec-hadoop { 'hdfs::service_check::cleanup':
# command => $cleanup_cmd,
@@ -71,44 +70,58 @@ class hdp-hadoop::hdfs::service_check()
# require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test'],
# before => Anchor['hdp-hadoop::hdfs::service_check::end']
#}
-
- if hdp_is_empty($hdp::params::journalnode_hosts) {
- ##No journalnode hosts, just run hdfs test
- hdp-hadoop::exec-hadoop { 'hdfs::service_check::test':
- command => $test_cmd,
- refreshonly => true,
- user => $hdp::params::smokeuser,
- require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'],
- before => Anchor['hdp-hadoop::hdfs::service_check::end']
- }
+
+ hdp-hadoop::exec-hadoop { 'hdfs::service_check::test':
+ command => $test_cmd,
+ refreshonly => true,
+ user => $hdp::params::smokeuser,
+ require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'],
+ before => Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:begin']
}
- else {
- ## Cluster has journalnode hosts, run hdfs test and test of journalnodes
- hdp-hadoop::exec-hadoop { 'hdfs::service_check::test':
- command => $test_cmd,
- refreshonly => true,
- user => $hdp::params::smokeuser,
- require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'],
- before => Class['hdp-hadoop::journalnode::service_check']
- }
-
+
+ anchor { 'hdp-hadoop::hdfs::service_check::journalnode_check:begin':}
+
+ if hdp_is_empty($hdp::params::journalnode_hosts) {
+ ##No journalnode hosts
+ Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:begin'] ->
+ Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:end']
+
+ } else {
+ ## Cluster has journalnode hosts, run test of journalnodes
$journalnode_hosts_comma_sep = hdp_comma_list_from_array($hdp::params::journalnode_hosts)
-
class { 'hdp-hadoop::journalnode::service_check':
journalnode_hosts => $journalnode_hosts_comma_sep,
- require => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test'],
- before => Anchor['hdp-hadoop::hdfs::service_check::end']
+ require => Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:begin'],
+ before => Anchor['hdp-hadoop::hdfs::service_check::journalnode_check:end']
+ }
+ }
+
+ anchor { 'hdp-hadoop::hdfs::service_check::journalnode_check:end':} ->
+ anchor { 'hdp-hadoop::hdfs::service_check::zkfc_check:begin':}
+
+ if hdp_is_empty($hdp::params::zkfc_hosts) {
+ ## No zkfc hosts
+ Anchor['hdp-hadoop::hdfs::service_check::zkfc_check:begin'] ->
+ Anchor['hdp-hadoop::hdfs::service_check::zkfc_check:end']
+ } else {
+ ## Cluster has zkfc hosts, run test of local zkfc daemon if current host
+ ## is namenode. If namenode has not ZKFC installed, it is also considered
+ ## as a misconfiguration.
+ if ($hdp::params::is_namenode_master) {
+ class { 'hdp-hadoop::zkfc::service_check':
+ require => Anchor['hdp-hadoop::hdfs::service_check::zkfc_check:begin'],
+ before => Anchor['hdp-hadoop::hdfs::service_check::zkfc_check:end']
+ }
}
}
-
- anchor{ 'hdp-hadoop::hdfs::service_check::end':}
+ anchor { 'hdp-hadoop::hdfs::service_check::zkfc_check:end':} ->
+ anchor{ 'hdp-hadoop::hdfs::service_check::end':}
}
class hdp-hadoop::journalnode::service_check($journalnode_hosts)
{
-
$journalnode_port = $hdp::params::journalnode_port
$smoke_test_user = $hdp::params::smokeuser
@@ -132,3 +145,26 @@ class hdp-hadoop::journalnode::service_check($journalnode_hosts)
}
anchor{"hdp-hadoop::smoketest::begin":} -> File[$checkWebUIFilePath] -> Exec[$checkWebUIFilePath] -> anchor{"hdp-hadoop::smoketest::end":}
}
+
+class hdp-hadoop::zkfc::service_check() inherits hdp-hadoop::params
+{
+ $hdfs_user = $hdp::params::hdfs_user
+ $pid_dir = "${hdp-hadoop::params::hadoop_pid_dir_prefix}/${hdfs_user}"
+ $pid_file = "${pid_dir}/hadoop-${hdfs_user}-zkfc.pid"
+
+ # Here we check if pid file exists and if yes, then we run 'ps pid' command
+ # that returns 1 if process is not running
+ $check_zkfc_process_cmd = "ls ${pid_file} >/dev/null 2>&1 && ps `cat ${pid_file}` >/dev/null 2>&1"
+
+ exec { $check_zkfc_process_cmd:
+ command => $check_zkfc_process_cmd,
+ tries => 3,
+ try_sleep => 5,
+ path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ logoutput => "true"
+ }
+
+ anchor{"hdp-hadoop::zkfc::service_check::begin":} -> Exec[$check_zkfc_process_cmd] ->
+ anchor{"hdp-hadoop::zkfc::service_check::end":}
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp
index 321e21a..0af21a9 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/service.pp
@@ -57,6 +57,8 @@ define hdp-hadoop::service(
} else {
$daemon_cmd = "su - ${user} -c '${cmd} start ${name}'"
}
+ # Here we check if pid file exists and if yes, then we run 'ps pid' command
+ # that returns 1 if process is not running
$service_is_up = "ls ${pid_file} >/dev/null 2>&1 && ps `cat ${pid_file}` >/dev/null 2>&1"
} elsif ($ensure == 'stopped') {
if ($run_as_root == true) {
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/zkfc.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/zkfc.pp b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/zkfc.pp
new file mode 100644
index 0000000..3ec0055
--- /dev/null
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/zkfc.pp
@@ -0,0 +1,51 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp-hadoop::zkfc(
+ $service_state = $hdp::params::cluster_service_state,
+ $opts = {}
+) inherits hdp-hadoop::params
+{
+
+ Hdp-hadoop::Common<||>{service_states +> $service_state}
+ Hdp-hadoop::Package<||>{include_64_bit => true}
+ Hdp-hadoop::Configfile<||>{sizes +> 64}
+
+ $hdfs_user = $hdp-hadoop::params::hdfs_user
+
+ if ($service_state == 'no_op') {
+ } elsif ($service_state in ['running','stopped','installed_and_configured','uninstalled']) {
+
+ #adds package, users and directories, and common hadoop configs
+ include hdp-hadoop::initialize
+
+ hdp-hadoop::service{ 'zkfc':
+ ensure => $service_state,
+ user => $hdp-hadoop::params::hdfs_user,
+ create_pid_dir => true,
+ create_log_dir => true
+ }
+
+ #top level does not need anchors
+ Anchor['hdp-hadoop::begin'] -> Hdp-hadoop::Service['zkfc'] -> Anchor['hdp-hadoop::end']
+ } else {
+ hdp_fail("TODO not implemented yet: service_state = ${service_state}")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp
index b768b1b..b7db742 100644
--- a/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp
@@ -34,6 +34,7 @@ define hdp::configfile(
$hs_host = $hdp::params::hs_host,
$slave_hosts = $hdp::params::slave_hosts,
$journalnode_hosts = $hdp::params::journalnode_hosts,
+ $zkfc_hosts = $hdp::params::zkfc_hosts,
$mapred_tt_hosts = $hdp::params::mapred_tt_hosts,
$all_hosts = $hdp::params::all_hosts,
$hbase_rs_hosts = $hdp::params::hbase_rs_hosts,
@@ -55,6 +56,7 @@ define hdp::configfile(
$public_nm_hosts = $hdp::params::public_nm_hosts,
$public_hs_host = $hdp::params::public_hs_host,
$public_journalnode_hosts = $hdp::params::public_journalnode_hosts,
+ $public_zkfc_hosts = $hdp::params::public_zkfc_hosts,
$public_jtnode_host = $hdp::params::public_jtnode_host,
$public_hbase_master_hosts = $hdp::params::public_hbase_master_hosts,
$public_zookeeper_hosts = $hdp::params::public_zookeeper_hosts,
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
index 21e2251..1f91194 100644
--- a/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp
@@ -65,6 +65,7 @@ class hdp::params()
$jtnode_host = hdp_default("jtnode_host")
$slave_hosts = hdp_default("slave_hosts")
$journalnode_hosts = hdp_default("journalnode_hosts")
+ $zkfc_hosts = hdp_default("zkfc_hosts")
$nn_principal_str = hdp_default("hdfs-site/dfs.namenode.kerberos.principal", "nn/_HOST@EXAMPLE.COM")
if ("_HOST" in $nn_principal_str and hdp_is_empty($namenode_host) == false) {
@@ -156,6 +157,7 @@ class hdp::params()
$public_nm_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$nm_hosts)
$public_hs_host = hdp_host_attribute($hostAttributes,"publicfqdn",$hs_host)
$public_journalnode_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$journalnode_hosts)
+ $public_zkfc_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$zkfc_hosts)
$public_jtnode_host = hdp_host_attribute($hostAttributes,"publicfqdn",$jtnode_host)
$public_hbase_master_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$hbase_master_hosts)
$public_zookeeper_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$zookeeper_hosts)
@@ -172,6 +174,7 @@ class hdp::params()
$public_nm_hosts = hdp_default("nm_hosts")
$public_hs_host = hdp_default("hs_host")
$public_journalnode_hosts = hdp_default("journalnode_hosts")
+ $public_zkfc_hosts = hdp_default("zkfc_hosts")
$public_jtnode_host = hdp_default("jtnode_host")
$public_hbase_master_hosts = hdp_default("hbase_master_hosts")
$public_zookeeper_hosts = hdp_default("zookeeper_hosts")
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
index 567044d..d838d69 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
@@ -136,7 +136,7 @@ rolesToClass = {
'YARN_SERVICE_CHECK': 'hdp-yarn::yarn::service_check',
'FLUME_SERVER': 'hdp-flume',
'JOURNALNODE': 'hdp-hadoop::journalnode',
- 'JOURNALNODE_SERVICE_CHECK': 'hdp-hadoop::journalnode::service_check'
+ 'ZKFC': 'hdp-hadoop::zkfc'
}
serviceStates = {
@@ -156,6 +156,7 @@ servicesToPidNames = {
'NODEMANAGER': 'yarn-{USER}-nodemanager.pid$',
'HISTORYSERVER': 'mapred-{USER}-historyserver.pid$',
'JOURNALNODE': 'hadoop-{USER}-journalnode.pid$',
+ 'ZKFC': 'hadoop-{USER}-zkfc.pid$',
'OOZIE_SERVER': 'oozie.pid',
'ZOOKEEPER_SERVER': 'zookeeper_server.pid',
'FLUME_SERVER': 'flume-node.pid',
@@ -180,6 +181,7 @@ servicesToLinuxUser = {
'SECONDARY_NAMENODE': 'hdfs_user',
'DATANODE': 'hdfs_user',
'JOURNALNODE': 'hdfs_user',
+ 'ZKFC': 'hdfs_user',
'JOBTRACKER': 'mapred_user',
'TASKTRACKER': 'mapred_user',
'RESOURCEMANAGER': 'yarn_user',
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
index 44e0588..b41b6e3 100644
--- a/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
+++ b/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
@@ -45,6 +45,8 @@ class LiveStatus:
"componentName" : "SECONDARY_NAMENODE"},
{"serviceName" : "HDFS",
"componentName" : "JOURNALNODE"},
+ {"serviceName" : "HDFS",
+ "componentName" : "ZKFC"},
{"serviceName" : "MAPREDUCE",
"componentName" : "JOBTRACKER"},
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-server/src/main/java/org/apache/ambari/server/Role.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/Role.java b/ambari-server/src/main/java/org/apache/ambari/server/Role.java
index 18f318c..8913f22 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/Role.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/Role.java
@@ -83,5 +83,6 @@ public enum Role {
HISTORYSERVER,
TEZ_CLIENT,
FLUME_SERVER,
- JOURNALNODE
+ JOURNALNODE,
+ ZKFC
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
index 59660f9..3262d92 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
@@ -90,6 +90,7 @@ public class StageUtils {
componentToClusterInfoKeyMap.put("NODEMANAGER", "nm_hosts");
componentToClusterInfoKeyMap.put("HISTORYSERVER", "hs_host");
componentToClusterInfoKeyMap.put("JOURNALNODE", "journalnode_hosts");
+ componentToClusterInfoKeyMap.put("ZKFC", "zkfc_hosts");
componentToClusterInfoKeyMap.put("ZOOKEEPER_SERVER", "zookeeper_hosts");
componentToClusterInfoKeyMap.put("FLUME_SERVER", "flume_hosts");
componentToClusterInfoKeyMap.put("HBASE_MASTER", "hbase_master_hosts");
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml
index 95dc294..2ea1033 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HDFS/metainfo.xml
@@ -45,6 +45,11 @@
<name>JOURNALNODE</name>
<category>SLAVE</category>
</component>
+
+ <component>
+ <name>ZKFC</name>
+ <category>SLAVE</category>
+ </component>
</components>
</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8871874f/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml
index b695bf3..f6f7586 100644
--- a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/HDFS/metainfo.xml
@@ -45,6 +45,11 @@
<name>JOURNALNODE</name>
<category>SLAVE</category>
</component>
+
+ <component>
+ <name>ZKFC</name>
+ <category>SLAVE</category>
+ </component>
</components>