You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2013/06/03 18:06:12 UTC
svn commit: r1489048 - in /incubator/ambari/branches/branch-1.4.0: ./
ambari-agent/src/main/puppet/modules/hdp-yarn/files/
ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/
ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/historyserver/ a...
Author: ncole
Date: Mon Jun 3 16:06:11 2013
New Revision: 1489048
URL: http://svn.apache.org/r1489048
Log:
AMBARI-2250. Hadoop 2: Create a service check for YARN
Added:
incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/yarn/
incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/yarn/service_check.pp
Modified:
incubator/ambari/branches/branch-1.4.0/CHANGES.txt
incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/files/validateYarnComponentStatus.py
incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/historyserver/service_check.pp
incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager/service_check.pp
incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp
incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
incubator/ambari/branches/branch-1.4.0/ambari-server/src/main/java/org/apache/ambari/server/Role.java
Modified: incubator/ambari/branches/branch-1.4.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-1.4.0/CHANGES.txt?rev=1489048&r1=1489047&r2=1489048&view=diff
==============================================================================
--- incubator/ambari/branches/branch-1.4.0/CHANGES.txt (original)
+++ incubator/ambari/branches/branch-1.4.0/CHANGES.txt Mon Jun 3 16:06:11 2013
@@ -17,6 +17,8 @@ Branch 1.4.0:
IMPROVEMENTS
+ AMBARI-2250. Hadoop 2: Create a service check for YARN. (Oleksandr Diachenko via ncole)
+
AMBARI-2244. Hadoop 2: Added YARN metrics. (ncole)
AMBARI-2237. Hadoop 2: Add user configs for MR2/YARN. (yusaku)
Modified: incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/files/validateYarnComponentStatus.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/files/validateYarnComponentStatus.py?rev=1489048&r1=1489047&r2=1489048&view=diff
==============================================================================
--- incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/files/validateYarnComponentStatus.py (original)
+++ incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/files/validateYarnComponentStatus.py Mon Jun 3 16:06:11 2013
@@ -23,11 +23,14 @@ import urllib2, urllib
import json
RESOURCEMANAGER = 'rm'
-HISTORYSERVER ='hs'
+NODEMANAGER = 'nm'
+HISTORYSERVER = 'hs'
STARTED_STATE = 'STARTED'
+RUNNING_STATE = 'RUNNING'
-def validate(component, path, address):
+#Return reponse for given path and address
+def getResponse(path, address):
try:
url = 'http://' + address + path
@@ -36,23 +39,42 @@ def validate(component, path, address):
request = urllib2.Request(url)
handler = urllib2.urlopen(request)
response = json.loads(handler.read())
- is_valid = validateResponse(component, response)
- if is_valid:
- exit(0)
- else:
+ if response == None:
+ print 'There is no response for url: ' + str(url)
exit(1)
+ return response
except Exception as e:
- print 'Error checking status of component', e
+ print 'Error getting response for url:' + str(url), e
exit(1)
+#Verify that REST api is available for given component
+def validateAvailability(component, path, address):
-def validateResponse(component, response):
+ try:
+ response = getResponse(path, address)
+ is_valid = validateAvailabilityResponse(component, response)
+ if not is_valid:
+ exit(1)
+ except Exception as e:
+ print 'Error checking availability status of component', e
+ exit(1)
+
+#Validate component-specific response
+def validateAvailabilityResponse(component, response):
try:
if component == RESOURCEMANAGER:
rm_state = response['clusterInfo']['state']
if rm_state == STARTED_STATE:
return True
else:
+ print 'Resourcemanager is not started'
+ return False
+
+ elif component == NODEMANAGER:
+ node_healthy = bool(response['nodeInfo']['nodeHealthy'])
+ if node_healthy:
+ return True
+ else:
return False
elif component == HISTORYSERVER:
hs_start_time = response['historyInfo']['startedOn']
@@ -63,7 +85,44 @@ def validateResponse(component, response
else:
return False
except Exception as e:
- print 'Error validation of response', e
+ print 'Error validation of availability response for ' + str(component), e
+ return False
+
+#Verify that component has required resources to work
+def validateAbility(component, path, address):
+
+ try:
+ response = getResponse(path, address)
+ is_valid = validateAbilityResponse(component, response)
+ if not is_valid:
+ exit(1)
+ except Exception as e:
+ print 'Error checking ability of component', e
+ exit(1)
+
+#Validate component-specific response that it has required resources to work
+def validateAbilityResponse(component, response):
+ try:
+ if component == RESOURCEMANAGER:
+ nodes = []
+ if response.has_key('nodes') and not response['nodes'] == None and response['nodes'].has_key('node'):
+ nodes = response['nodes']['node']
+ connected_nodes_count = len(nodes)
+ if connected_nodes_count == 0:
+ print 'There is no connected nodemanagers to resourcemanager'
+ return False
+ active_nodes = filter(lambda x: x['state'] == RUNNING_STATE, nodes)
+ active_nodes_count = len(active_nodes)
+
+ if connected_nodes_count == 0:
+ print 'There is no connected active nodemanagers to resourcemanager'
+ return False
+ else:
+ return True
+ else:
+ return False
+ except Exception as e:
+ print 'Error validation of ability response', e
return False
#
@@ -73,21 +132,26 @@ def main():
parser = optparse.OptionParser(usage="usage: %prog [options] component ")
parser.add_option("-p", "--port", dest="address", help="Host:Port for REST API of a desired component")
-
(options, args) = parser.parse_args()
component = args[0]
-
+
address = options.address
-
+
if component == RESOURCEMANAGER:
path = '/ws/v1/cluster/info'
+ elif component == NODEMANAGER:
+ path = '/ws/v1/node/info'
elif component == HISTORYSERVER:
path = '/ws/v1/history/info'
else:
parser.error("Invalid component")
- validate(component, path, address)
+ validateAvailability(component, path, address)
+
+ if component == RESOURCEMANAGER:
+ path = '/ws/v1/cluster/nodes'
+ validateAbility(component, path, address)
if __name__ == "__main__":
main()
Modified: incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/historyserver/service_check.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/historyserver/service_check.pp?rev=1489048&r1=1489047&r2=1489048&view=diff
==============================================================================
--- incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/historyserver/service_check.pp (original)
+++ incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/historyserver/service_check.pp Mon Jun 3 16:06:11 2013
@@ -20,5 +20,5 @@
#
class hdp-yarn::historyserver::service_check() inherits hdp-yarn::params
{
- class { 'hdp-yarn::smoketest': component_name => 'historyserver'}
+ hdp-yarn::smoketest{'hdp-yarn::smoketest:rm': component_name => 'historyserver'}
}
\ No newline at end of file
Modified: incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp?rev=1489048&r1=1489047&r2=1489048&view=diff
==============================================================================
--- incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp (original)
+++ incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp Mon Jun 3 16:06:11 2013
@@ -23,6 +23,7 @@ class hdp-yarn::params(
{
$conf_dir = $hdp::params::yarn_conf_dir
+ $stack_version = $hdp::params::stack_version
## yarn-env
$hadoop_libexec_dir = hdp_default("yarn/yarn-env/hadoop_libexec_dir","/usr/lib/hadoop/libexec")
@@ -36,10 +37,19 @@ class hdp-yarn::params(
$yarn_pid_dir_prefix = hdp_default("hadoop/yarn-env/yarn_pid_dir_prefix","/var/run/hadoop-yarn")
## yarn-site
- $rm_webui_address = hdp_default("yarn-site/yarn.resourcemanager.webapp.address", "localhost:8088")
- $nm_webui_address = hdp_default("yarn-site/yarn.nodemanager.webapp.address", "localhost:8042")
- $hs_webui_address = hdp_default("mapred-site/mapreduce.jobhistory.webapp.address", "localhost:19888")
+ $rm_webui_address = hdp_default("yarn-site/yarn.resourcemanager.webapp.address", "0.0.0.0:8088")
+ $nm_webui_address = hdp_default("yarn-site/yarn.nodemanager.webapp.address", "0.0.0.0:8042")
+ $hs_webui_address = hdp_default("mapred-site/mapreduce.jobhistory.webapp.address", "0.0.0.0:19888")
$nm_local_dirs = hdp_default("yarn-site/yarn.nodemanager.local-dirs", "$hadoop_tmp_dir/nm-local-dir")
$nm_log_dirs = hdp_default("yarn-site/yarn.nodemanager.log-dirs", "/var/log/hadoop-yarn/yarn")
+
+ ##smoke test configs
+ $distrAppJarNameToStack = {'2.0.1' => "hadoop-yarn-applications-distributedshell-2.0.3.22-alpha.jar"}
+
+ $distrAppJarName = $distrAppJarNameToStack[$stack_version]
+
+ if hdp_is_empty($distrAppJarName) {
+ hdp_fail("No yarn-applications-distributedshell jar for stack: $stack_version")
+ }
}
Modified: incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager/service_check.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager/service_check.pp?rev=1489048&r1=1489047&r2=1489048&view=diff
==============================================================================
--- incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager/service_check.pp (original)
+++ incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/resourcemanager/service_check.pp Mon Jun 3 16:06:11 2013
@@ -20,5 +20,5 @@
#
class hdp-yarn::resourcemanager::service_check() inherits hdp-yarn::params
{
- class { 'hdp-yarn::smoketest': component_name => 'resourcemanager'}
+ hdp-yarn::smoketest{'hdp-yarn::smoketest:rm': component_name => 'resourcemanager'}
}
\ No newline at end of file
Modified: incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp?rev=1489048&r1=1489047&r2=1489048&view=diff
==============================================================================
--- incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp (original)
+++ incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/smoketest.pp Mon Jun 3 16:06:11 2013
@@ -18,7 +18,7 @@
# under the License.
#
#
-class hdp-yarn::smoketest(
+define hdp-yarn::smoketest(
$component_name = undef
)
{
@@ -29,15 +29,18 @@ class hdp-yarn::smoketest(
if ($component_name == 'resourcemanager') {
$component_type = 'rm'
$component_address = $rm_webui_address
+ } elsif ($component_name == 'nodemanager') {
+ $component_type = 'nm'
+ $component_address = $nm_webui_address
} elsif ($component_name == 'historyserver') {
- $component_type = 'hs'
+ $component_type = 'hs'
$component_address = $hs_webui_address
} else {
hdp_fail("Unsupported component name: $component_name")
}
$smoke_test_user = $hdp::params::smokeuser
-
+
$validateStatusFileName = "validateYarnComponentStatus.py"
$validateStatusFilePath = "/tmp/$validateStatusFileName"
@@ -55,6 +58,6 @@ class hdp-yarn::smoketest(
try_sleep => 5,
path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
logoutput => "true"
- }
- File[$validateStatusFilePath] -> Exec[$validateStatusFilePath]
+}
+ anchor{"hdp-yarn::smoketest::begin":} -> File[$validateStatusFilePath] -> Exec[$validateStatusFilePath] -> anchor{"hdp-yarn::smoketest::end":}
}
Added: incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/yarn/service_check.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/yarn/service_check.pp?rev=1489048&view=auto
==============================================================================
--- incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/yarn/service_check.pp (added)
+++ incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/yarn/service_check.pp Mon Jun 3 16:06:11 2013
@@ -0,0 +1,37 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp-yarn::yarn::service_check() inherits hdp-yarn::params
+{
+
+ $jar_path = "$hadoop_yarn_home/$distrAppJarName"
+ $run_dist_shell_app_cmd = "jar $jar_path -appname yarnservicecheck -master_memory 512 -container_memory 128 -num_containers 2 -shell_command \"ls\" -jar $jar_path"
+
+ ## Check availability of REST api
+ hdp-yarn::smoketest{'hdp-yarn::smoketest:rm': component_name => 'resourcemanager'}
+
+ ## Run distributed shell application check
+ hdp-hadoop::exec-hadoop { 'hdp-yarn::yarn::service_check':
+ command => $run_dist_shell_app_cmd,
+ user => $smoke_test_user
+ }
+
+ anchor{"hdp-yarn::yarn::service_check::begin":} -> Hdp-yarn::Smoketest['hdp-yarn::smoketest:rm'] -> Hdp-hadoop::Exec-hadoop['hdp-yarn::yarn::service_check'] -> anchor{"hdp-yarn::yarn::service_check::end":}
+}
\ No newline at end of file
Modified: incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py?rev=1489048&r1=1489047&r2=1489048&view=diff
==============================================================================
--- incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py (original)
+++ incubator/ambari/branches/branch-1.4.0/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py Mon Jun 3 16:06:11 2013
@@ -127,6 +127,7 @@ rolesToClass = {
'HUE_SERVICE_CHECK': 'hdp-hue::service_check',
'RESOURCEMANAGER_SERVICE_CHECK': 'hdp-yarn::resourcemanager::service_check',
'HISTORYSERVER_SERVICE_CHECK': 'hdp-yarn::historyserver::service_check',
+ 'YARN_SERVICE_CHECK': 'hdp-yarn::yarn::service_check',
'TEZ_CLIENT': 'hdp-tez::tez_client'
}
Modified: incubator/ambari/branches/branch-1.4.0/ambari-server/src/main/java/org/apache/ambari/server/Role.java
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-1.4.0/ambari-server/src/main/java/org/apache/ambari/server/Role.java?rev=1489048&r1=1489047&r2=1489048&view=diff
==============================================================================
--- incubator/ambari/branches/branch-1.4.0/ambari-server/src/main/java/org/apache/ambari/server/Role.java (original)
+++ incubator/ambari/branches/branch-1.4.0/ambari-server/src/main/java/org/apache/ambari/server/Role.java Mon Jun 3 16:06:11 2013
@@ -75,6 +75,7 @@ public enum Role {
RESOURCEMANAGER_SERVICE_CHECK,
HISTORYSERVER_SERVICE_CHECK,
NODEMANAGER,
+ YARN_SERVICE_CHECK,
YARN_CLIENT,
HISTORYSERVER,
TEZ_CLIENT