You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2015/01/07 03:35:54 UTC
[1/8] ambari git commit: AMBARI-8878: Common Services: Refactor
HDP-2.0.6 YARN service (Jayush Luniya)
Repository: ambari
Updated Branches:
refs/heads/trunk 7159bdc40 -> 53c391541
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
index 2735538..cddb89b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
@@ -26,20 +26,28 @@ origin_exists = os.path.exists
side_effect=lambda *args: origin_exists(args[0])
if args[0][-2:] == "j2" else True))
class TestResourceManager(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
+ STACK_VERSION = "2.0.6"
def test_configure_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="configure",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_default()
self.assertNoMoreResources()
def test_start_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="start",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_default()
@@ -58,10 +66,13 @@ class TestResourceManager(RMFTestCase):
self.assertNoMoreResources()
def test_stop_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="stop",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled('Execute', 'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config /etc/hadoop/conf stop resourcemanager',
user='yarn')
@@ -71,17 +82,23 @@ class TestResourceManager(RMFTestCase):
def test_configure_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="configure",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_secured()
def test_start_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="start",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_secured()
@@ -100,10 +117,13 @@ class TestResourceManager(RMFTestCase):
self.assertNoMoreResources()
def test_stop_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="stop",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled('Execute', 'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config /etc/hadoop/conf stop resourcemanager',
user='yarn')
@@ -113,10 +133,12 @@ class TestResourceManager(RMFTestCase):
def test_decommission_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname = "Resourcemanager",
command = "decommission",
- config_file="default.json"
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
@@ -130,10 +152,12 @@ class TestResourceManager(RMFTestCase):
self.assertNoMoreResources()
def test_decommission_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname = "Resourcemanager",
command = "decommission",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
@@ -455,10 +479,12 @@ class TestResourceManager(RMFTestCase):
get_params_mock.return_value = security_params
validate_security_config_mock.return_value = result_issues
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="security_status",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
build_exp_mock.assert_called_with('yarn-site', props_value_check, props_empty_check, props_read_check)
@@ -477,10 +503,12 @@ class TestResourceManager(RMFTestCase):
cached_kinit_executor_mock.side_effect = Exception("Invalid command")
try:
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="security_status",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
except:
self.assertTrue(True)
@@ -492,10 +520,12 @@ class TestResourceManager(RMFTestCase):
put_structured_out_mock.reset_mock()
get_params_mock.return_value = empty_security_params
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="security_status",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."})
@@ -508,17 +538,21 @@ class TestResourceManager(RMFTestCase):
validate_security_config_mock.return_value = result_issues_with_params
get_params_mock.return_value = security_params
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="security_status",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
# Testing with security_enable = false
- self.executeScript("2.0.6/services/YARN/package/scripts/resourcemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
command="security_status",
- config_file="default.json"
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
index d3b7bdb..a87cfef 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
@@ -27,12 +27,16 @@ origin_exists = os.path.exists
side_effect=lambda *args: origin_exists(args[0])
if args[0][-2:] == "j2" else True))
class TestYarnClient(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
+ STACK_VERSION = "2.0.6"
def test_configure_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/yarn_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/yarn_client.py",
classname = "YarnClient",
command = "configure",
- config_file="default.json"
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
@@ -163,10 +167,12 @@ class TestYarnClient(RMFTestCase):
def test_configure_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/yarn_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/yarn_client.py",
classname = "YarnClient",
command = "configure",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
@@ -311,11 +317,13 @@ class TestYarnClient(RMFTestCase):
self.assertNoMoreResources()
def test_restart_client(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/yarn_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/yarn_client.py",
classname = "YarnClient",
command = "restart",
config_file="default.json",
- config_overrides = { 'roleParams' : { "component_category": "CLIENT" } }
+ config_overrides = { 'roleParams' : { "component_category": "CLIENT" } },
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
@@ -446,10 +454,13 @@ class TestYarnClient(RMFTestCase):
def test_upgrade(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/yarn_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/yarn_client.py",
classname = "YarnClient",
command = "restart",
- config_file="client-upgrade.json")
+ config_file="client-upgrade.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled("Execute", "hdp-select set hadoop-client 2.2.1.0-2067")
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py
index 4e4f88e..0daa824 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py
@@ -23,13 +23,17 @@ from stacks.utils.RMFTestCase import *
@patch("sys.executable", new = '/usr/bin/python2.6')
class TestServiceCheck(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
+ STACK_VERSION = "2.0.6"
def test_service_check_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/service_check.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
classname="ServiceCheck",
command="service_check",
- config_file="default.json"
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('File', '/tmp/validateYarnComponentStatus.py',
content = StaticFile('validateYarnComponentStatus.py'),
@@ -49,10 +53,12 @@ class TestServiceCheck(RMFTestCase):
self.assertNoMoreResources()
def test_service_check_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/service_check.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
classname="ServiceCheck",
command="service_check",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('File', '/tmp/validateYarnComponentStatus.py',
content = StaticFile('validateYarnComponentStatus.py'),
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
index 4da558a..79e6fe7 100644
--- a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
+++ b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
@@ -28,22 +28,29 @@ origin_exists = os.path.exists
side_effect=lambda *args: origin_exists(args[0])
if args[0][-2:] == "j2" else True))
class TestAppTimelineServer(RMFTestCase):
-
+ COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
+ STACK_VERSION = "2.0.6"
def test_configure_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/application_timeline_server.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/application_timeline_server.py",
classname="ApplicationTimelineServer",
command="configure",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_default()
self.assertNoMoreResources()
def test_start_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/application_timeline_server.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/application_timeline_server.py",
classname="ApplicationTimelineServer",
command="start",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_default()
@@ -62,10 +69,13 @@ class TestAppTimelineServer(RMFTestCase):
self.assertNoMoreResources()
def test_stop_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/application_timeline_server.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/application_timeline_server.py",
classname="ApplicationTimelineServer",
command="stop",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled('Execute', 'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config /etc/hadoop/conf stop timelineserver',
user='yarn')
@@ -208,10 +218,13 @@ class TestAppTimelineServer(RMFTestCase):
def test_status(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/application_timeline_server.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/application_timeline_server.py",
classname="ApplicationTimelineServer",
command="status",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled('Execute', 'mv /var/run/hadoop-yarn/yarn/yarn-yarn-historyserver.pid /var/run/hadoop-yarn/yarn/yarn-yarn-timelineserver.pid',
only_if = 'test -e /var/run/hadoop-yarn/yarn/yarn-yarn-historyserver.pid',
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-web/app/data/HDP2/site_properties.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js
index 1f80d58..acb0bed 100644
--- a/ambari-web/app/data/HDP2/site_properties.js
+++ b/ambari-web/app/data/HDP2/site_properties.js
@@ -56,6 +56,7 @@ module.exports =
"displayType": "user",
"isOverridable": false,
"isVisible": App.get('isHadoopWindowsStack'),
+ "serviceName": "MISC",
"filename": "cluster-env.xml",
"category": "Users and Groups",
"belongsToService": ["HDFS"],
@@ -70,6 +71,7 @@ module.exports =
"displayType": "password",
"isOverridable": false,
"isVisible": App.get('isHadoopWindowsStack'),
+ "serviceName": "MISC",
"filename": "cluster-env.xml",
"category": "Users and Groups",
"belongsToService": ["HDFS"],
[2/8] ambari git commit: AMBARI-8878: Common Services: Refactor
HDP-2.0.6 YARN service (Jayush Luniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/yarn.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/yarn.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/yarn.conf.j2
deleted file mode 100644
index 3bd7a45..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/yarn.conf.j2
+++ /dev/null
@@ -1,35 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{{yarn_user}} - nofile 32768
-{{yarn_user}} - nproc 65536
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/capacity-scheduler.xml
deleted file mode 100644
index aeee46a..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/capacity-scheduler.xml
+++ /dev/null
@@ -1,131 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration supports_final="false">
-
- <property>
- <name>yarn.scheduler.capacity.maximum-applications</name>
- <value>10000</value>
- <description>
- Maximum number of applications that can be pending and running.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
- <value>0.2</value>
- <description>
- Maximum percent of resources in the cluster which can be used to run
- application masters i.e. controls number of concurrent running
- applications.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.queues</name>
- <value>default</value>
- <description>
- The queues at the this level (root is the root queue).
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.capacity</name>
- <value>100</value>
- <description>
- The total capacity as a percentage out of 100 for this queue.
- If it has child queues then this includes their capacity as well.
- The child queues capacity should add up to their parent queue's capacity
- or less.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.capacity</name>
- <value>100</value>
- <description>Default queue target capacity.</description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
- <value>1</value>
- <description>
- Default queue user limit a percentage from 0.0 to 1.0.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
- <value>100</value>
- <description>
- The maximum capacity of the default queue.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.state</name>
- <value>RUNNING</value>
- <description>
- The state of the default queue. State can be one of RUNNING or STOPPED.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
- <value>*</value>
- <description>
- The ACL of who can submit jobs to the default queue.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
- <value>*</value>
- <description>
- The ACL of who can administer jobs on the default queue.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.acl_administer_queue</name>
- <value>*</value>
- <description>
- The ACL for who can administer this queue i.e. change sub-queue
- allocations.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.node-locality-delay</name>
- <value>40</value>
- <description>
- Number of missed scheduling opportunities after which the CapacityScheduler
- attempts to schedule rack-local containers.
- Typically this should be set to number of nodes in the cluster, By default is setting
- approximately number of nodes in one rack which is 40.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.default.minimum-user-limit-percent</name>
- <value>100</value>
- <description>
- Default minimum queue resource limit depends on the number of users who have submitted applications.
- </description>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
index f2a5c37..dd3e3ef 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
@@ -22,158 +22,8 @@
<configuration>
<property>
- <name>yarn_log_dir_prefix</name>
- <value>/var/log/hadoop-yarn</value>
- <description>YARN Log Dir Prefix</description>
- </property>
- <property>
- <name>yarn_pid_dir_prefix</name>
- <value>/var/run/hadoop-yarn</value>
- <description>YARN PID Dir Prefix</description>
- </property>
- <property>
- <name>yarn_user</name>
- <value>yarn</value>
- <property-type>USER</property-type>
- <description>YARN User</description>
- </property>
- <property>
- <name>yarn_heapsize</name>
- <value>1024</value>
- <description>Max heapsize for all YARN components using a numerical value in the scale of MB</description>
- </property>
- <property>
- <name>resourcemanager_heapsize</name>
- <value>1024</value>
- <description>Max heapsize for ResourceManager using a numerical value in the scale of MB</description>
- </property>
- <property>
- <name>nodemanager_heapsize</name>
- <value>1024</value>
- <description>Max heapsize for NodeManager using a numerical value in the scale of MB</description>
- </property>
- <property>
<name>apptimelineserver_heapsize</name>
<value>1024</value>
<description>Max heapsize for AppTimelineServer using a numerical value in the scale of MB</description>
</property>
-
- <!-- yarn-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for yarn-env.sh file</description>
- <value>
-export HADOOP_YARN_HOME={{hadoop_yarn_home}}
-export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER
-export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
-export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
-export JAVA_HOME={{java64_home}}
-
-# User for YARN daemons
-export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
-
-# resolve links - $0 may be a softlink
-export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
-
-# some Java parameters
-# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
-if [ "$JAVA_HOME" != "" ]; then
- #echo "run java in $JAVA_HOME"
- JAVA_HOME=$JAVA_HOME
-fi
-
-if [ "$JAVA_HOME" = "" ]; then
- echo "Error: JAVA_HOME is not set."
- exit 1
-fi
-
-JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m
-
-# For setting YARN specific HEAP sizes please use this
-# Parameter and set appropriately
-YARN_HEAPSIZE={{yarn_heapsize}}
-
-# check envvars which might override default args
-if [ "$YARN_HEAPSIZE" != "" ]; then
- JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
-fi
-
-# Resource Manager specific parameters
-
-# Specify the max Heapsize for the ResourceManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_RESOURCEMANAGER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}
-
-# Specify the JVM options to be used when starting the ResourceManager.
-# These options will be appended to the options specified as YARN_OPTS
-# and therefore may override any similar flags set in YARN_OPTS
-#export YARN_RESOURCEMANAGER_OPTS=
-
-# Node Manager specific parameters
-
-# Specify the max Heapsize for the NodeManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_NODEMANAGER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}
-
-# Specify the max Heapsize for the HistoryManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1024.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_HISTORYSERVER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-export YARN_HISTORYSERVER_HEAPSIZE={{apptimelineserver_heapsize}}
-
-# Specify the JVM options to be used when starting the NodeManager.
-# These options will be appended to the options specified as YARN_OPTS
-# and therefore may override any similar flags set in YARN_OPTS
-#export YARN_NODEMANAGER_OPTS=
-
-# so that filenames w/ spaces are handled correctly in loops below
-IFS=
-
-
-# default log directory and file
-if [ "$YARN_LOG_DIR" = "" ]; then
- YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-fi
-if [ "$YARN_LOGFILE" = "" ]; then
- YARN_LOGFILE='yarn.log'
-fi
-
-# default policy file for service-level authorization
-if [ "$YARN_POLICYFILE" = "" ]; then
- YARN_POLICYFILE="hadoop-policy.xml"
-fi
-
-# restore ordinary behaviour
-unset IFS
-
-
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
-YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
-YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
- YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
-fi
-YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
- </value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-site.xml
index e9f5c3c..23f9c30 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-site.xml
@@ -20,325 +20,6 @@
<!-- Put site-specific property overrides in this file. -->
<configuration supports_final="true" xmlns:xi="http://www.w3.org/2001/XInclude">
-
- <!-- ResourceManager -->
-
- <property>
- <name>yarn.resourcemanager.hostname</name>
- <value>localhost</value>
- <description>The hostname of the RM.</description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>localhost:8025</value>
- <description> The address of ResourceManager. </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.address</name>
- <value>localhost:8030</value>
- <description>The address of the scheduler interface.</description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>localhost:8050</value>
- <description>
- The address of the applications manager interface in the
- RM.
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.admin.address</name>
- <value>localhost:8141</value>
- <description>The address of the RM admin interface.</description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
- <description>The class to use as the resource scheduler.</description>
- </property>
-
- <property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>512</value>
- <description>
- The minimum allocation for every container request at the RM,
- in MBs. Memory requests lower than this won't take effect,
- and the specified value will get allocated at minimum.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>2048</value>
- <description>
- The maximum allocation for every container request at the RM,
- in MBs. Memory requests higher than this won't take effect,
- and will get capped to this value.
- </description>
- </property>
-
- <property>
- <name>yarn.acl.enable</name>
- <value>false</value>
- <description> Are acls enabled. </description>
- </property>
-
- <property>
- <name>yarn.admin.acl</name>
- <value></value>
- <description> ACL of who can be admin of the YARN cluster. </description>
- </property>
-
- <!-- NodeManager -->
-
- <property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:45454</value>
- <description>The address of the container manager in the NM.</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>5120</value>
- <description>Amount of physical memory, in MB, that can be allocated
- for containers.</description>
- </property>
-
- <property>
- <name>yarn.application.classpath</name>
- <value>/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*</value>
- <description>Classpath for typical applications.</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>2.1</value>
- <description>Ratio between virtual memory to physical memory when
- setting memory limits for containers. Container allocations are
- expressed in terms of physical memory, and virtual memory usage
- is allowed to exceed this allocation by this ratio.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
- <description>ContainerExecutor for launching containers</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
- <description>Unix group of the NodeManager</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can
- not start with numbers</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
- <description>The auxiliary service class to use </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-dirs</name>
- <value>/hadoop/yarn/log</value>
- <description>
- Where to store container logs. An application's localized log directory
- will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
- Individual containers' log directories will be below this, in directories
- named container_{$contid}. Each container directory will contain the files
- stderr, stdin, and syslog generated by that container.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.local-dirs</name>
- <value>/hadoop/yarn/local</value>
- <description>
- List of directories to store localized files in. An
- application's localized file directory will be found in:
- ${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}.
- Individual containers' work directories, called container_${contid}, will
- be subdirectories of this.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-monitor.interval-ms</name>
- <value>3000</value>
- <description>
- The interval, in milliseconds, for which the node manager
- waits between two cycles of monitoring its containers' memory usage.
- </description>
- </property>
-
- <!--
- <property>
- <name>yarn.nodemanager.health-checker.script.path</name>
- <value>/etc/hadoop/conf/health_check_nodemanager</value>
- <description>The health check script to run.</description>
- </property>
- -->
-
- <property>
- <name>yarn.nodemanager.health-checker.interval-ms</name>
- <value>135000</value>
- <description>Frequency of running node health script.</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>60000</value>
- <description>Script time out period.</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.log.retain-second</name>
- <value>604800</value>
- <description>
- Time in seconds to retain user logs. Only applicable if
- log aggregation is disabled.
- </description>
- </property>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- <description>Whether to enable log aggregation. </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
- <description>Location to aggregate logs to. </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
- <value>logs</value>
- <description>
- The remote log dir will be created at
- {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.compression-type</name>
- <value>gz</value>
- <description>
- T-file compression types used to compress aggregated logs.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>0</value>
- <description>
- Number of seconds after an application finishes before the nodemanager's
- DeletionService will delete the application's localized file directory
- and log directory.
-
- To diagnose Yarn application problems, set this property's value large
- enough (for example, to 600 = 10 minutes) to permit examination of these
- directories. After changing the property's value, you must restart the
- nodemanager in order for it to have an effect.
-
- The roots of Yarn applications' work directories is configurable with
- the yarn.nodemanager.local-dirs property (see below), and the roots
- of the Yarn applications' log directories is configurable with the
- yarn.nodemanager.log-dirs property (see also below).
- </description>
- </property>
-
- <property>
- <name>yarn.log-aggregation.retain-seconds</name>
- <value>2592000</value>
- <description>
- How long to keep aggregation logs before deleting them. -1 disables.
- Be careful set this too small and you will spam the name node.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.admin-env</name>
- <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
- <description>
- Environment variables that should be forwarded from the NodeManager's
- environment to the container's.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
- <value>0.25</value>
- <description>
- The minimum fraction of number of disks to be healthy for the nodemanager
- to launch new containers. This correspond to both
- yarn-nodemanager.local-dirs and yarn.nodemanager.log-dirs. i.e.
- If there are less number of healthy local-dirs (or log-dirs) available,
- then new containers will not be launched on this node.
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.am.max-attempts</name>
- <value>2</value>
- <description>
- The maximum number of application attempts. It's a global
- setting for all application masters. Each application master can specify
- its individual maximum number of application attempts via the API, but the
- individual number cannot be more than the global upper bound. If it is,
- the resourcemanager will override it. The default number is set to 2, to
- allow at least one retry for AM.
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.webapp.address</name>
- <value>localhost:8088</value>
- <description>
- The address of the RM web application.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- <description>
- Whether virtual memory limits will be enforced for containers.
- </description>
- </property>
-
- <property>
- <name>yarn.log.server.url</name>
- <value>http://localhost:19888/jobhistory/logs</value>
- <description>
- URI for the HistoryServer's log resource
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.nodes.exclude-path</name>
- <value>/etc/hadoop/conf/yarn.exclude</value>
- <description>
- Names a file that contains a list of hosts that are
- not permitted to connect to the resource manager. The full pathname of the
- file must be specified. If the value is empty, no hosts are
- excluded.
- </description>
- </property>
-
<property>
<name>yarn.timeline-service.enabled</name>
<value>true</value>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/metainfo.xml
index 570723e..6f79033 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/metainfo.xml
@@ -21,7 +21,6 @@
<services>
<service>
<name>YARN</name>
- <displayName>YARN</displayName>
<version>2.4.0.2.1</version>
<components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration-mapred/mapred-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration-mapred/mapred-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration-mapred/mapred-env.xml
new file mode 100644
index 0000000..a8feead
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration-mapred/mapred-env.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>mapred_log_dir_prefix</name>
+ <value>c:\hadoop\logs\hadoop-mapreduce</value>
+ <description>Mapreduce Log Dir Prefix</description>
+ </property>
+ <property>
+ <name>mapred_pid_dir_prefix</name>
+ <value>c:\hadoop\run\hadoop-mapreduce</value>
+ <description>Mapreduce PID Dir Prefix</description>
+ </property>
+ <property>
+ <name>mapred_user</name>
+ <deleted>true</deleted>
+ </property>
+
+ <!-- mapred-env.cmd -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for mapred-env.cmd file</description>
+ <value>
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration-mapred/mapred-site.xml
index 3fcb7e7..332eef7 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration-mapred/mapred-site.xml
@@ -2,238 +2,18 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
- <!-- MR AM properties -->
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
- </property>
- <property>
- <name>yarn.app.mapreduce.am.staging-dir</name>
- <value>/user</value>
- </property>
- <property>
- <name>mapreduce.job.hdfs-servers</name>
- <value>${fs.defaultFS}</value>
- </property>
- <property>
- <name>mapreduce.map.speculative</name>
- <value>false</value>
- <description>If true, then multiple instances of some map tasks
- may be executed in parallel.</description>
- </property>
- <property>
- <name>mapreduce.reduce.speculative</name>
- <value>false</value>
- <description>If true, then multiple instances of some reduce tasks
- may be executed in parallel.</description>
- </property>
- <property>
- <name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>0.05</value>
- <description>Fraction of the number of maps in the job which should be
- complete before reduces are scheduled for the job.
- </description>
- </property>
- <property>
- <name>mapreduce.task.timeout</name>
- <value>600000</value>
- <description>The number of milliseconds before a task will be
- terminated if it neither reads an input, writes an output, nor
- updates its status string. A value of 0 disables the timeout.
- </description>
- </property>
- <property>
- <name>jetty.connector</name>
- <value>org.mortbay.jetty.nio.SelectChannelConnector</value>
- <description>No description</description>
- </property>
- <property>
- <name>mapred.child.root.logger</name>
- <value>INFO,TLA</value>
- </property>
- <property>
- <name>mapreduce.fileoutputcommitter.marksuccessfuljobs</name>
- <value>true</value>
- </property>
- <property>
- <name>mapreduce.job.acl-view-job</name>
- <value>*</value>
- </property>
- <!-- i/o properties -->
- <property>
- <name>io.sort.mb</name>
- <value>200</value>
- <description>No description</description>
- </property>
- <property>
- <name>io.sort.spill.percent</name>
- <value>0.9</value>
- <description>No description</description>
- </property>
- <property>
- <name>io.sort.factor</name>
- <value>100</value>
- <description>No description</description>
- </property>
- <!-- map tasks' properties -->
- <property>
- <name>mapreduce.map.output.compress</name>
- <value>true</value>
- <description>Should the outputs of the maps be compressed before being
- sent across the network. Uses SequenceFile compression.
- </description>
- </property>
- <property>
- <name>mapreduce.map.output.compress.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>If the map outputs are compressed, how should they be
- compressed?
- </description>
- </property>
- <!-- reduce tasks' properties -->
- <property>
- <name>mapreduce.reduce.shuffle.parallelcopies</name>
- <value>30</value>
- <description>The default number of parallel transfers run by reduce
- during the copy(shuffle) phase.
- </description>
- </property>
- <property>
- <name>mapreduce.reduce.merge.inmem.threshold</name>
- <value>1000</value>
- <description>The threshold, in terms of the number of files
- for the in-memory merge process. When we accumulate threshold number of files
- we initiate the in-memory merge and spill to disk. A value of 0 or less than
- 0 indicates we want to DON'T have any threshold and instead depend only on
- the ramfs's memory consumption to trigger the merge.
- </description>
- </property>
- <property>
- <name>mapreduce.reduce.shuffle.merge.percent</name>
- <value>0.66</value>
- <description>The usage threshold at which an in-memory merge will be
- initiated, expressed as a percentage of the total memory allocated to
- storing in-memory map outputs, as defined by
- mapreduce.reduce.shuffle.input.buffer.percent.
- </description>
- </property>
- <property>
- <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
- <value>0.70</value>
- <description>The percentage of memory to be allocated from the maximum heap
- size to storing map outputs during the shuffle.
- </description>
- </property>
- <!-- JobHistory Server -->
- <property>
- <name>mapreduce.jobhistory.intermediate-done-dir</name>
- <value>/mapred/history/done_intermediate</value>
- </property>
- <property>
- <name>mapreduce.jobhistory.done-dir</name>
- <value>/mapred/history/done</value>
- </property>
- <property>
- <name>mapreduce.jobhistory.address</name>
- <value>localhost:10020</value>
- </property>
- <property>
- <name>mapreduce.jobhistory.webapp.address</name>
- <value>localhost:19888</value>
- </property>
- <property>
- <name>mapreduce.jobhistory.webapp.https.address</name>
- <value>localhost:19888</value>
- </property>
- <property>
- <name>yarn.app.mapreduce.am.create-intermediate-jh-base-dir</name>
- <value>false</value>
- </property>
- <!-- JobHistory Security Settings -->
- <property>
- <name>mapreduce.application.classpath</name>
- <value>%HADOOP_CONF_DIR%,%HADOOP_COMMON_HOME%/share/hadoop/common/*,%HADOOP_COMMON_HOME%/share/hadoop/common/lib/*,%HADOOP_HDFS_HOME%/share/hadoop/hdfs/*,%HADOOP_HDFS_HOME%/share/hadoop/hdfs/lib/*,%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/*,%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/lib/*,%HADOOP_YARN_HOME%/share/hadoop/yarn/*,%HADOOP_YARN_HOME%/share/hadoop/yarn/lib/*</value>
- <description>CLASSPATH for MR applications. A comma-separated list
- of CLASSPATH entries</description>
- </property>
- <property>
- <name>mapreduce.shuffle.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>mapreduce.ssl.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>mapreduce.job.counters.max</name>
- <value>20000</value>
- </property>
- <property>
- <name>mapreduce.job.counters.groups.max</name>
- <value>10000</value>
- </property>
- <property>
- <name>mapreduce.job.counters.group.name.max</name>
- <value>1000</value>
- </property>
- <property>
- <name>mapreduce.job.counters.counter.name.max</name>
- <value>1000</value>
- </property>
<property>
<name>mapreduce.cluster.local.dir</name>
- <value>c:\hdpdata\hadoop\local</value>
- </property>
-
- <property>
- <name>mapred.job.tracker.history.completed.location</name>
- <value>/mapred/history/done</value>
+ <value>c:\hadoop\temp\local\hadoop-mapreduce</value>
</property>
<property>
<name>mapred.local.dir</name>
- <value>c:\hdpdata\hdfs\mapred\local</value>
- </property>
-
- <property>
- <name>mapreduce.map.java.opts</name>
- <value>-Xmx756m</value>
+ <value>c:\hadoop\temp\local\hadoop-mapreduce</value>
</property>
<property>
<name>mapred.child.tmp</name>
- <value>c:\hdp\temp\hadoop</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.java.opts</name>
- <value>-Xmx756m</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.mb</name>
- <value>200</value>
- <description>
- The total amount of buffer memory to use while sorting files, in megabytes.
- By default, gives each merge stream 1MB, which should minimize seeks.
- </description>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
- <description>The amount of memory the MR AppMaster needs.</description>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>1024</value>
- <description>Virtual memory for single Map task</description>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>1024</value>
- <description>Virtual memory for single Reduce task</description>
+ <value>c:\hadoop\temp\hadoop-mapred</value>
</property>
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/capacity-scheduler.xml
deleted file mode 100644
index c2be95b..0000000
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/capacity-scheduler.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<configuration supports_final="false" supports_adding_forbidden="true">
- <property>
- <name>yarn.scheduler.capacity.maximum-applications</name>
- <value>10000</value>
- <description>
- Maximum number of applications that can be pending and running.
- </description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
- <value>0.1</value>
- <description>
- Maximum percent of resources in the cluster which can be used to run
- application masters i.e. controls number of concurrent running
- applications.
- </description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.resource-calculator</name>
- <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
- <description>
- The ResourceCalculator implementation to be used to compare
- Resources in the scheduler.
- The default i.e. DefaultResourceCalculator only uses Memory while
- DominantResourceCalculator uses dominant-resource to compare
- multi-dimensional resources such as Memory, CPU etc.
- </description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.root.queues</name>
- <value>default,joblauncher</value>
- <description>
- The queues at the this level (root is the root queue).
- </description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.root.default.capacity</name>
- <value>95</value>
- <description>Default queue target capacity.</description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
- <value>10</value>
- <description>
- Default queue user limit a percentage from 0.0 to 1.0.
- </description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
- <value>100</value>
- <description>
- The maximum capacity of the default queue.
- </description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.root.default.state</name>
- <value>RUNNING</value>
- <description>
- The state of the default queue. State can be one of RUNNING or STOPPED.
- </description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
- <value>*</value>
- <description>
- The ACL of who can submit jobs to the default queue.
- </description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
- <value>*</value>
- <description>
- The ACL of who can administer jobs on the default queue.
- </description>
- </property>
- <property>
- <name>yarn.scheduler.capacity.node-locality-delay</name>
- <value>40</value>
- <description>
- Number of missed scheduling opportunities after which the CapacityScheduler
- attempts to schedule rack-local containers.
- Typically this should be set to number of nodes in the cluster, By default is setting
- approximately number of nodes in one rack which is 40.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.joblauncher.capacity</name>
- <value>5</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.joblauncher.user-limit-factor</name>
- <value>10</value>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.joblauncher.maximum-capacity</name>
- <value>50</value>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/yarn-env.xml
new file mode 100644
index 0000000..726704d
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/yarn-env.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>yarn_log_dir_prefix</name>
+ <value>c:\hadoop\logs\hadoop-yarn</value>
+ <description>YARN Log Dir Prefix</description>
+ </property>
+ <property>
+ <name>yarn_pid_dir_prefix</name>
+ <value>c:\hadoop\run\hadoop-yarn</value>
+ <description>YARN PID Dir Prefix</description>
+ </property>
+ <property>
+ <name>yarn_user</name>
+ <deleted>true</deleted>
+ </property>
+ <!-- yarn-env.cmd -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for yarn-env.cmd file</description>
+ <value>
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/yarn-site.xml
index b22bb5a..295ca80 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/configuration/yarn-site.xml
@@ -2,33 +2,8 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration supports_final="true" xmlns:xi="http://www.w3.org/2001/XInclude">
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>5120</value>
- <description>Amount of physical memory, in MB, that can be allocated
- for containers.</description>
- </property>
-
- <property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>512</value>
- <description>
- The minimum allocation for every container request at the RM,
- in MBs. Memory requests lower than this won't take effect,
- and the specified value will get allocated at minimum.
- </description>
- </property>
<property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>2048</value>
- <description>
- The maximum allocation for every container request at the RM,
- in MBs. Memory requests higher than this won't take effect,
- and will get capped to this value.
- </description>
- </property>
- <property>
<name>yarn.nodemanager.pmem-check-enabled</name>
<value>true</value>
</property>
@@ -38,150 +13,51 @@
</property>
<!-- NodeManager -->
<property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:45454</value>
- <description>The address of the container manager in the NM.</description>
- </property>
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>5120</value>
- <description>Amount of physical memory, in MB, that can be allocated
- for containers.</description>
- </property>
- <property>
<name>yarn.nodemanager.webapp.address</name>
<value>0.0.0.0:50060</value>
</property>
+
<property>
- <name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>2.1</value>
- <description>Ratio between virtual memory to physical memory when
- setting memory limits for containers. Container allocations are
- expressed in terms of physical memory, and virtual memory usage
- is allowed to exceed this allocation by this ratio.
- </description>
- </property>
- <property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
- <description>ContainerExecutor for launching containers</description>
- </property>
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- <description>Auxilliary services of NodeManager</description>
- </property>
- <property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
- </property>
- <property>
- <name>yarn.nodemanager.container-monitor.interval-ms</name>
- <value>3000</value>
- <description>The interval, in milliseconds, for which the node manager
- waits between two cycles of monitoring its containers' memory usage.
- </description>
- </property>
- <property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
- <description>Unix group of the NodeManager</description>
- </property>
- <property>
- <name>yarn.nodemanager.log.retain-second</name>
- <value>604800</value>
- </property>
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
- <property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
- </property>
- <property>
- <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
- <value>logs</value>
- </property>
- <property>
- <name>yarn.nodemanager.log-aggregation.compression-type</name>
- <value>gz</value>
- </property>
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>36000</value>
- </property>
- <property>
- <description>Store class name for history store, defaulting to file system store</description>
<name>yarn.timeline-service.generic-application-history.store-class</name>
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore</value>
+ <description>Store class name for history store, defaulting to file system store</description>
</property>
<!-- Use a directory that is set up on HDFS to store generic history -->
<property>
+ <name>yarn.timeline-service.generic-application-history.fs-history-store.uri</name>
+ <value>/yarn/generic-history/</value>
<description>URI pointing to the location of the FileSystem path where the history will be persisted. This must be
supplied when using org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore as
the value for yarn.timeline-service.generic-application-history.store-class
</description>
- <name>yarn.timeline-service.generic-application-history.fs-history-store.uri</name>
- <value>/yarn/generic-history/</value>
</property>
<property>
- <description>T-file compression types used to compress history data.</description>
<name>yarn.timeline-service.generic-application-history.fs-history-store.compression-type</name>
<value>none</value>
+ <description>T-file compression types used to compress history data.</description>
</property>
<property>
+ <name>yarn.timeline-service.generic-application-history.enabled</name>
+ <value>false</value>
<description>Indicate to ResourceManager as well as clients whether
history-service is enabled or not. If enabled, ResourceManager starts
recording historical data that ApplicationHistory service can consume.
Similarly, clients can redirect to the history service when applications
finish if this is enabled.
</description>
- <name>yarn.timeline-service.generic-application-history.enabled</name>
- <value>false</value>
</property>
<property>
- <description>Indicate to clients whether timeline service is enabled or not.
- If enabled, clients will put entities and events to the timeline server.
- </description>
<name>yarn.timeline-service.enabled</name>
<value>false</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
- <description>The class to use as the resource scheduler.</description>
- </property>
-
- <property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>512</value>
- <description>
- The minimum allocation for every container request at the RM,
- in MBs. Memory requests lower than this won't take effect,
- and the specified value will get allocated at minimum.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>2048</value>
- <description>
- The maximum allocation for every container request at the RM,
- in MBs. Memory requests higher than this won't take effect,
- and will get capped to this value.
+ <description>Indicate to clients whether timeline service is enabled or not.
+ If enabled, clients will put entities and events to the timeline server.
</description>
</property>
- <property>
- <name>yarn.resourcemanager.hostname</name>
- <value>localhost</value>
- </property>
<property>
<name>yarn.nodemanager.local-dirs</name>
- <value>c:\hdpdata\hadoop\local</value>
+ <value>c:\hadoop\temp\local\hadoop-yarn</value>
</property>
<property>
@@ -191,24 +67,11 @@
<property>
<name>yarn.nodemanager.log-dirs</name>
- <value>c:\hdpdata\hadoop\logs</value>
- </property>
-
- <property>
- <name>yarn.log.server.url</name>
- <value>http://localhost:19888/jobhistory/logs</value>
- <description>
- URI for the HistoryServer's log resource
- </description>
+ <value>c:\hadoop\logs\hadoop-yarn</value>
</property>
- <property>
+ <property>
<name>yarn.timeline-service.hostname</name>
<value>localhost</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.webapp.address</name>
- <value>localhost:8088</value>
- </property>
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml
index 9c41776..a83f5dd 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/YARN/metainfo.xml
@@ -21,167 +21,31 @@
<services>
<service>
<name>YARN</name>
- <displayName>YARN</displayName>
- <comment>Apache Hadoop NextGen MapReduce (YARN)</comment>
+ <extends>common-services/YARN/2.1.0.2.0</extends>
<version>2.4.0.2.1.1.0</version>
<components>
-
- <component>
- <name>RESOURCEMANAGER</name>
- <displayName>ResourceManager</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <commandScript>
- <script>scripts/resourcemanager.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>DECOMMISSION</name>
- <commandScript>
- <script>scripts/resourcemanager.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- <customCommand>
- <name>REFRESHQUEUES</name>
- <commandScript>
- <script>scripts/resourcemanager.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
- <configuration-dependencies>
- <config-type>capacity-scheduler</config-type>
- </configuration-dependencies>
- </component>
-
- <component>
- <name>NODEMANAGER</name>
- <displayName>NodeManager</displayName>
- <category>SLAVE</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/nodemanager.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
<component>
<name>YARN_CLIENT</name>
- <displayName>Yarn Client</displayName>
- <category>CLIENT</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/yarn_client.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
<configFiles>
<configFile>
- <type>xml</type>
- <fileName>yarn-site.xml</fileName>
- <dictionaryName>yarn-site</dictionaryName>
- </configFile>
- <configFile>
- <type>xml</type>
- <fileName>core-site.xml</fileName>
- <dictionaryName>core-site</dictionaryName>
- </configFile>
- <configFile>
<type>env</type>
<fileName>yarn-env.cmd</fileName>
<dictionaryName>yarn-env</dictionaryName>
</configFile>
- <configFile>
- <type>env</type>
- <fileName>log4j.properties</fileName>
- <dictionaryName>hdfs-log4j,yarn-log4j</dictionaryName>
- </configFile>
- <configFile>
- <type>xml</type>
- <fileName>capacity-scheduler.xml</fileName>
- <dictionaryName>capacity-scheduler</dictionaryName>
- </configFile>
</configFiles>
</component>
</components>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>yarn-site</config-type>
- <config-type>yarn-env</config-type>
- <config-type>core-site</config-type>
- <config-type>yarn-log4j</config-type>
- </configuration-dependencies>
</service>
<service>
<name>MAPREDUCE2</name>
- <displayName>MapReduce2</displayName>
- <comment>Apache Hadoop NextGen MapReduce (YARN)</comment>
+ <extends>common-services/MAPREDUCE2/2.1.0.2.0.6.0</extends>
<version>2.4.0.2.1.1.0</version>
<components>
<component>
- <name>HISTORYSERVER</name>
- <displayName>History Server</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <auto-deploy>
- <enabled>true</enabled>
- <co-locate>YARN/RESOURCEMANAGER</co-locate>
- </auto-deploy>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/historyserver.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
<name>MAPREDUCE2_CLIENT</name>
- <displayName>MapReduce2 Client</displayName>
- <category>CLIENT</category>
- <cardinality>0+</cardinality>
- <commandScript>
- <script>scripts/mapreduce2_client.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
<configFiles>
<configFile>
- <type>xml</type>
- <fileName>mapred-site.xml</fileName>
- <dictionaryName>mapred-site</dictionaryName>
- </configFile>
- <configFile>
- <type>xml</type>
- <fileName>core-site.xml</fileName>
- <dictionaryName>core-site</dictionaryName>
- </configFile>
- <configFile>
<type>env</type>
<fileName>mapred-env.cmd</fileName>
<dictionaryName>mapred-env</dictionaryName>
@@ -189,35 +53,7 @@
</configFiles>
</component>
</components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>hadoop-mapreduce</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandScript>
- <script>scripts/mapred_service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>YARN</service>
- </requiredServices>
-
<configuration-dir>configuration-mapred</configuration-dir>
-
- <configuration-dependencies>
- <config-type>core-site</config-type>
- <config-type>mapred-site</config-type>
- <config-type>mapred-env</config-type>
- </configuration-dependencies>
</service>
</services>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
index b8f4b83..7c6d8c6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
@@ -26,20 +26,28 @@ origin_exists = os.path.exists
side_effect=lambda *args: origin_exists(args[0])
if args[0][-2:] == "j2" else True))
class TestHistoryServer(RMFTestCase):
-
+ COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
+ STACK_VERSION = "2.0.6"
+
def test_configure_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="configure",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_default()
self.assertNoMoreResources()
def test_start_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="start",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_default()
pid_check_cmd = 'ls /var/run/hadoop-mapreduce/mapred/mapred-mapred-historyserver.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop-mapreduce/mapred/mapred-mapred-historyserver.pid` >/dev/null 2>&1'
@@ -57,10 +65,13 @@ class TestHistoryServer(RMFTestCase):
self.assertNoMoreResources()
def test_stop_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="stop",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled('Execute', 'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop-mapreduce/sbin/mr-jobhistory-daemon.sh --config /etc/hadoop/conf stop historyserver',
user='mapred')
@@ -70,18 +81,24 @@ class TestHistoryServer(RMFTestCase):
def test_configure_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="configure",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_secured()
self.assertNoMoreResources()
def test_start_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="start",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_secured()
@@ -100,10 +117,13 @@ class TestHistoryServer(RMFTestCase):
self.assertNoMoreResources()
def test_stop_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="stop",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled('Execute', 'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop-mapreduce/sbin/mr-jobhistory-daemon.sh --config /etc/hadoop/conf stop historyserver',
user='mapred')
@@ -533,10 +553,13 @@ class TestHistoryServer(RMFTestCase):
get_params_mock.return_value = security_params
validate_security_config_mock.return_value = result_issues
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="security_status",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
import status_params
@@ -564,10 +587,13 @@ class TestHistoryServer(RMFTestCase):
cached_kinit_executor_mock.side_effect = Exception("Invalid command")
try:
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="security_status",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
except:
self.assertTrue(True)
@@ -578,10 +604,13 @@ class TestHistoryServer(RMFTestCase):
put_structured_out_mock.reset_mock()
get_params_mock.return_value = empty_security_params
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="security_status",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal not set."})
# Testing with not empty result_issues
@@ -592,15 +621,21 @@ class TestHistoryServer(RMFTestCase):
validate_security_config_mock.return_value = result_issues_with_params
get_params_mock.return_value = security_params
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="security_status",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
# Testing with security_enable = false
- self.executeScript("2.0.6/services/YARN/package/scripts/historyserver.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname="HistoryServer",
command="security_status",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
index 3c95ac2..b42265c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
@@ -26,12 +26,16 @@ origin_exists = os.path.exists
side_effect=lambda *args: origin_exists(args[0])
if args[0][-2:] == "j2" else True))
class TestMapReduce2Client(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
+ STACK_VERSION = "2.0.6"
def test_configure_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/mapreduce2_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapreduce2_client.py",
classname = "MapReduce2Client",
command = "configure",
- config_file="default.json"
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
@@ -162,10 +166,12 @@ class TestMapReduce2Client(RMFTestCase):
def test_configure_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/mapreduce2_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapreduce2_client.py",
classname = "MapReduce2Client",
command = "configure",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
@@ -310,10 +316,13 @@ class TestMapReduce2Client(RMFTestCase):
self.assertNoMoreResources()
def test_upgrade(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/mapreduce2_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapreduce2_client.py",
classname = "MapReduce2Client",
command = "restart",
- config_file="client-upgrade.json")
+ config_file="client-upgrade.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled("Execute", "hdp-select set hadoop-client 2.2.1.0-2067")
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_service_check.py
index fd93279..f9f9812 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_service_check.py
@@ -26,13 +26,17 @@ origin_exists = os.path.exists
side_effect=lambda *args: origin_exists(args[0])
if args[0][-2:] == "j2" else True))
class TestServiceCheck(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
+ STACK_VERSION = "2.0.6"
def test_service_check_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/mapred_service_check.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapred_service_check.py",
classname="MapReduce2ServiceCheck",
command="service_check",
- config_file="default.json"
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('ExecuteHadoop', 'fs -rm -r -f /user/ambari-qa/mapredsmokeoutput /user/ambari-qa/mapredsmokeinput',
try_sleep = 5,
@@ -65,10 +69,12 @@ class TestServiceCheck(RMFTestCase):
def test_service_check_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/mapred_service_check.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapred_service_check.py",
classname="MapReduce2ServiceCheck",
command="service_check",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
user = 'ambari-qa',
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
index 2365520..2854399 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
@@ -28,20 +28,28 @@ origin_exists = os.path.exists
side_effect=lambda *args: origin_exists(args[0])
if args[0][-2:] == "j2" else True))
class TestNodeManager(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
+ STACK_VERSION = "2.0.6"
def test_configure_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="configure",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_default()
self.assertNoMoreResources()
def test_start_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="start",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_default()
pid_check_cmd = 'ls /var/run/hadoop-yarn/yarn/yarn-yarn-nodemanager.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop-yarn/yarn/yarn-yarn-nodemanager.pid` >/dev/null 2>&1'
@@ -59,10 +67,13 @@ class TestNodeManager(RMFTestCase):
self.assertNoMoreResources()
def test_stop_default(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="stop",
- config_file="default.json")
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled('Execute', 'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config /etc/hadoop/conf stop nodemanager',
user='yarn')
self.assertResourceCalled('File', '/var/run/hadoop-yarn/yarn/yarn-yarn-nodemanager.pid',
@@ -71,18 +82,24 @@ class TestNodeManager(RMFTestCase):
def test_configure_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="configure",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_secured()
self.assertNoMoreResources()
def test_start_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="start",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assert_configure_secured()
@@ -101,10 +118,13 @@ class TestNodeManager(RMFTestCase):
self.assertNoMoreResources()
def test_stop_secured(self):
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="stop",
- config_file="secured.json")
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertResourceCalled('Execute', 'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config /etc/hadoop/conf stop nodemanager',
user='yarn')
self.assertResourceCalled('File', '/var/run/hadoop-yarn/yarn/yarn-yarn-nodemanager.pid',
@@ -568,8 +588,13 @@ class TestNodeManager(RMFTestCase):
process_mock.return_value = (0, process_output)
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
- classname="Nodemanager", command = "post_rolling_restart", config_file="default.json")
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
+ classname="Nodemanager",
+ command = "post_rolling_restart",
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.assertTrue(process_mock.called)
self.assertEqual(process_mock.call_count,1)
@@ -585,8 +610,13 @@ class TestNodeManager(RMFTestCase):
process_mock.return_value = (0, process_output)
try:
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
- classname="Nodemanager", command = "post_rolling_restart", config_file="default.json")
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
+ classname="Nodemanager",
+ command = "post_rolling_restart",
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.fail('Missing NodeManager should have caused a failure')
except Fail,fail:
self.assertTrue(process_mock.called)
@@ -603,8 +633,13 @@ class TestNodeManager(RMFTestCase):
process_mock.return_value = (999, process_output)
try:
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
- classname="Nodemanager", command = "post_rolling_restart", config_file="default.json")
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
+ classname="Nodemanager",
+ command = "post_rolling_restart",
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
self.fail('Invalid return code should cause a failure')
except Fail,fail:
self.assertTrue(process_mock.called)
@@ -640,10 +675,12 @@ class TestNodeManager(RMFTestCase):
get_params_mock.return_value = security_params
validate_security_config_mock.return_value = result_issues
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="security_status",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
build_exp_mock.assert_called_with('yarn-site', props_value_check, props_empty_check, props_read_check)
@@ -662,10 +699,12 @@ class TestNodeManager(RMFTestCase):
cached_kinit_executor_mock.side_effect = Exception("Invalid command")
try:
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="security_status",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
except:
self.assertTrue(True)
@@ -677,10 +716,12 @@ class TestNodeManager(RMFTestCase):
put_structured_out_mock.reset_mock()
get_params_mock.return_value = empty_security_params
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="security_status",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."})
@@ -693,17 +734,21 @@ class TestNodeManager(RMFTestCase):
validate_security_config_mock.return_value = result_issues_with_params
get_params_mock.return_value = security_params
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="security_status",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
# Testing with security_enable = false
- self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/nodemanager.py",
classname="Nodemanager",
command="security_status",
- config_file="default.json"
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
\ No newline at end of file
[8/8] ambari git commit: AMBARI-8878: Common Services: Refactor
HDP-2.0.6 YARN service (Jayush Luniya)
Posted by jl...@apache.org.
AMBARI-8878: Common Services: Refactor HDP-2.0.6 YARN service (Jayush Luniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/53c39154
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/53c39154
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/53c39154
Branch: refs/heads/trunk
Commit: 53c391541f8701cb8b3923da7ae713d20e4d14d7
Parents: 7159bdc
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Tue Jan 6 18:35:39 2015 -0800
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Tue Jan 6 18:35:39 2015 -0800
----------------------------------------------------------------------
.../ambari/server/stack/StackManager.java | 3 +-
.../common-services/YARN/2.1.0.2.0/alerts.json | 341 ++
.../configuration-mapred/mapred-env.xml | 65 +
.../configuration-mapred/mapred-site.xml | 360 ++
.../configuration/capacity-scheduler.xml | 132 +
.../YARN/2.1.0.2.0/configuration/yarn-env.xml | 179 +
.../YARN/2.1.0.2.0/configuration/yarn-log4j.xml | 71 +
.../YARN/2.1.0.2.0/configuration/yarn-site.xml | 342 ++
.../common-services/YARN/2.1.0.2.0/metainfo.xml | 242 +
.../common-services/YARN/2.1.0.2.0/metrics.json | 5360 ++++++++++++++++++
.../package/alerts/alert_nodemanager_health.py | 123 +
.../files/validateYarnComponentStatus.py | 170 +
.../YARN/2.1.0.2.0/package/scripts/__init__.py | 20 +
.../scripts/application_timeline_server.py | 132 +
.../2.1.0.2.0/package/scripts/historyserver.py | 123 +
.../package/scripts/mapred_service_check.py | 77 +
.../package/scripts/mapreduce2_client.py | 49 +
.../2.1.0.2.0/package/scripts/nodemanager.py | 137 +
.../package/scripts/nodemanager_upgrade.py | 74 +
.../YARN/2.1.0.2.0/package/scripts/params.py | 209 +
.../package/scripts/resourcemanager.py | 173 +
.../YARN/2.1.0.2.0/package/scripts/service.py | 76 +
.../2.1.0.2.0/package/scripts/service_check.py | 68 +
.../2.1.0.2.0/package/scripts/status_params.py | 43 +
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 249 +
.../2.1.0.2.0/package/scripts/yarn_client.py | 49 +
.../package/templates/container-executor.cfg.j2 | 40 +
.../package/templates/exclude_hosts_list.j2 | 21 +
.../package/templates/mapreduce.conf.j2 | 35 +
.../package/templates/taskcontroller.cfg.j2 | 38 +
.../2.1.0.2.0/package/templates/yarn.conf.j2 | 35 +
.../stacks/HDP/2.0.6/services/YARN/alerts.json | 341 --
.../YARN/configuration-mapred/mapred-env.xml | 65 -
.../YARN/configuration-mapred/mapred-site.xml | 360 --
.../YARN/configuration/capacity-scheduler.xml | 132 -
.../services/YARN/configuration/yarn-env.xml | 179 -
.../services/YARN/configuration/yarn-log4j.xml | 71 -
.../services/YARN/configuration/yarn-site.xml | 342 --
.../stacks/HDP/2.0.6/services/YARN/metainfo.xml | 215 +-
.../stacks/HDP/2.0.6/services/YARN/metrics.json | 5360 ------------------
.../package/alerts/alert_nodemanager_health.py | 123 -
.../files/validateYarnComponentStatus.py | 170 -
.../services/YARN/package/scripts/__init__.py | 20 -
.../scripts/application_timeline_server.py | 132 -
.../YARN/package/scripts/historyserver.py | 123 -
.../package/scripts/mapred_service_check.py | 77 -
.../YARN/package/scripts/mapreduce2_client.py | 49 -
.../YARN/package/scripts/nodemanager.py | 137 -
.../YARN/package/scripts/nodemanager_upgrade.py | 74 -
.../services/YARN/package/scripts/params.py | 209 -
.../YARN/package/scripts/resourcemanager.py | 173 -
.../services/YARN/package/scripts/service.py | 76 -
.../YARN/package/scripts/service_check.py | 68 -
.../YARN/package/scripts/status_params.py | 43 -
.../2.0.6/services/YARN/package/scripts/yarn.py | 249 -
.../YARN/package/scripts/yarn_client.py | 49 -
.../package/templates/container-executor.cfg.j2 | 40 -
.../package/templates/exclude_hosts_list.j2 | 21 -
.../YARN/package/templates/mapreduce.conf.j2 | 35 -
.../package/templates/taskcontroller.cfg.j2 | 38 -
.../YARN/package/templates/yarn.conf.j2 | 35 -
.../YARN/configuration/capacity-scheduler.xml | 131 -
.../services/YARN/configuration/yarn-env.xml | 150 -
.../services/YARN/configuration/yarn-site.xml | 319 --
.../stacks/HDP/2.1/services/YARN/metainfo.xml | 1 -
.../YARN/configuration-mapred/mapred-env.xml | 46 +
.../YARN/configuration-mapred/mapred-site.xml | 226 +-
.../YARN/configuration/capacity-scheduler.xml | 114 -
.../services/YARN/configuration/yarn-env.xml | 45 +
.../services/YARN/configuration/yarn-site.xml | 161 +-
.../HDPWIN/2.1/services/YARN/metainfo.xml | 168 +-
.../stacks/2.0.6/YARN/test_historyserver.py | 81 +-
.../stacks/2.0.6/YARN/test_mapreduce2_client.py | 21 +-
.../2.0.6/YARN/test_mapreduce2_service_check.py | 14 +-
.../stacks/2.0.6/YARN/test_nodemanager.py | 101 +-
.../stacks/2.0.6/YARN/test_resourcemanager.py | 86 +-
.../stacks/2.0.6/YARN/test_yarn_client.py | 27 +-
.../2.0.6/YARN/test_yarn_service_check.py | 14 +-
.../stacks/2.1/YARN/test_apptimelineserver.py | 31 +-
ambari-web/app/data/HDP2/site_properties.js | 2 +
80 files changed, 9414 insertions(+), 10366 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
index 216fffd..e55120e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
@@ -249,7 +249,8 @@ public class StackManager {
ServiceMetainfoXml metaInfoXml = serviceDirectory.getMetaInfoFile();
for (ServiceInfo serviceInfo : metaInfoXml.getServices()) {
ServiceModule serviceModule = new ServiceModule(stackContext, serviceInfo, serviceDirectory, true);
- String commonServiceKey = serviceName + StackManager.PATH_DELIMITER + serviceVersion;
+
+ String commonServiceKey = serviceInfo.getName() + StackManager.PATH_DELIMITER + serviceInfo.getVersion();
commonServiceModules.put(commonServiceKey, serviceModule);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json
new file mode 100644
index 0000000..f3813ec
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json
@@ -0,0 +1,341 @@
+{
+ "MAPREDUCE2": {
+ "service": [],
+ "HISTORYSERVER": [
+ {
+ "name": "mapreduce_history_server_webui",
+ "label": "History Server Web UI",
+ "description": "This host-level alert is triggered if the History Server Web UI is unreachable.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "WEB",
+ "uri": {
+ "http": "{{mapred-site/mapreduce.jobhistory.webapp.address}}",
+ "https": "{{mapred-site/mapreduce.jobhistory.webapp.https.address}}",
+ "https_property": "{{mapred-site/mapreduce.jobhistory.http.policy}}",
+ "https_property_value": "HTTPS_ONLY"
+ },
+ "reporting": {
+ "ok": {
+ "text": "HTTP {0} response in {2:.3f} seconds"
+ },
+ "warning":{
+ "text": "HTTP {0} response in {2:.3f} seconds"
+ },
+ "critical": {
+ "text": "Connection failed to {1}"
+ }
+ }
+ }
+ },
+ {
+ "name": "mapreduce_history_server_cpu",
+ "label": "History Server CPU Utilization",
+ "description": "This host-level alert is triggered if the percent of CPU utilization on the History Server exceeds the configured critical threshold. The threshold values are in percent.",
+ "interval": 5,
+ "scope": "ANY",
+ "enabled": true,
+ "source": {
+ "type": "METRIC",
+ "uri": {
+ "http": "{{mapred-site/mapreduce.jobhistory.webapp.address}}",
+ "https": "{{mapred-site/mapreduce.jobhistory.webapp.https.address}}",
+ "https_property": "{{mapred-site/mapreduce.jobhistory.http.policy}}",
+ "https_property_value": "HTTPS_ONLY"
+ },
+ "reporting": {
+ "ok": {
+ "text": "{1} CPU, load {0:.1%}"
+ },
+ "warning": {
+ "text": "{1} CPU, load {0:.1%}",
+ "value": 200
+ },
+ "critical": {
+ "text": "{1} CPU, load {0:.1%}",
+ "value": 250
+ },
+ "units" : "%"
+ },
+ "jmx": {
+ "property_list": [
+ "java.lang:type=OperatingSystem/SystemCpuLoad",
+ "java.lang:type=OperatingSystem/AvailableProcessors"
+ ],
+ "value": "{0} * 100"
+ }
+ }
+ },
+ {
+ "name": "mapreduce_history_server_rpc_latency",
+ "label": "History Server RPC Latency",
+ "description": "This host-level alert is triggered if the History Server operations RPC latency exceeds the configured critical threshold. Typically an increase in the RPC processing time increases the RPC queue length, causing the average queue wait time to increase for operations. The threshold values are in milliseconds.",
+ "interval": 5,
+ "scope": "ANY",
+ "enabled": true,
+ "source": {
+ "type": "METRIC",
+ "uri": {
+ "http": "{{mapred-site/mapreduce.jobhistory.webapp.address}}",
+ "https": "{{mapred-site/mapreduce.jobhistory.webapp.https.address}}",
+ "https_property": "{{mapred-site/mapreduce.jobhistory.http.policy}}",
+ "https_property_value": "HTTPS_ONLY"
+ },
+ "reporting": {
+ "ok": {
+ "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]"
+ },
+ "warning": {
+ "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
+ "value": 3000
+ },
+ "critical": {
+ "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
+ "value": 5000
+ },
+ "units" : "ms"
+ },
+ "jmx": {
+ "property_list": [
+ "Hadoop:service=JobHistoryServer,name=RpcActivityForPort*/RpcQueueTimeAvgTime",
+ "Hadoop:service=JobHistoryServer,name=RpcActivityForPort*/RpcProcessingTimeAvgTime"
+ ],
+ "value": "{0}"
+ }
+ }
+ },
+ {
+ "name": "mapreduce_history_server_process",
+ "label": "History Server Process",
+ "description": "This host-level alert is triggered if the History Server process cannot be established to be up and listening on the network.",
+ "interval": 1,
+ "scope": "ANY",
+ "enabled": true,
+ "source": {
+ "type": "PORT",
+ "uri": "{{mapred-site/mapreduce.jobhistory.webapp.address}}",
+ "default_port": 19888,
+ "reporting": {
+ "ok": {
+ "text": "TCP OK - {0:.3f}s response on port {1}"
+ },
+ "warning": {
+ "text": "TCP OK - {0:.3f}s response on port {1}",
+ "value": 1.5
+ },
+ "critical": {
+ "text": "Connection failed: {0} to {1}:{2}",
+ "value": 5.0
+ }
+ }
+ }
+ }
+ ]
+ },
+ "YARN": {
+ "service": [
+ {
+ "name": "yarn_nodemanager_webui_percent",
+ "label": "Percent NodeManagers Available",
+ "description": "This alert is triggered if the number of down NodeManagers in the cluster is greater than the configured critical threshold. It aggregates the results of NodeManager process checks.",
+ "interval": 1,
+ "scope": "SERVICE",
+ "enabled": true,
+ "source": {
+ "type": "AGGREGATE",
+ "alert_name": "yarn_nodemanager_webui",
+ "reporting": {
+ "ok": {
+ "text": "affected: [{1}], total: [{0}]"
+ },
+ "warning": {
+ "text": "affected: [{1}], total: [{0}]",
+ "value": 0.1
+ },
+ "critical": {
+ "text": "affected: [{1}], total: [{0}]",
+ "value": 0.3
+ }
+ }
+ }
+ }
+ ],
+ "NODEMANAGER": [
+ {
+ "name": "yarn_nodemanager_webui",
+ "label": "NodeManager Web UI",
+ "description": "This host-level alert is triggered if the NodeManager Web UI is unreachable.",
+ "interval": 1,
+ "scope": "HOST",
+ "source": {
+ "type": "WEB",
+ "uri": {
+ "http": "{{yarn-site/yarn.nodemanager.webapp.address}}",
+ "https": "{{yarn-site/yarn.nodemanager.webapp.https.address}}",
+ "https_property": "{{yarn-site/yarn.http.policy}}",
+ "https_property_value": "HTTPS_ONLY",
+ "default_port": 8042
+ },
+ "reporting": {
+ "ok": {
+ "text": "HTTP {0} response in {2:.3f} seconds"
+ },
+ "warning":{
+ "text": "HTTP {0} response in {2:.3f} seconds"
+ },
+ "critical": {
+ "text": "Connection failed to {1}"
+ }
+ }
+ }
+ },
+ {
+ "name": "yarn_nodemanager_health",
+ "label": "NodeManager Health",
+ "description": "This host-level alert checks the node health property available from the NodeManager component.",
+ "interval": 1,
+ "scope": "HOST",
+ "enabled": true,
+ "source": {
+ "type": "SCRIPT",
+ "path": "HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py"
+ }
+ }
+ ],
+ "RESOURCEMANAGER": [
+ {
+ "name": "yarn_resourcemanager_webui",
+ "label": "ResourceManager Web UI",
+ "description": "This host-level alert is triggered if the ResourceManager Web UI is unreachable.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "WEB",
+ "uri": {
+ "http": "{{yarn-site/yarn.resourcemanager.webapp.address}}",
+ "https": "{{yarn-site/yarn.resourcemanager.webapp.https.address}}",
+ "https_property": "{{yarn-site/yarn.http.policy}}",
+ "https_property_value": "HTTPS_ONLY"
+ },
+ "reporting": {
+ "ok": {
+ "text": "HTTP {0} response in {2:.3f} seconds"
+ },
+ "warning":{
+ "text": "HTTP {0} response in {2:.3f} seconds"
+ },
+ "critical": {
+ "text": "Connection failed to {1}"
+ }
+ }
+ }
+ },
+ {
+ "name": "yarn_resourcemanager_cpu",
+ "label": "ResourceManager CPU Utilization",
+ "description": "This host-level alert is triggered if CPU utilization of the ResourceManager exceeds certain warning and critical thresholds. It checks the ResourceManager JMX Servlet for the SystemCPULoad property. The threshold values are in percent.",
+ "interval": 5,
+ "scope": "ANY",
+ "enabled": true,
+ "source": {
+ "type": "METRIC",
+ "uri": {
+ "http": "{{yarn-site/yarn.resourcemanager.webapp.address}}",
+ "https": "{{yarn-site/yarn.resourcemanager.webapp.https.address}}",
+ "https_property": "{{yarn-site/yarn.http.policy}}",
+ "https_property_value": "HTTPS_ONLY"
+ },
+ "reporting": {
+ "ok": {
+ "text": "{1} CPU, load {0:.1%}"
+ },
+ "warning": {
+ "text": "{1} CPU, load {0:.1%}",
+ "value": 200
+ },
+ "critical": {
+ "text": "{1} CPU, load {0:.1%}",
+ "value": 250
+ },
+ "units" : "%"
+ },
+ "jmx": {
+ "property_list": [
+ "java.lang:type=OperatingSystem/SystemCpuLoad",
+ "java.lang:type=OperatingSystem/AvailableProcessors"
+ ],
+ "value": "{0} * 100"
+ }
+ }
+ },
+ {
+ "name": "yarn_resourcemanager_rpc_latency",
+ "label": "ResourceManager RPC Latency",
+ "description": "This host-level alert is triggered if the ResourceManager operations RPC latency exceeds the configured critical threshold. Typically an increase in the RPC processing time increases the RPC queue length, causing the average queue wait time to increase for ResourceManager operations. The threshold values are in milliseconds.",
+ "interval": 5,
+ "scope": "ANY",
+ "enabled": true,
+ "source": {
+ "type": "METRIC",
+ "uri": {
+ "http": "{{yarn-site/yarn.resourcemanager.webapp.address}}",
+ "https": "{{yarn-site/yarn.resourcemanager.webapp.https.address}}",
+ "https_property": "{{yarn-site/yarn.http.policy}}",
+ "https_property_value": "HTTPS_ONLY"
+ },
+ "reporting": {
+ "ok": {
+ "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]"
+ },
+ "warning": {
+ "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
+ "value": 3000
+ },
+ "critical": {
+ "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
+ "value": 5000
+ },
+ "units" : "ms"
+ },
+ "jmx": {
+ "property_list": [
+ "Hadoop:service=ResourceManager,name=RpcActivityForPort*/RpcQueueTimeAvgTime",
+ "Hadoop:service=ResourceManager,name=RpcActivityForPort*/RpcProcessingTimeAvgTime"
+ ],
+ "value": "{0}"
+ }
+ }
+ }
+ ],
+ "APP_TIMELINE_SERVER": [
+ {
+ "name": "yarn_app_timeline_server_webui",
+ "label": "App Timeline Web UI",
+ "description": "This host-level alert is triggered if the App Timeline Server Web UI is unreachable.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "WEB",
+ "uri": {
+ "http": "{{yarn-site/yarn.timeline-service.webapp.address}}",
+ "https": "{{yarn-site/yarn.timeline-service.webapp.https.address}}",
+ "https_property": "{{yarn-site/yarn.http.policy}}",
+ "https_property_value": "HTTPS_ONLY"
+ },
+ "reporting": {
+ "ok": {
+ "text": "HTTP {0} response in {2:.3f} seconds"
+ },
+ "warning":{
+ "text": "HTTP {0} response in {2:.3f} seconds"
+ },
+ "critical": {
+ "text": "Connection failed to {1}"
+ }
+ }
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-env.xml
new file mode 100644
index 0000000..14ae20b
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-env.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>mapred_log_dir_prefix</name>
+ <value>/var/log/hadoop-mapreduce</value>
+ <description>Mapreduce Log Dir Prefix</description>
+ </property>
+ <property>
+ <name>mapred_pid_dir_prefix</name>
+ <value>/var/run/hadoop-mapreduce</value>
+ <description>Mapreduce PID Dir Prefix</description>
+ </property>
+ <property>
+ <name>mapred_user</name>
+ <value>mapred</value>
+ <property-type>USER</property-type>
+ <description>Mapreduce User</description>
+ </property>
+ <property>
+ <name>jobhistory_heapsize</name>
+ <value>900</value>
+ <description>Value for JobHistoryServer heap_size variable in hadoop-env.sh</description>
+ </property>
+
+ <!-- mapred-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for mapred-env.sh file</description>
+ <value>
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+
+export HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}
+
+export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+
+#export HADOOP_JOB_HISTORYSERVER_OPTS=
+#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.
+#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.
+#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.
+#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
+#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
new file mode 100644
index 0000000..7955cb2
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
@@ -0,0 +1,360 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration supports_final="true" xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<!-- i/o properties -->
+
+ <property>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>200</value>
+ <description>
+ The total amount of buffer memory to use while sorting files, in megabytes.
+ By default, gives each merge stream 1MB, which should minimize seeks.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.map.sort.spill.percent</name>
+ <value>0.7</value>
+ <description>
+ The soft limit in the serialization buffer. Once reached, a thread will
+ begin to spill the contents to disk in the background. Note that
+ collection will not block if this threshold is exceeded while a spill
+ is already in progress, so spills may be larger than this threshold when
+ it is set to less than .5
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.io.sort.factor</name>
+ <value>100</value>
+ <description>
+ The number of streams to merge at once while sorting files.
+ This determines the number of open file handles.
+ </description>
+ </property>
+
+<!-- map/reduce properties -->
+ <property>
+ <name>mapreduce.cluster.administrators</name>
+ <value> hadoop</value>
+ <description>
+ Administrators for MapReduce applications.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.shuffle.parallelcopies</name>
+ <value>30</value>
+ <description>
+ The default number of parallel transfers run by reduce during
+ the copy(shuffle) phase.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.map.speculative</name>
+ <value>false</value>
+ <description>
+ If true, then multiple instances of some map tasks
+ may be executed in parallel.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.speculative</name>
+ <value>false</value>
+ <description>
+ If true, then multiple instances of some reduce tasks may be
+ executed in parallel.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.job.reduce.slowstart.completedmaps</name>
+ <value>0.05</value>
+ <description>
+ Fraction of the number of maps in the job which should be complete before
+ reduces are scheduled for the job.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.shuffle.merge.percent</name>
+ <value>0.66</value>
+ <description>
+ The usage threshold at which an in-memory merge will be
+ initiated, expressed as a percentage of the total memory allocated to
+ storing in-memory map outputs, as defined by
+ mapreduce.reduce.shuffle.input.buffer.percent.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
+ <value>0.7</value>
+ <description>
+ The percentage of memory to be allocated from the maximum heap
+ size to storing map outputs during the shuffle.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+ <description>
+ If the job outputs are to compressed as SequenceFiles, how should
+ they be compressed? Should be one of NONE, RECORD or BLOCK.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.input.buffer.percent</name>
+ <value>0.0</value>
+ <description>
+ The percentage of memory- relative to the maximum heap size- to
+ retain map outputs during the reduce. When the shuffle is concluded, any
+ remaining map outputs in memory must consume less than this threshold before
+ the reduce can begin.
+ </description>
+ </property>
+
+ <!-- copied from kryptonite configuration -->
+ <property>
+ <name>mapreduce.map.output.compress</name>
+ <value>false</value>
+ <description>
+ Should the outputs of the maps be compressed before being sent across the network. Uses SequenceFile compression.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.timeout</name>
+ <value>300000</value>
+ <description>
+ The number of milliseconds before a task will be
+ terminated if it neither reads an input, writes an output, nor
+ updates its status string.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>1024</value>
+ <description>Virtual memory for single Map task</description>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>1024</value>
+ <description>Virtual memory for single Reduce task</description>
+ </property>
+
+ <property>
+ <name>mapreduce.shuffle.port</name>
+ <value>13562</value>
+ <description>
+ Default port that the ShuffleHandler will run on.
+ ShuffleHandler is a service run at the NodeManager to facilitate
+ transfers of intermediate Map outputs to requesting Reducers.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.intermediate-done-dir</name>
+ <value>/mr-history/tmp</value>
+ <description>
+ Directory where history files are written by MapReduce jobs.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.done-dir</name>
+ <value>/mr-history/done</value>
+ <description>
+ Directory where history files are managed by the MR JobHistory Server.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.address</name>
+ <value>localhost:10020</value>
+ <description>Enter your JobHistoryServer hostname.</description>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>localhost:19888</value>
+ <description>Enter your JobHistoryServer hostname.</description>
+ </property>
+
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ <description>
+ The runtime framework for executing MapReduce jobs. Can be one of local,
+ classic or yarn.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
+ <description>
+ The staging dir used while submitting jobs.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <value>512</value>
+ <description>The amount of memory the MR AppMaster needs.</description>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.command-opts</name>
+ <value>-Xmx312m</value>
+ <description>
+ Java opts for the MR App Master processes.
+ The following symbol, if present, will be interpolated: @taskid@ is replaced
+ by current TaskID. Any other occurrences of '@' will go unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+ -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+
+ Usage of -Djava.library.path can cause programs to no longer function if
+ hadoop native libraries are used. These values should instead be set as part
+ of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
+ mapreduce.reduce.env config settings.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.admin-command-opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
+ <description>
+ Java opts for the MR App Master processes for admin purposes.
+ It will appears before the opts set by yarn.app.mapreduce.am.command-opts and
+ thus its options can be overridden user.
+
+ Usage of -Djava.library.path can cause programs to no longer function if
+ hadoop native libraries are used. These values should instead be set as part
+ of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
+ mapreduce.reduce.env config settings.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.log.level</name>
+ <value>INFO</value>
+ <description>MR App Master process log level.</description>
+ </property>
+
+ <property>
+ <name>mapreduce.admin.map.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
+ <description>This property stores Java options for map tasks.</description>
+ </property>
+
+ <property>
+ <name>mapreduce.admin.reduce.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
+ <description>This property stores Java options for reduce tasks.</description>
+ </property>
+
+ <property>
+ <name>mapreduce.application.classpath</name>
+ <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
+ <description>
+ CLASSPATH for MR applications. A comma-separated list of CLASSPATH
+ entries.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.am.max-attempts</name>
+ <value>2</value>
+ <description>
+ The maximum number of application attempts. It is a
+ application-specific setting. It should not be larger than the global number
+ set by resourcemanager. Otherwise, it will be override. The default number is
+ set to 2, to allow at least one retry for AM.
+ </description>
+ </property>
+
+
+
+ <property>
+ <name>mapreduce.map.java.opts</name>
+ <value>-Xmx756m</value>
+ <description>
+ Larger heap-size for child jvms of maps.
+ </description>
+ </property>
+
+
+ <property>
+ <name>mapreduce.reduce.java.opts</name>
+ <value>-Xmx756m</value>
+ <description>
+ Larger heap-size for child jvms of reduces.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.map.log.level</name>
+ <value>INFO</value>
+ <description>
+ The logging level for the map task. The allowed levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.log.level</name>
+ <value>INFO</value>
+ <description>
+ The logging level for the reduce task. The allowed levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:/usr/lib/hadoop/lib/native/Linux-amd64-64</value>
+ <description>
+ Additional execution environment entries for map and reduce task processes.
+ This is not an additive property. You must preserve the original value if
+ you want your map and reduce tasks to have access to native libraries (compression, etc)
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>false</value>
+ <description>
+ Should the job outputs be compressed?
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/capacity-scheduler.xml
new file mode 100644
index 0000000..6c21848
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/capacity-scheduler.xml
@@ -0,0 +1,132 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration supports_final="false" supports_adding_forbidden="true">
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-applications</name>
+ <value>10000</value>
+ <description>
+ Maximum number of applications that can be pending and running.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.2</value>
+ <description>
+ Maximum percent of resources in the cluster which can be used to run
+ application masters i.e. controls number of concurrent running
+ applications.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ <value>default</value>
+ <description>
+ The queues at the this level (root is the root queue).
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.capacity</name>
+ <value>100</value>
+ <description>
+ The total capacity as a percentage out of 100 for this queue.
+ If it has child queues then this includes their capacity as well.
+ The child queues capacity should add up to their parent queue's capacity
+ or less.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.capacity</name>
+ <value>100</value>
+ <description>Default queue target capacity.</description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
+ <value>1</value>
+ <description>
+ Default queue user limit a percentage from 0.0 to 1.0.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
+ <value>100</value>
+ <description>
+ The maximum capacity of the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.state</name>
+ <value>RUNNING</value>
+ <description>
+ The state of the default queue. State can be one of RUNNING or STOPPED.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
+ <value>*</value>
+ <description>
+ The ACL of who can submit jobs to the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
+ <value>*</value>
+ <description>
+ The ACL of who can administer jobs on the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.acl_administer_queue</name>
+ <value>*</value>
+ <description>
+ The ACL for who can administer this queue i.e. change sub-queue
+ allocations.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.node-locality-delay</name>
+ <value>40</value>
+ <description>
+ Number of missed scheduling opportunities after which the CapacityScheduler
+ attempts to schedule rack-local containers.
+ Typically this should be set to number of nodes in the cluster, By default is setting
+ approximately number of nodes in one rack which is 40.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.default.minimum-user-limit-percent</name>
+ <value>100</value>
+ <description>
+ Default minimum queue resource limit depends on the number of users who have submitted applications.
+ </description>
+ </property>
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
new file mode 100644
index 0000000..1c23ade
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
@@ -0,0 +1,179 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>yarn_log_dir_prefix</name>
+ <value>/var/log/hadoop-yarn</value>
+ <description>YARN Log Dir Prefix</description>
+ </property>
+ <property>
+ <name>yarn_pid_dir_prefix</name>
+ <value>/var/run/hadoop-yarn</value>
+ <description>YARN PID Dir Prefix</description>
+ </property>
+ <property>
+ <name>yarn_user</name>
+ <value>yarn</value>
+ <property-type>USER</property-type>
+ <description>YARN User</description>
+ </property>
+ <property>
+ <name>yarn_heapsize</name>
+ <value>1024</value>
+ <description>Max heapsize for all YARN components using a numerical value in the scale of MB</description>
+ </property>
+ <property>
+ <name>resourcemanager_heapsize</name>
+ <value>1024</value>
+ <description>Max heapsize for ResourceManager using a numerical value in the scale of MB</description>
+ </property>
+ <property>
+ <name>nodemanager_heapsize</name>
+ <value>1024</value>
+ <description>Max heapsize for NodeManager using a numerical value in the scale of MB</description>
+ </property>
+ <property>
+ <name>min_user_id</name>
+ <value>1000</value>
+ <description>Set to 0 to disallow root from submitting jobs. Set to 1000 to disallow all superusers from submitting jobs</description>
+ </property>
+
+ <!-- yarn-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for yarn-env.sh file</description>
+ <value>
+export HADOOP_YARN_HOME={{hadoop_yarn_home}}
+export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER
+export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
+export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
+export JAVA_HOME={{java64_home}}
+
+# User for YARN daemons
+export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+# resolve links - $0 may be a softlink
+export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+
+# some Java parameters
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+if [ "$JAVA_HOME" != "" ]; then
+ #echo "run java in $JAVA_HOME"
+ JAVA_HOME=$JAVA_HOME
+fi
+
+if [ "$JAVA_HOME" = "" ]; then
+ echo "Error: JAVA_HOME is not set."
+ exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m
+
+# For setting YARN specific HEAP sizes please use this
+# Parameter and set appropriately
+YARN_HEAPSIZE={{yarn_heapsize}}
+
+# check envvars which might override default args
+if [ "$YARN_HEAPSIZE" != "" ]; then
+ JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+fi
+
+# Resource Manager specific parameters
+
+# Specify the max Heapsize for the ResourceManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_RESOURCEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}
+
+# Specify the JVM options to be used when starting the ResourceManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_RESOURCEMANAGER_OPTS=
+
+# Node Manager specific parameters
+
+# Specify the max Heapsize for the NodeManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_NODEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}
+
+# Specify the max Heapsize for the HistoryManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1024.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_HISTORYSERVER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_HISTORYSERVER_HEAPSIZE={{apptimelineserver_heapsize}}
+
+# Specify the JVM options to be used when starting the NodeManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_NODEMANAGER_OPTS=
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+
+# default log directory and file
+if [ "$YARN_LOG_DIR" = "" ]; then
+ YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
+fi
+if [ "$YARN_LOGFILE" = "" ]; then
+ YARN_LOGFILE='yarn.log'
+fi
+
+# default policy file for service-level authorization
+if [ "$YARN_POLICYFILE" = "" ]; then
+ YARN_POLICYFILE="hadoop-policy.xml"
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
+YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
+YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+ YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi
+YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+ </value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-log4j.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-log4j.xml
new file mode 100644
index 0000000..8c44b9e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-log4j.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false">
+
+ <property>
+ <name>content</name>
+ <description>Custom log4j.properties</description>
+ <value>
+#Relative to Yarn Log Dir Prefix
+yarn.log.dir=.
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file rolled daily:
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
+# Set the ResourceManager summary log filename
+yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log
+# Set the ResourceManager summary log level and appender
+yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
+#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
+
+# To enable AppSummaryLogging for the RM,
+# set yarn.server.resourcemanager.appsummary.logger to
+# LEVEL,RMSUMMARY in hadoop-env.sh
+
+# Appender for ResourceManager Application Summary Log
+# Requires the following properties to be set
+# - hadoop.log.dir (Hadoop Log directory)
+# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
+# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
+log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
+log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
+log4j.appender.RMSUMMARY.MaxFileSize=256MB
+log4j.appender.RMSUMMARY.MaxBackupIndex=20
+log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
+log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+log4j.appender.JSA.DatePattern=.yyyy-MM-dd
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
+log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
+ </value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
new file mode 100644
index 0000000..6572935
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
@@ -0,0 +1,342 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration supports_final="true" xmlns:xi="http://www.w3.org/2001/XInclude">
+
+ <!-- ResourceManager -->
+
+ <property>
+ <name>yarn.resourcemanager.hostname</name>
+ <value>localhost</value>
+ <description>The hostname of the RM.</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>localhost:8025</value>
+ <description> The address of ResourceManager. </description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>localhost:8030</value>
+ <description>The address of the scheduler interface.</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>localhost:8050</value>
+ <description>
+ The address of the applications manager interface in the
+ RM.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.admin.address</name>
+ <value>localhost:8141</value>
+ <description>The address of the RM admin interface.</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
+ <description>The class to use as the resource scheduler.</description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>512</value>
+ <description>
+ The minimum allocation for every container request at the RM,
+ in MBs. Memory requests lower than this won't take effect,
+ and the specified value will get allocated at minimum.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>2048</value>
+ <description>
+ The maximum allocation for every container request at the RM,
+ in MBs. Memory requests higher than this won't take effect,
+ and will get capped to this value.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.acl.enable</name>
+ <value>false</value>
+ <description> Are acls enabled. </description>
+ </property>
+
+ <property>
+ <name>yarn.admin.acl</name>
+ <value></value>
+ <description> ACL of who can be admin of the YARN cluster. </description>
+ </property>
+
+ <!-- NodeManager -->
+
+ <property>
+ <name>yarn.nodemanager.address</name>
+ <value>0.0.0.0:45454</value>
+ <description>The address of the container manager in the NM.</description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>5120</value>
+ <description>Amount of physical memory, in MB, that can be allocated
+ for containers.</description>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*</value>
+ <description>Classpath for typical applications.</description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.vmem-pmem-ratio</name>
+ <value>2.1</value>
+ <description>Ratio between virtual memory to physical memory when
+ setting memory limits for containers. Container allocations are
+ expressed in terms of physical memory, and virtual memory usage
+ is allowed to exceed this allocation by this ratio.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
+ <description>ContainerExecutor for launching containers</description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>hadoop</value>
+ <description>Unix group of the NodeManager</description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can
+ not start with numbers</description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ <description>The auxiliary service class to use </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>/hadoop/yarn/log</value>
+ <description>
+ Where to store container logs. An application's localized log directory
+ will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
+ Individual containers' log directories will be below this, in directories
+ named container_{$contid}. Each container directory will contain the files
+ stderr, stdin, and syslog generated by that container.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value>/hadoop/yarn/local</value>
+ <description>
+ List of directories to store localized files in. An
+ application's localized file directory will be found in:
+ ${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}.
+ Individual containers' work directories, called container_${contid}, will
+ be subdirectories of this.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.container-monitor.interval-ms</name>
+ <value>3000</value>
+ <description>
+ The interval, in milliseconds, for which the node manager
+ waits between two cycles of monitoring its containers' memory usage.
+ </description>
+ </property>
+
+ <!--
+ <property>
+ <name>yarn.nodemanager.health-checker.script.path</name>
+ <value>/etc/hadoop/conf/health_check_nodemanager</value>
+ <description>The health check script to run.</description>
+ </property>
+ -->
+
+ <property>
+ <name>yarn.nodemanager.health-checker.interval-ms</name>
+ <value>135000</value>
+ <description>Frequency of running node health script.</description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
+ <value>60000</value>
+ <description>Script time out period.</description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log.retain-second</name>
+ <value>604800</value>
+ <description>
+ Time in seconds to retain user logs. Only applicable if
+ log aggregation is disabled.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ <description>Whether to enable log aggregation. </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <value>/app-logs</value>
+ <description>Location to aggregate logs to. </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
+ <value>logs</value>
+ <description>
+ The remote log dir will be created at
+ {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log-aggregation.compression-type</name>
+ <value>gz</value>
+ <description>
+ T-file compression types used to compress aggregated logs.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>0</value>
+ <description>
+ Number of seconds after an application finishes before the nodemanager's
+ DeletionService will delete the application's localized file directory
+ and log directory.
+
+ To diagnose Yarn application problems, set this property's value large
+ enough (for example, to 600 = 10 minutes) to permit examination of these
+ directories. After changing the property's value, you must restart the
+ nodemanager in order for it to have an effect.
+
+ The roots of Yarn applications' work directories is configurable with
+ the yarn.nodemanager.local-dirs property (see below), and the roots
+ of the Yarn applications' log directories is configurable with the
+ yarn.nodemanager.log-dirs property (see also below).
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.log-aggregation.retain-seconds</name>
+ <value>2592000</value>
+ <description>
+ How long to keep aggregation logs before deleting them. -1 disables.
+ Be careful set this too small and you will spam the name node.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.admin-env</name>
+ <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
+ <description>
+ Environment variables that should be forwarded from the NodeManager's
+ environment to the container's.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
+ <value>0.25</value>
+ <description>
+ The minimum fraction of number of disks to be healthy for the nodemanager
+ to launch new containers. This correspond to both
+ yarn-nodemanager.local-dirs and yarn.nodemanager.log-dirs. i.e.
+ If there are less number of healthy local-dirs (or log-dirs) available,
+ then new containers will not be launched on this node.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.am.max-attempts</name>
+ <value>2</value>
+ <description>
+ The maximum number of application attempts. It's a global
+ setting for all application masters. Each application master can specify
+ its individual maximum number of application attempts via the API, but the
+ individual number cannot be more than the global upper bound. If it is,
+ the resourcemanager will override it. The default number is set to 2, to
+ allow at least one retry for AM.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>localhost:8088</value>
+ <description>
+ The address of the RM web application.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ <description>
+ Whether virtual memory limits will be enforced for containers.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.log.server.url</name>
+ <value>http://localhost:19888/jobhistory/logs</value>
+ <description>
+ URI for the HistoryServer's log resource
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.nodes.exclude-path</name>
+ <value>/etc/hadoop/conf/yarn.exclude</value>
+ <description>
+ Names a file that contains a list of hosts that are
+ not permitted to connect to the resource manager. The full pathname of the
+ file must be specified. If the value is empty, no hosts are
+ excluded.
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
new file mode 100644
index 0000000..9c4e4ea
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
@@ -0,0 +1,242 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>YARN</name>
+ <displayName>YARN</displayName>
+ <comment>Apache Hadoop NextGen MapReduce (YARN)</comment>
+ <version>2.1.0.2.0</version>
+ <components>
+
+ <component>
+ <name>RESOURCEMANAGER</name>
+ <displayName>ResourceManager</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <commandScript>
+ <script>scripts/resourcemanager.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <customCommands>
+ <customCommand>
+ <name>DECOMMISSION</name>
+ <commandScript>
+ <script>scripts/resourcemanager.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
+ <name>REFRESHQUEUES</name>
+ <commandScript>
+ <script>scripts/resourcemanager.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
+ <configuration-dependencies>
+ <config-type>capacity-scheduler</config-type>
+ </configuration-dependencies>
+ </component>
+
+ <component>
+ <name>NODEMANAGER</name>
+ <displayName>NodeManager</displayName>
+ <category>SLAVE</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/nodemanager.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>YARN_CLIENT</name>
+ <displayName>YARN Client</displayName>
+ <category>CLIENT</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/yarn_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>yarn-site.xml</fileName>
+ <dictionaryName>yarn-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>xml</type>
+ <fileName>core-site.xml</fileName>
+ <dictionaryName>core-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>yarn-env.sh</fileName>
+ <dictionaryName>yarn-env</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>log4j.properties</fileName>
+ <dictionaryName>hdfs-log4j,yarn-log4j</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>xml</type>
+ <fileName>capacity-scheduler.xml</fileName>
+ <dictionaryName>capacity-scheduler</dictionaryName>
+ </configFile>
+ </configFiles>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>hadoop-yarn</name>
+ </package>
+ <package>
+ <name>hadoop-hdfs</name>
+ </package>
+ <package>
+ <name>hadoop-mapreduce</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>yarn-site</config-type>
+ <config-type>yarn-env</config-type>
+ <config-type>core-site</config-type>
+ <config-type>mapred-site</config-type>
+ <config-type>yarn-log4j</config-type>
+ </configuration-dependencies>
+ </service>
+
+ <service>
+ <name>MAPREDUCE2</name>
+ <displayName>MapReduce2</displayName>
+ <comment>Apache Hadoop NextGen MapReduce (YARN)</comment>
+ <version>2.1.0.2.0.6.0</version>
+ <components>
+ <component>
+ <name>HISTORYSERVER</name>
+ <displayName>History Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <auto-deploy>
+ <enabled>true</enabled>
+ <co-locate>YARN/RESOURCEMANAGER</co-locate>
+ </auto-deploy>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/historyserver.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>MAPREDUCE2_CLIENT</name>
+ <displayName>MapReduce2 Client</displayName>
+ <category>CLIENT</category>
+ <cardinality>0+</cardinality>
+ <commandScript>
+ <script>scripts/mapreduce2_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>mapred-site.xml</fileName>
+ <dictionaryName>mapred-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>xml</type>
+ <fileName>core-site.xml</fileName>
+ <dictionaryName>core-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>mapred-env.sh</fileName>
+ <dictionaryName>mapred-env</dictionaryName>
+ </configFile>
+ </configFiles>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>hadoop-mapreduce</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/mapred_service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>YARN</service>
+ </requiredServices>
+
+ <configuration-dir>configuration-mapred</configuration-dir>
+
+ <configuration-dependencies>
+ <config-type>core-site</config-type>
+ <config-type>mapred-site</config-type>
+ <config-type>mapred-env</config-type>
+ </configuration-dependencies>
+ </service>
+
+ </services>
+</metainfo>
[5/8] ambari git commit: AMBARI-8878: Common Services: Refactor
HDP-2.0.6 YARN service (Jayush Luniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn.conf.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn.conf.j2
new file mode 100644
index 0000000..3bd7a45
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn.conf.j2
@@ -0,0 +1,35 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{yarn_user}} - nofile 32768
+{{yarn_user}} - nproc 65536
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/alerts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/alerts.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/alerts.json
deleted file mode 100644
index f3813ec..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/alerts.json
+++ /dev/null
@@ -1,341 +0,0 @@
-{
- "MAPREDUCE2": {
- "service": [],
- "HISTORYSERVER": [
- {
- "name": "mapreduce_history_server_webui",
- "label": "History Server Web UI",
- "description": "This host-level alert is triggered if the History Server Web UI is unreachable.",
- "interval": 1,
- "scope": "ANY",
- "source": {
- "type": "WEB",
- "uri": {
- "http": "{{mapred-site/mapreduce.jobhistory.webapp.address}}",
- "https": "{{mapred-site/mapreduce.jobhistory.webapp.https.address}}",
- "https_property": "{{mapred-site/mapreduce.jobhistory.http.policy}}",
- "https_property_value": "HTTPS_ONLY"
- },
- "reporting": {
- "ok": {
- "text": "HTTP {0} response in {2:.3f} seconds"
- },
- "warning":{
- "text": "HTTP {0} response in {2:.3f} seconds"
- },
- "critical": {
- "text": "Connection failed to {1}"
- }
- }
- }
- },
- {
- "name": "mapreduce_history_server_cpu",
- "label": "History Server CPU Utilization",
- "description": "This host-level alert is triggered if the percent of CPU utilization on the History Server exceeds the configured critical threshold. The threshold values are in percent.",
- "interval": 5,
- "scope": "ANY",
- "enabled": true,
- "source": {
- "type": "METRIC",
- "uri": {
- "http": "{{mapred-site/mapreduce.jobhistory.webapp.address}}",
- "https": "{{mapred-site/mapreduce.jobhistory.webapp.https.address}}",
- "https_property": "{{mapred-site/mapreduce.jobhistory.http.policy}}",
- "https_property_value": "HTTPS_ONLY"
- },
- "reporting": {
- "ok": {
- "text": "{1} CPU, load {0:.1%}"
- },
- "warning": {
- "text": "{1} CPU, load {0:.1%}",
- "value": 200
- },
- "critical": {
- "text": "{1} CPU, load {0:.1%}",
- "value": 250
- },
- "units" : "%"
- },
- "jmx": {
- "property_list": [
- "java.lang:type=OperatingSystem/SystemCpuLoad",
- "java.lang:type=OperatingSystem/AvailableProcessors"
- ],
- "value": "{0} * 100"
- }
- }
- },
- {
- "name": "mapreduce_history_server_rpc_latency",
- "label": "History Server RPC Latency",
- "description": "This host-level alert is triggered if the History Server operations RPC latency exceeds the configured critical threshold. Typically an increase in the RPC processing time increases the RPC queue length, causing the average queue wait time to increase for operations. The threshold values are in milliseconds.",
- "interval": 5,
- "scope": "ANY",
- "enabled": true,
- "source": {
- "type": "METRIC",
- "uri": {
- "http": "{{mapred-site/mapreduce.jobhistory.webapp.address}}",
- "https": "{{mapred-site/mapreduce.jobhistory.webapp.https.address}}",
- "https_property": "{{mapred-site/mapreduce.jobhistory.http.policy}}",
- "https_property_value": "HTTPS_ONLY"
- },
- "reporting": {
- "ok": {
- "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]"
- },
- "warning": {
- "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
- "value": 3000
- },
- "critical": {
- "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
- "value": 5000
- },
- "units" : "ms"
- },
- "jmx": {
- "property_list": [
- "Hadoop:service=JobHistoryServer,name=RpcActivityForPort*/RpcQueueTimeAvgTime",
- "Hadoop:service=JobHistoryServer,name=RpcActivityForPort*/RpcProcessingTimeAvgTime"
- ],
- "value": "{0}"
- }
- }
- },
- {
- "name": "mapreduce_history_server_process",
- "label": "History Server Process",
- "description": "This host-level alert is triggered if the History Server process cannot be established to be up and listening on the network.",
- "interval": 1,
- "scope": "ANY",
- "enabled": true,
- "source": {
- "type": "PORT",
- "uri": "{{mapred-site/mapreduce.jobhistory.webapp.address}}",
- "default_port": 19888,
- "reporting": {
- "ok": {
- "text": "TCP OK - {0:.3f}s response on port {1}"
- },
- "warning": {
- "text": "TCP OK - {0:.3f}s response on port {1}",
- "value": 1.5
- },
- "critical": {
- "text": "Connection failed: {0} to {1}:{2}",
- "value": 5.0
- }
- }
- }
- }
- ]
- },
- "YARN": {
- "service": [
- {
- "name": "yarn_nodemanager_webui_percent",
- "label": "Percent NodeManagers Available",
- "description": "This alert is triggered if the number of down NodeManagers in the cluster is greater than the configured critical threshold. It aggregates the results of NodeManager process checks.",
- "interval": 1,
- "scope": "SERVICE",
- "enabled": true,
- "source": {
- "type": "AGGREGATE",
- "alert_name": "yarn_nodemanager_webui",
- "reporting": {
- "ok": {
- "text": "affected: [{1}], total: [{0}]"
- },
- "warning": {
- "text": "affected: [{1}], total: [{0}]",
- "value": 0.1
- },
- "critical": {
- "text": "affected: [{1}], total: [{0}]",
- "value": 0.3
- }
- }
- }
- }
- ],
- "NODEMANAGER": [
- {
- "name": "yarn_nodemanager_webui",
- "label": "NodeManager Web UI",
- "description": "This host-level alert is triggered if the NodeManager Web UI is unreachable.",
- "interval": 1,
- "scope": "HOST",
- "source": {
- "type": "WEB",
- "uri": {
- "http": "{{yarn-site/yarn.nodemanager.webapp.address}}",
- "https": "{{yarn-site/yarn.nodemanager.webapp.https.address}}",
- "https_property": "{{yarn-site/yarn.http.policy}}",
- "https_property_value": "HTTPS_ONLY",
- "default_port": 8042
- },
- "reporting": {
- "ok": {
- "text": "HTTP {0} response in {2:.3f} seconds"
- },
- "warning":{
- "text": "HTTP {0} response in {2:.3f} seconds"
- },
- "critical": {
- "text": "Connection failed to {1}"
- }
- }
- }
- },
- {
- "name": "yarn_nodemanager_health",
- "label": "NodeManager Health",
- "description": "This host-level alert checks the node health property available from the NodeManager component.",
- "interval": 1,
- "scope": "HOST",
- "enabled": true,
- "source": {
- "type": "SCRIPT",
- "path": "HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py"
- }
- }
- ],
- "RESOURCEMANAGER": [
- {
- "name": "yarn_resourcemanager_webui",
- "label": "ResourceManager Web UI",
- "description": "This host-level alert is triggered if the ResourceManager Web UI is unreachable.",
- "interval": 1,
- "scope": "ANY",
- "source": {
- "type": "WEB",
- "uri": {
- "http": "{{yarn-site/yarn.resourcemanager.webapp.address}}",
- "https": "{{yarn-site/yarn.resourcemanager.webapp.https.address}}",
- "https_property": "{{yarn-site/yarn.http.policy}}",
- "https_property_value": "HTTPS_ONLY"
- },
- "reporting": {
- "ok": {
- "text": "HTTP {0} response in {2:.3f} seconds"
- },
- "warning":{
- "text": "HTTP {0} response in {2:.3f} seconds"
- },
- "critical": {
- "text": "Connection failed to {1}"
- }
- }
- }
- },
- {
- "name": "yarn_resourcemanager_cpu",
- "label": "ResourceManager CPU Utilization",
- "description": "This host-level alert is triggered if CPU utilization of the ResourceManager exceeds certain warning and critical thresholds. It checks the ResourceManager JMX Servlet for the SystemCPULoad property. The threshold values are in percent.",
- "interval": 5,
- "scope": "ANY",
- "enabled": true,
- "source": {
- "type": "METRIC",
- "uri": {
- "http": "{{yarn-site/yarn.resourcemanager.webapp.address}}",
- "https": "{{yarn-site/yarn.resourcemanager.webapp.https.address}}",
- "https_property": "{{yarn-site/yarn.http.policy}}",
- "https_property_value": "HTTPS_ONLY"
- },
- "reporting": {
- "ok": {
- "text": "{1} CPU, load {0:.1%}"
- },
- "warning": {
- "text": "{1} CPU, load {0:.1%}",
- "value": 200
- },
- "critical": {
- "text": "{1} CPU, load {0:.1%}",
- "value": 250
- },
- "units" : "%"
- },
- "jmx": {
- "property_list": [
- "java.lang:type=OperatingSystem/SystemCpuLoad",
- "java.lang:type=OperatingSystem/AvailableProcessors"
- ],
- "value": "{0} * 100"
- }
- }
- },
- {
- "name": "yarn_resourcemanager_rpc_latency",
- "label": "ResourceManager RPC Latency",
- "description": "This host-level alert is triggered if the ResourceManager operations RPC latency exceeds the configured critical threshold. Typically an increase in the RPC processing time increases the RPC queue length, causing the average queue wait time to increase for ResourceManager operations. The threshold values are in milliseconds.",
- "interval": 5,
- "scope": "ANY",
- "enabled": true,
- "source": {
- "type": "METRIC",
- "uri": {
- "http": "{{yarn-site/yarn.resourcemanager.webapp.address}}",
- "https": "{{yarn-site/yarn.resourcemanager.webapp.https.address}}",
- "https_property": "{{yarn-site/yarn.http.policy}}",
- "https_property_value": "HTTPS_ONLY"
- },
- "reporting": {
- "ok": {
- "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]"
- },
- "warning": {
- "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
- "value": 3000
- },
- "critical": {
- "text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
- "value": 5000
- },
- "units" : "ms"
- },
- "jmx": {
- "property_list": [
- "Hadoop:service=ResourceManager,name=RpcActivityForPort*/RpcQueueTimeAvgTime",
- "Hadoop:service=ResourceManager,name=RpcActivityForPort*/RpcProcessingTimeAvgTime"
- ],
- "value": "{0}"
- }
- }
- }
- ],
- "APP_TIMELINE_SERVER": [
- {
- "name": "yarn_app_timeline_server_webui",
- "label": "App Timeline Web UI",
- "description": "This host-level alert is triggered if the App Timeline Server Web UI is unreachable.",
- "interval": 1,
- "scope": "ANY",
- "source": {
- "type": "WEB",
- "uri": {
- "http": "{{yarn-site/yarn.timeline-service.webapp.address}}",
- "https": "{{yarn-site/yarn.timeline-service.webapp.https.address}}",
- "https_property": "{{yarn-site/yarn.http.policy}}",
- "https_property_value": "HTTPS_ONLY"
- },
- "reporting": {
- "ok": {
- "text": "HTTP {0} response in {2:.3f} seconds"
- },
- "warning":{
- "text": "HTTP {0} response in {2:.3f} seconds"
- },
- "critical": {
- "text": "Connection failed to {1}"
- }
- }
- }
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/mapred-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/mapred-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/mapred-env.xml
deleted file mode 100644
index 14ae20b..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/mapred-env.xml
+++ /dev/null
@@ -1,65 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <property>
- <name>mapred_log_dir_prefix</name>
- <value>/var/log/hadoop-mapreduce</value>
- <description>Mapreduce Log Dir Prefix</description>
- </property>
- <property>
- <name>mapred_pid_dir_prefix</name>
- <value>/var/run/hadoop-mapreduce</value>
- <description>Mapreduce PID Dir Prefix</description>
- </property>
- <property>
- <name>mapred_user</name>
- <value>mapred</value>
- <property-type>USER</property-type>
- <description>Mapreduce User</description>
- </property>
- <property>
- <name>jobhistory_heapsize</name>
- <value>900</value>
- <description>Value for JobHistoryServer heap_size variable in hadoop-env.sh</description>
- </property>
-
- <!-- mapred-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for mapred-env.sh file</description>
- <value>
-# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
-
-export HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}
-
-export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
-
-#export HADOOP_JOB_HISTORYSERVER_OPTS=
-#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.
-#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.
-#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.
-#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
-#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
- </value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/mapred-site.xml
deleted file mode 100644
index 7955cb2..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/mapred-site.xml
+++ /dev/null
@@ -1,360 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration supports_final="true" xmlns:xi="http://www.w3.org/2001/XInclude">
-
-<!-- i/o properties -->
-
- <property>
- <name>mapreduce.task.io.sort.mb</name>
- <value>200</value>
- <description>
- The total amount of buffer memory to use while sorting files, in megabytes.
- By default, gives each merge stream 1MB, which should minimize seeks.
- </description>
- </property>
-
- <property>
- <name>mapreduce.map.sort.spill.percent</name>
- <value>0.7</value>
- <description>
- The soft limit in the serialization buffer. Once reached, a thread will
- begin to spill the contents to disk in the background. Note that
- collection will not block if this threshold is exceeded while a spill
- is already in progress, so spills may be larger than this threshold when
- it is set to less than .5
- </description>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.factor</name>
- <value>100</value>
- <description>
- The number of streams to merge at once while sorting files.
- This determines the number of open file handles.
- </description>
- </property>
-
-<!-- map/reduce properties -->
- <property>
- <name>mapreduce.cluster.administrators</name>
- <value> hadoop</value>
- <description>
- Administrators for MapReduce applications.
- </description>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.parallelcopies</name>
- <value>30</value>
- <description>
- The default number of parallel transfers run by reduce during
- the copy(shuffle) phase.
- </description>
- </property>
-
- <property>
- <name>mapreduce.map.speculative</name>
- <value>false</value>
- <description>
- If true, then multiple instances of some map tasks
- may be executed in parallel.
- </description>
- </property>
-
- <property>
- <name>mapreduce.reduce.speculative</name>
- <value>false</value>
- <description>
- If true, then multiple instances of some reduce tasks may be
- executed in parallel.
- </description>
- </property>
-
- <property>
- <name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>0.05</value>
- <description>
- Fraction of the number of maps in the job which should be complete before
- reduces are scheduled for the job.
- </description>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.merge.percent</name>
- <value>0.66</value>
- <description>
- The usage threshold at which an in-memory merge will be
- initiated, expressed as a percentage of the total memory allocated to
- storing in-memory map outputs, as defined by
- mapreduce.reduce.shuffle.input.buffer.percent.
- </description>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
- <value>0.7</value>
- <description>
- The percentage of memory to be allocated from the maximum heap
- size to storing map outputs during the shuffle.
- </description>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress.type</name>
- <value>BLOCK</value>
- <description>
- If the job outputs are to compressed as SequenceFiles, how should
- they be compressed? Should be one of NONE, RECORD or BLOCK.
- </description>
- </property>
-
- <property>
- <name>mapreduce.reduce.input.buffer.percent</name>
- <value>0.0</value>
- <description>
- The percentage of memory- relative to the maximum heap size- to
- retain map outputs during the reduce. When the shuffle is concluded, any
- remaining map outputs in memory must consume less than this threshold before
- the reduce can begin.
- </description>
- </property>
-
- <!-- copied from kryptonite configuration -->
- <property>
- <name>mapreduce.map.output.compress</name>
- <value>false</value>
- <description>
- Should the outputs of the maps be compressed before being sent across the network. Uses SequenceFile compression.
- </description>
- </property>
-
- <property>
- <name>mapreduce.task.timeout</name>
- <value>300000</value>
- <description>
- The number of milliseconds before a task will be
- terminated if it neither reads an input, writes an output, nor
- updates its status string.
- </description>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>1024</value>
- <description>Virtual memory for single Map task</description>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>1024</value>
- <description>Virtual memory for single Reduce task</description>
- </property>
-
- <property>
- <name>mapreduce.shuffle.port</name>
- <value>13562</value>
- <description>
- Default port that the ShuffleHandler will run on.
- ShuffleHandler is a service run at the NodeManager to facilitate
- transfers of intermediate Map outputs to requesting Reducers.
- </description>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.intermediate-done-dir</name>
- <value>/mr-history/tmp</value>
- <description>
- Directory where history files are written by MapReduce jobs.
- </description>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.done-dir</name>
- <value>/mr-history/done</value>
- <description>
- Directory where history files are managed by the MR JobHistory Server.
- </description>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.address</name>
- <value>localhost:10020</value>
- <description>Enter your JobHistoryServer hostname.</description>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.webapp.address</name>
- <value>localhost:19888</value>
- <description>Enter your JobHistoryServer hostname.</description>
- </property>
-
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
- <description>
- The runtime framework for executing MapReduce jobs. Can be one of local,
- classic or yarn.
- </description>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.staging-dir</name>
- <value>/user</value>
- <description>
- The staging dir used while submitting jobs.
- </description>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
- <description>The amount of memory the MR AppMaster needs.</description>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx312m</value>
- <description>
- Java opts for the MR App Master processes.
- The following symbol, if present, will be interpolated: @taskid@ is replaced
- by current TaskID. Any other occurrences of '@' will go unchanged.
- For example, to enable verbose gc logging to a file named for the taskid in
- /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
- -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
-
- Usage of -Djava.library.path can cause programs to no longer function if
- hadoop native libraries are used. These values should instead be set as part
- of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
- mapreduce.reduce.env config settings.
- </description>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.admin-command-opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
- <description>
- Java opts for the MR App Master processes for admin purposes.
- It will appears before the opts set by yarn.app.mapreduce.am.command-opts and
- thus its options can be overridden user.
-
- Usage of -Djava.library.path can cause programs to no longer function if
- hadoop native libraries are used. These values should instead be set as part
- of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
- mapreduce.reduce.env config settings.
- </description>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.log.level</name>
- <value>INFO</value>
- <description>MR App Master process log level.</description>
- </property>
-
- <property>
- <name>mapreduce.admin.map.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
- <description>This property stores Java options for map tasks.</description>
- </property>
-
- <property>
- <name>mapreduce.admin.reduce.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN</value>
- <description>This property stores Java options for reduce tasks.</description>
- </property>
-
- <property>
- <name>mapreduce.application.classpath</name>
- <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
- <description>
- CLASSPATH for MR applications. A comma-separated list of CLASSPATH
- entries.
- </description>
- </property>
-
- <property>
- <name>mapreduce.am.max-attempts</name>
- <value>2</value>
- <description>
- The maximum number of application attempts. It is a
- application-specific setting. It should not be larger than the global number
- set by resourcemanager. Otherwise, it will be override. The default number is
- set to 2, to allow at least one retry for AM.
- </description>
- </property>
-
-
-
- <property>
- <name>mapreduce.map.java.opts</name>
- <value>-Xmx756m</value>
- <description>
- Larger heap-size for child jvms of maps.
- </description>
- </property>
-
-
- <property>
- <name>mapreduce.reduce.java.opts</name>
- <value>-Xmx756m</value>
- <description>
- Larger heap-size for child jvms of reduces.
- </description>
- </property>
-
- <property>
- <name>mapreduce.map.log.level</name>
- <value>INFO</value>
- <description>
- The logging level for the map task. The allowed levels are:
- OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
- </description>
- </property>
-
- <property>
- <name>mapreduce.reduce.log.level</name>
- <value>INFO</value>
- <description>
- The logging level for the reduce task. The allowed levels are:
- OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
- </description>
- </property>
-
- <property>
- <name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:/usr/lib/hadoop/lib/native/Linux-amd64-64</value>
- <description>
- Additional execution environment entries for map and reduce task processes.
- This is not an additive property. You must preserve the original value if
- you want your map and reduce tasks to have access to native libraries (compression, etc)
- </description>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress</name>
- <value>false</value>
- <description>
- Should the job outputs be compressed?
- </description>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/capacity-scheduler.xml
deleted file mode 100644
index 6c21848..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/capacity-scheduler.xml
+++ /dev/null
@@ -1,132 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration supports_final="false" supports_adding_forbidden="true">
-
- <property>
- <name>yarn.scheduler.capacity.maximum-applications</name>
- <value>10000</value>
- <description>
- Maximum number of applications that can be pending and running.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
- <value>0.2</value>
- <description>
- Maximum percent of resources in the cluster which can be used to run
- application masters i.e. controls number of concurrent running
- applications.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.queues</name>
- <value>default</value>
- <description>
- The queues at the this level (root is the root queue).
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.capacity</name>
- <value>100</value>
- <description>
- The total capacity as a percentage out of 100 for this queue.
- If it has child queues then this includes their capacity as well.
- The child queues capacity should add up to their parent queue's capacity
- or less.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.capacity</name>
- <value>100</value>
- <description>Default queue target capacity.</description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
- <value>1</value>
- <description>
- Default queue user limit a percentage from 0.0 to 1.0.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
- <value>100</value>
- <description>
- The maximum capacity of the default queue.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.state</name>
- <value>RUNNING</value>
- <description>
- The state of the default queue. State can be one of RUNNING or STOPPED.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
- <value>*</value>
- <description>
- The ACL of who can submit jobs to the default queue.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
- <value>*</value>
- <description>
- The ACL of who can administer jobs on the default queue.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.acl_administer_queue</name>
- <value>*</value>
- <description>
- The ACL for who can administer this queue i.e. change sub-queue
- allocations.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.node-locality-delay</name>
- <value>40</value>
- <description>
- Number of missed scheduling opportunities after which the CapacityScheduler
- attempts to schedule rack-local containers.
- Typically this should be set to number of nodes in the cluster, By default is setting
- approximately number of nodes in one rack which is 40.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.default.minimum-user-limit-percent</name>
- <value>100</value>
- <description>
- Default minimum queue resource limit depends on the number of users who have submitted applications.
- </description>
- </property>
-
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-env.xml
deleted file mode 100644
index 1c23ade..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-env.xml
+++ /dev/null
@@ -1,179 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <property>
- <name>yarn_log_dir_prefix</name>
- <value>/var/log/hadoop-yarn</value>
- <description>YARN Log Dir Prefix</description>
- </property>
- <property>
- <name>yarn_pid_dir_prefix</name>
- <value>/var/run/hadoop-yarn</value>
- <description>YARN PID Dir Prefix</description>
- </property>
- <property>
- <name>yarn_user</name>
- <value>yarn</value>
- <property-type>USER</property-type>
- <description>YARN User</description>
- </property>
- <property>
- <name>yarn_heapsize</name>
- <value>1024</value>
- <description>Max heapsize for all YARN components using a numerical value in the scale of MB</description>
- </property>
- <property>
- <name>resourcemanager_heapsize</name>
- <value>1024</value>
- <description>Max heapsize for ResourceManager using a numerical value in the scale of MB</description>
- </property>
- <property>
- <name>nodemanager_heapsize</name>
- <value>1024</value>
- <description>Max heapsize for NodeManager using a numerical value in the scale of MB</description>
- </property>
- <property>
- <name>min_user_id</name>
- <value>1000</value>
- <description>Set to 0 to disallow root from submitting jobs. Set to 1000 to disallow all superusers from submitting jobs</description>
- </property>
-
- <!-- yarn-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for yarn-env.sh file</description>
- <value>
-export HADOOP_YARN_HOME={{hadoop_yarn_home}}
-export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER
-export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
-export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
-export JAVA_HOME={{java64_home}}
-
-# User for YARN daemons
-export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
-
-# resolve links - $0 may be a softlink
-export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
-
-# some Java parameters
-# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
-if [ "$JAVA_HOME" != "" ]; then
- #echo "run java in $JAVA_HOME"
- JAVA_HOME=$JAVA_HOME
-fi
-
-if [ "$JAVA_HOME" = "" ]; then
- echo "Error: JAVA_HOME is not set."
- exit 1
-fi
-
-JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m
-
-# For setting YARN specific HEAP sizes please use this
-# Parameter and set appropriately
-YARN_HEAPSIZE={{yarn_heapsize}}
-
-# check envvars which might override default args
-if [ "$YARN_HEAPSIZE" != "" ]; then
- JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
-fi
-
-# Resource Manager specific parameters
-
-# Specify the max Heapsize for the ResourceManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_RESOURCEMANAGER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}
-
-# Specify the JVM options to be used when starting the ResourceManager.
-# These options will be appended to the options specified as YARN_OPTS
-# and therefore may override any similar flags set in YARN_OPTS
-#export YARN_RESOURCEMANAGER_OPTS=
-
-# Node Manager specific parameters
-
-# Specify the max Heapsize for the NodeManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_NODEMANAGER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}
-
-# Specify the max Heapsize for the HistoryManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1024.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_HISTORYSERVER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-export YARN_HISTORYSERVER_HEAPSIZE={{apptimelineserver_heapsize}}
-
-# Specify the JVM options to be used when starting the NodeManager.
-# These options will be appended to the options specified as YARN_OPTS
-# and therefore may override any similar flags set in YARN_OPTS
-#export YARN_NODEMANAGER_OPTS=
-
-# so that filenames w/ spaces are handled correctly in loops below
-IFS=
-
-
-# default log directory and file
-if [ "$YARN_LOG_DIR" = "" ]; then
- YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-fi
-if [ "$YARN_LOGFILE" = "" ]; then
- YARN_LOGFILE='yarn.log'
-fi
-
-# default policy file for service-level authorization
-if [ "$YARN_POLICYFILE" = "" ]; then
- YARN_POLICYFILE="hadoop-policy.xml"
-fi
-
-# restore ordinary behaviour
-unset IFS
-
-
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
-YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
-YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
-YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
-YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
- YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
-fi
-YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml
deleted file mode 100644
index 8c44b9e..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration supports_final="false">
-
- <property>
- <name>content</name>
- <description>Custom log4j.properties</description>
- <value>
-#Relative to Yarn Log Dir Prefix
-yarn.log.dir=.
-#
-# Job Summary Appender
-#
-# Use following logger to send summary to separate file defined by
-# hadoop.mapreduce.jobsummary.log.file rolled daily:
-# hadoop.mapreduce.jobsummary.logger=INFO,JSA
-#
-hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
-hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
-log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
-# Set the ResourceManager summary log filename
-yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log
-# Set the ResourceManager summary log level and appender
-yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
-#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
-
-# To enable AppSummaryLogging for the RM,
-# set yarn.server.resourcemanager.appsummary.logger to
-# LEVEL,RMSUMMARY in hadoop-env.sh
-
-# Appender for ResourceManager Application Summary Log
-# Requires the following properties to be set
-# - hadoop.log.dir (Hadoop Log directory)
-# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
-# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
-log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
-log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
-log4j.appender.RMSUMMARY.MaxFileSize=256MB
-log4j.appender.RMSUMMARY.MaxBackupIndex=20
-log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
-log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.appender.JSA.DatePattern=.yyyy-MM-dd
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
-log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
deleted file mode 100644
index 6572935..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
+++ /dev/null
@@ -1,342 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration supports_final="true" xmlns:xi="http://www.w3.org/2001/XInclude">
-
- <!-- ResourceManager -->
-
- <property>
- <name>yarn.resourcemanager.hostname</name>
- <value>localhost</value>
- <description>The hostname of the RM.</description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>localhost:8025</value>
- <description> The address of ResourceManager. </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.address</name>
- <value>localhost:8030</value>
- <description>The address of the scheduler interface.</description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>localhost:8050</value>
- <description>
- The address of the applications manager interface in the
- RM.
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.admin.address</name>
- <value>localhost:8141</value>
- <description>The address of the RM admin interface.</description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
- <description>The class to use as the resource scheduler.</description>
- </property>
-
- <property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>512</value>
- <description>
- The minimum allocation for every container request at the RM,
- in MBs. Memory requests lower than this won't take effect,
- and the specified value will get allocated at minimum.
- </description>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>2048</value>
- <description>
- The maximum allocation for every container request at the RM,
- in MBs. Memory requests higher than this won't take effect,
- and will get capped to this value.
- </description>
- </property>
-
- <property>
- <name>yarn.acl.enable</name>
- <value>false</value>
- <description> Are acls enabled. </description>
- </property>
-
- <property>
- <name>yarn.admin.acl</name>
- <value></value>
- <description> ACL of who can be admin of the YARN cluster. </description>
- </property>
-
- <!-- NodeManager -->
-
- <property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:45454</value>
- <description>The address of the container manager in the NM.</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>5120</value>
- <description>Amount of physical memory, in MB, that can be allocated
- for containers.</description>
- </property>
-
- <property>
- <name>yarn.application.classpath</name>
- <value>/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*</value>
- <description>Classpath for typical applications.</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>2.1</value>
- <description>Ratio between virtual memory to physical memory when
- setting memory limits for containers. Container allocations are
- expressed in terms of physical memory, and virtual memory usage
- is allowed to exceed this allocation by this ratio.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
- <description>ContainerExecutor for launching containers</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
- <description>Unix group of the NodeManager</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can
- not start with numbers</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
- <description>The auxiliary service class to use </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-dirs</name>
- <value>/hadoop/yarn/log</value>
- <description>
- Where to store container logs. An application's localized log directory
- will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
- Individual containers' log directories will be below this, in directories
- named container_{$contid}. Each container directory will contain the files
- stderr, stdin, and syslog generated by that container.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.local-dirs</name>
- <value>/hadoop/yarn/local</value>
- <description>
- List of directories to store localized files in. An
- application's localized file directory will be found in:
- ${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}.
- Individual containers' work directories, called container_${contid}, will
- be subdirectories of this.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-monitor.interval-ms</name>
- <value>3000</value>
- <description>
- The interval, in milliseconds, for which the node manager
- waits between two cycles of monitoring its containers' memory usage.
- </description>
- </property>
-
- <!--
- <property>
- <name>yarn.nodemanager.health-checker.script.path</name>
- <value>/etc/hadoop/conf/health_check_nodemanager</value>
- <description>The health check script to run.</description>
- </property>
- -->
-
- <property>
- <name>yarn.nodemanager.health-checker.interval-ms</name>
- <value>135000</value>
- <description>Frequency of running node health script.</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>60000</value>
- <description>Script time out period.</description>
- </property>
-
- <property>
- <name>yarn.nodemanager.log.retain-second</name>
- <value>604800</value>
- <description>
- Time in seconds to retain user logs. Only applicable if
- log aggregation is disabled.
- </description>
- </property>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- <description>Whether to enable log aggregation. </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
- <description>Location to aggregate logs to. </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
- <value>logs</value>
- <description>
- The remote log dir will be created at
- {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.compression-type</name>
- <value>gz</value>
- <description>
- T-file compression types used to compress aggregated logs.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>0</value>
- <description>
- Number of seconds after an application finishes before the nodemanager's
- DeletionService will delete the application's localized file directory
- and log directory.
-
- To diagnose Yarn application problems, set this property's value large
- enough (for example, to 600 = 10 minutes) to permit examination of these
- directories. After changing the property's value, you must restart the
- nodemanager in order for it to have an effect.
-
- The roots of Yarn applications' work directories is configurable with
- the yarn.nodemanager.local-dirs property (see below), and the roots
- of the Yarn applications' log directories is configurable with the
- yarn.nodemanager.log-dirs property (see also below).
- </description>
- </property>
-
- <property>
- <name>yarn.log-aggregation.retain-seconds</name>
- <value>2592000</value>
- <description>
- How long to keep aggregation logs before deleting them. -1 disables.
- Be careful set this too small and you will spam the name node.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.admin-env</name>
- <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
- <description>
- Environment variables that should be forwarded from the NodeManager's
- environment to the container's.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
- <value>0.25</value>
- <description>
- The minimum fraction of number of disks to be healthy for the nodemanager
- to launch new containers. This correspond to both
- yarn-nodemanager.local-dirs and yarn.nodemanager.log-dirs. i.e.
- If there are less number of healthy local-dirs (or log-dirs) available,
- then new containers will not be launched on this node.
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.am.max-attempts</name>
- <value>2</value>
- <description>
- The maximum number of application attempts. It's a global
- setting for all application masters. Each application master can specify
- its individual maximum number of application attempts via the API, but the
- individual number cannot be more than the global upper bound. If it is,
- the resourcemanager will override it. The default number is set to 2, to
- allow at least one retry for AM.
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.webapp.address</name>
- <value>localhost:8088</value>
- <description>
- The address of the RM web application.
- </description>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- <description>
- Whether virtual memory limits will be enforced for containers.
- </description>
- </property>
-
- <property>
- <name>yarn.log.server.url</name>
- <value>http://localhost:19888/jobhistory/logs</value>
- <description>
- URI for the HistoryServer's log resource
- </description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.nodes.exclude-path</name>
- <value>/etc/hadoop/conf/yarn.exclude</value>
- <description>
- Names a file that contains a list of hosts that are
- not permitted to connect to the resource manager. The full pathname of the
- file must be specified. If the value is empty, no hosts are
- excluded.
- </description>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/metainfo.xml
index 9c4e4ea..7a1c715 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/metainfo.xml
@@ -21,222 +21,11 @@
<services>
<service>
<name>YARN</name>
- <displayName>YARN</displayName>
- <comment>Apache Hadoop NextGen MapReduce (YARN)</comment>
- <version>2.1.0.2.0</version>
- <components>
-
- <component>
- <name>RESOURCEMANAGER</name>
- <displayName>ResourceManager</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <commandScript>
- <script>scripts/resourcemanager.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>1200</timeout>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>DECOMMISSION</name>
- <commandScript>
- <script>scripts/resourcemanager.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- <customCommand>
- <name>REFRESHQUEUES</name>
- <commandScript>
- <script>scripts/resourcemanager.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
- <configuration-dependencies>
- <config-type>capacity-scheduler</config-type>
- </configuration-dependencies>
- </component>
-
- <component>
- <name>NODEMANAGER</name>
- <displayName>NodeManager</displayName>
- <category>SLAVE</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/nodemanager.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>1200</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>YARN_CLIENT</name>
- <displayName>YARN Client</displayName>
- <category>CLIENT</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/yarn_client.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>1200</timeout>
- </commandScript>
- <configFiles>
- <configFile>
- <type>xml</type>
- <fileName>yarn-site.xml</fileName>
- <dictionaryName>yarn-site</dictionaryName>
- </configFile>
- <configFile>
- <type>xml</type>
- <fileName>core-site.xml</fileName>
- <dictionaryName>core-site</dictionaryName>
- </configFile>
- <configFile>
- <type>env</type>
- <fileName>yarn-env.sh</fileName>
- <dictionaryName>yarn-env</dictionaryName>
- </configFile>
- <configFile>
- <type>env</type>
- <fileName>log4j.properties</fileName>
- <dictionaryName>hdfs-log4j,yarn-log4j</dictionaryName>
- </configFile>
- <configFile>
- <type>xml</type>
- <fileName>capacity-scheduler.xml</fileName>
- <dictionaryName>capacity-scheduler</dictionaryName>
- </configFile>
- </configFiles>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>hadoop-yarn</name>
- </package>
- <package>
- <name>hadoop-hdfs</name>
- </package>
- <package>
- <name>hadoop-mapreduce</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>yarn-site</config-type>
- <config-type>yarn-env</config-type>
- <config-type>core-site</config-type>
- <config-type>mapred-site</config-type>
- <config-type>yarn-log4j</config-type>
- </configuration-dependencies>
+ <extends>common-services/YARN/2.1.0.2.0</extends>
</service>
-
<service>
<name>MAPREDUCE2</name>
- <displayName>MapReduce2</displayName>
- <comment>Apache Hadoop NextGen MapReduce (YARN)</comment>
- <version>2.1.0.2.0.6.0</version>
- <components>
- <component>
- <name>HISTORYSERVER</name>
- <displayName>History Server</displayName>
- <category>MASTER</category>
- <cardinality>1</cardinality>
- <auto-deploy>
- <enabled>true</enabled>
- <co-locate>YARN/RESOURCEMANAGER</co-locate>
- </auto-deploy>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/historyserver.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>1200</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>MAPREDUCE2_CLIENT</name>
- <displayName>MapReduce2 Client</displayName>
- <category>CLIENT</category>
- <cardinality>0+</cardinality>
- <commandScript>
- <script>scripts/mapreduce2_client.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>1200</timeout>
- </commandScript>
- <configFiles>
- <configFile>
- <type>xml</type>
- <fileName>mapred-site.xml</fileName>
- <dictionaryName>mapred-site</dictionaryName>
- </configFile>
- <configFile>
- <type>xml</type>
- <fileName>core-site.xml</fileName>
- <dictionaryName>core-site</dictionaryName>
- </configFile>
- <configFile>
- <type>env</type>
- <fileName>mapred-env.sh</fileName>
- <dictionaryName>mapred-env</dictionaryName>
- </configFile>
- </configFiles>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>hadoop-mapreduce</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandScript>
- <script>scripts/mapred_service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>YARN</service>
- </requiredServices>
-
- <configuration-dir>configuration-mapred</configuration-dir>
-
- <configuration-dependencies>
- <config-type>core-site</config-type>
- <config-type>mapred-site</config-type>
- <config-type>mapred-env</config-type>
- </configuration-dependencies>
+ <extends>common-services/MAPREDUCE2/2.1.0.2.0.6.0</extends>
</service>
-
</services>
</metainfo>
[3/8] ambari git commit: AMBARI-8878: Common Services: Refactor
HDP-2.0.6 YARN service (Jayush Luniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py
deleted file mode 100644
index b1de951..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/alerts/alert_nodemanager_health.py
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import json
-import socket
-import urllib2
-
-RESULT_CODE_OK = 'OK'
-RESULT_CODE_CRITICAL = 'CRITICAL'
-RESULT_CODE_UNKNOWN = 'UNKNOWN'
-
-NODEMANAGER_HTTP_ADDRESS_KEY = '{{yarn-site/yarn.nodemanager.webapp.address}}'
-NODEMANAGER_HTTPS_ADDRESS_KEY = '{{yarn-site/yarn.nodemanager.webapp.https.address}}'
-YARN_HTTP_POLICY_KEY = '{{yarn-site/yarn.http.policy}}'
-
-OK_MESSAGE = 'NodeManager Healthy'
-CRITICAL_CONNECTION_MESSAGE = 'Connection failed to {0}'
-CRITICAL_NODEMANAGER_STATUS_MESSAGE = 'NodeManager returned an unexpected status of "{0}"'
-CRITICAL_NODEMANAGER_UNKNOWN_JSON_MESSAGE = 'Unable to determine NodeManager health from unexpected JSON response'
-
-NODEMANAGER_DEFAULT_PORT = 8042
-
-def get_tokens():
- """
- Returns a tuple of tokens in the format {{site/property}} that will be used
- to build the dictionary passed into execute
- """
- return (NODEMANAGER_HTTP_ADDRESS_KEY,NODEMANAGER_HTTPS_ADDRESS_KEY,
- YARN_HTTP_POLICY_KEY)
-
-
-def execute(parameters=None, host_name=None):
- """
- Returns a tuple containing the result code and a pre-formatted result label
-
- Keyword arguments:
- parameters (dictionary): a mapping of parameter key to value
- host_name (string): the name of this host where the alert is running
- """
- result_code = RESULT_CODE_UNKNOWN
-
- if parameters is None:
- return (result_code, ['There were no parameters supplied to the script.'])
-
- scheme = 'http'
- http_uri = None
- https_uri = None
- http_policy = 'HTTP_ONLY'
-
- if NODEMANAGER_HTTP_ADDRESS_KEY in parameters:
- http_uri = parameters[NODEMANAGER_HTTP_ADDRESS_KEY]
-
- if NODEMANAGER_HTTPS_ADDRESS_KEY in parameters:
- https_uri = parameters[NODEMANAGER_HTTPS_ADDRESS_KEY]
-
- if YARN_HTTP_POLICY_KEY in parameters:
- http_policy = parameters[YARN_HTTP_POLICY_KEY]
-
- # determine the right URI and whether to use SSL
- uri = http_uri
- if http_policy == 'HTTPS_ONLY':
- scheme = 'https'
-
- if https_uri is not None:
- uri = https_uri
-
- label = ''
- url_response = None
- node_healthy = 'false'
- total_time = 0
-
- # some yarn-site structures don't have the web ui address
- if uri is None:
- if host_name is None:
- host_name = socket.getfqdn()
-
- uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT)
-
- try:
- query = "{0}://{1}/ws/v1/node/info".format(scheme,uri)
-
- # execute the query for the JSON that includes templeton status
- url_response = urllib2.urlopen(query)
- except:
- label = CRITICAL_CONNECTION_MESSAGE.format(uri)
- return (RESULT_CODE_CRITICAL, [label])
-
- # URL response received, parse it
- try:
- json_response = json.loads(url_response.read())
- node_healthy = json_response['nodeInfo']['nodeHealthy']
-
- # convert boolean to string
- node_healthy = str(node_healthy)
- except:
- return (RESULT_CODE_CRITICAL, [query])
-
- # proper JSON received, compare against known value
- if node_healthy.lower() == 'true':
- result_code = RESULT_CODE_OK
- label = OK_MESSAGE
- else:
- result_code = RESULT_CODE_CRITICAL
- label = CRITICAL_NODEMANAGER_STATUS_MESSAGE.format(node_healthy)
-
- return (result_code, [label])
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/files/validateYarnComponentStatus.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/files/validateYarnComponentStatus.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/files/validateYarnComponentStatus.py
deleted file mode 100644
index 33ed8b1..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/files/validateYarnComponentStatus.py
+++ /dev/null
@@ -1,170 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import optparse
-import subprocess
-import json
-
-RESOURCEMANAGER = 'rm'
-NODEMANAGER = 'nm'
-HISTORYSERVER = 'hs'
-
-STARTED_STATE = 'STARTED'
-RUNNING_STATE = 'RUNNING'
-
-#Return reponse for given path and address
-def getResponse(path, address, ssl_enabled):
-
- command = "curl"
- httpGssnegotiate = "--negotiate"
- userpswd = "-u:"
- insecure = "-k"# This is smoke test, no need to check CA of server
- if ssl_enabled:
- url = 'https://' + address + path
- else:
- url = 'http://' + address + path
-
- command_with_flags = [command,httpGssnegotiate,userpswd,insecure,url]
-
- proc = subprocess.Popen(command_with_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (stdout, stderr) = proc.communicate()
- response = json.loads(stdout)
- if response == None:
- print 'There is no response for url: ' + str(url)
- raise Exception('There is no response for url: ' + str(url))
- return response
-
-#Verify that REST api is available for given component
-def validateAvailability(component, path, addresses, ssl_enabled):
- responses = {}
- for address in addresses.split(','):
- try:
- responses[address] = getResponse(path, address, ssl_enabled)
- except Exception as e:
- print 'Error checking availability status of component.', e
-
- if not responses:
- exit(1)
-
- is_valid = validateAvailabilityResponse(component, responses.values()[0])
- if not is_valid:
- exit(1)
-
-#Validate component-specific response
-def validateAvailabilityResponse(component, response):
- try:
- if component == RESOURCEMANAGER:
- rm_state = response['clusterInfo']['state']
- if rm_state == STARTED_STATE:
- return True
- else:
- print 'Resourcemanager is not started'
- return False
-
- elif component == NODEMANAGER:
- node_healthy = bool(response['nodeInfo']['nodeHealthy'])
- if node_healthy:
- return True
- else:
- return False
- elif component == HISTORYSERVER:
- hs_start_time = response['historyInfo']['startedOn']
- if hs_start_time > 0:
- return True
- else:
- return False
- else:
- return False
- except Exception as e:
- print 'Error validation of availability response for ' + str(component), e
- return False
-
-#Verify that component has required resources to work
-def validateAbility(component, path, addresses, ssl_enabled):
- responses = {}
- for address in addresses.split(','):
- try:
- responses[address] = getResponse(path, address, ssl_enabled)
- except Exception as e:
- print 'Error checking ability of component.', e
-
- if not responses:
- exit(1)
-
- is_valid = validateAbilityResponse(component, responses.values()[0])
- if not is_valid:
- exit(1)
-
-#Validate component-specific response that it has required resources to work
-def validateAbilityResponse(component, response):
- try:
- if component == RESOURCEMANAGER:
- nodes = []
- if response.has_key('nodes') and not response['nodes'] == None and response['nodes'].has_key('node'):
- nodes = response['nodes']['node']
- connected_nodes_count = len(nodes)
- if connected_nodes_count == 0:
- print 'There is no connected nodemanagers to resourcemanager'
- return False
- active_nodes = filter(lambda x: x['state'] == RUNNING_STATE, nodes)
- active_nodes_count = len(active_nodes)
-
- if connected_nodes_count == 0:
- print 'There is no connected active nodemanagers to resourcemanager'
- return False
- else:
- return True
- else:
- return False
- except Exception as e:
- print 'Error validation of ability response', e
- return False
-
-#
-# Main.
-#
-def main():
- parser = optparse.OptionParser(usage="usage: %prog [options] component ")
- parser.add_option("-p", "--port", dest="address", help="Host:Port for REST API of a desired component")
- parser.add_option("-s", "--ssl", dest="ssl_enabled", help="Is SSL enabled for UI of component")
-
- (options, args) = parser.parse_args()
-
- component = args[0]
-
- address = options.address
- ssl_enabled = (options.ssl_enabled) in 'true'
- if component == RESOURCEMANAGER:
- path = '/ws/v1/cluster/info'
- elif component == NODEMANAGER:
- path = '/ws/v1/node/info'
- elif component == HISTORYSERVER:
- path = '/ws/v1/history/info'
- else:
- parser.error("Invalid component")
-
- validateAvailability(component, path, address, ssl_enabled)
-
- if component == RESOURCEMANAGER:
- path = '/ws/v1/cluster/nodes'
- validateAbility(component, path, address, ssl_enabled)
-
-if __name__ == "__main__":
- main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/__init__.py
deleted file mode 100644
index 35de4bb..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/application_timeline_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/application_timeline_server.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/application_timeline_server.py
deleted file mode 100644
index e9c0fa1..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/application_timeline_server.py
+++ /dev/null
@@ -1,132 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
-from resource_management.libraries.functions.security_commons import build_expectations, \
- cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties,\
- FILE_TYPE_XML
-from resource_management.libraries.functions.format import format
-
-from yarn import yarn
-from service import service
-
-class ApplicationTimelineServer(Script):
-
- def install(self, env):
- self.install_packages(env)
- #self.configure(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- yarn(name='apptimelineserver')
-
- def pre_rolling_restart(self, env):
- Logger.info("Executing Rolling Upgrade pre-restart")
- import params
- env.set_params(params)
-
- if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
- Execute(format("hdp-select set hadoop-yarn-timelineserver {version}"))
-
- def start(self, env, rolling_restart=False):
- import params
- env.set_params(params)
- self.configure(env) # FOR SECURITY
- service('timelineserver', action='start')
-
- def stop(self, env, rolling_restart=False):
- import params
- env.set_params(params)
- service('timelineserver', action='stop')
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- Execute(format("mv {yarn_historyserver_pid_file_old} {yarn_historyserver_pid_file}"),
- only_if = format("test -e {yarn_historyserver_pid_file_old}", user=status_params.yarn_user))
- functions.check_process_status(status_params.yarn_historyserver_pid_file)
-
- def security_status(self, env):
- import status_params
- env.set_params(status_params)
-
- props_value_check = {"yarn.timeline-service.enabled": "true",
- "yarn.timeline-service.http-authentication.type": "kerberos",
- "yarn.acl.enable": "true"}
- props_empty_check = ["yarn.timeline-service.principal",
- "yarn.timeline-service.keytab",
- "yarn.timeline-service.http-authentication.kerberos.principal",
- "yarn.timeline-service.http-authentication.kerberos.keytab"]
-
- props_read_check = ["yarn.timeline-service.keytab",
- "yarn.timeline-service.http-authentication.kerberos.keytab"]
- yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
- props_read_check)
-
- yarn_expectations ={}
- yarn_expectations.update(yarn_site_props)
-
- security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
- {'yarn-site.xml': FILE_TYPE_XML})
- result_issues = validate_security_config_properties(security_params, yarn_expectations)
- if not result_issues: # If all validations passed successfully
- try:
- # Double check the dict before calling execute
- if ( 'yarn-site' not in security_params
- or 'yarn.timeline-service.keytab' not in security_params['yarn-site']
- or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \
- or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \
- or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']:
- self.put_structured_out({"securityState": "UNSECURED"})
- self.put_structured_out(
- {"securityIssuesFound": "Keytab file or principal are not set property."})
- return
-
- cached_kinit_executor(status_params.kinit_path_local,
- status_params.yarn_user,
- security_params['yarn-site']['yarn.timeline-service.keytab'],
- security_params['yarn-site']['yarn.timeline-service.principal'],
- status_params.hostname,
- status_params.tmp_dir,
- 30)
- cached_kinit_executor(status_params.kinit_path_local,
- status_params.yarn_user,
- security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'],
- security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'],
- status_params.hostname,
- status_params.tmp_dir,
- 30)
- self.put_structured_out({"securityState": "SECURED_KERBEROS"})
- except Exception as e:
- self.put_structured_out({"securityState": "ERROR"})
- self.put_structured_out({"securityStateErrorInfo": str(e)})
- else:
- issues = []
- for cf in result_issues:
- issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
- self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
- self.put_structured_out({"securityState": "UNSECURED"})
-
-
-if __name__ == "__main__":
- ApplicationTimelineServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py
deleted file mode 100644
index 66c2143..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/historyserver.py
+++ /dev/null
@@ -1,123 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
-from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions.security_commons import build_expectations, \
- cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
- FILE_TYPE_XML
-
-from yarn import yarn
-from service import service
-
-class HistoryServer(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- yarn(name="historyserver")
-
- def pre_rolling_restart(self, env):
- Logger.info("Executing Rolling Upgrade pre-restart")
- import params
- env.set_params(params)
-
- if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
- Execute(format("hdp-select set hadoop-mapreduce-historyserver {version}"))
- copy_tarballs_to_hdfs('mapreduce', params.mapred_user, params.hdfs_user, params.user_group)
-
- def start(self, env, rolling_restart=False):
- import params
- env.set_params(params)
- self.configure(env) # FOR SECURITY
- copy_tarballs_to_hdfs('mapreduce', params.mapred_user, params.hdfs_user, params.user_group)
- service('historyserver', action='start', serviceName='mapreduce')
-
- def stop(self, env, rolling_restart=False):
- import params
- env.set_params(params)
- service('historyserver', action='stop', serviceName='mapreduce')
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- check_process_status(status_params.mapred_historyserver_pid_file)
-
- def security_status(self, env):
- import status_params
- env.set_params(status_params)
-
- expectations = {}
- expectations.update(build_expectations('mapred-site',
- None,
- [
- 'mapreduce.jobhistory.keytab',
- 'mapreduce.jobhistory.principal',
- 'mapreduce.jobhistory.webapp.spnego-keytab-file',
- 'mapreduce.jobhistory.webapp.spnego-principal'
- ],
- None))
-
- security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
- {'mapred-site.xml': FILE_TYPE_XML})
- result_issues = validate_security_config_properties(security_params, expectations)
- if not result_issues: # If all validations passed successfully
- try:
- # Double check the dict before calling execute
- if ( 'mapred-site' not in security_params or
- 'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or
- 'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or
- 'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or
- 'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']):
- self.put_structured_out({"securityState": "UNSECURED"})
- self.put_structured_out(
- {"securityIssuesFound": "Keytab file or principal not set."})
- return
-
- cached_kinit_executor(status_params.kinit_path_local,
- status_params.mapred_user,
- security_params['mapred-site']['mapreduce.jobhistory.keytab'],
- security_params['mapred-site']['mapreduce.jobhistory.principal'],
- status_params.hostname,
- status_params.tmp_dir)
- cached_kinit_executor(status_params.kinit_path_local,
- status_params.mapred_user,
- security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'],
- security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'],
- status_params.hostname,
- status_params.tmp_dir)
- self.put_structured_out({"securityState": "SECURED_KERBEROS"})
- except Exception as e:
- self.put_structured_out({"securityState": "ERROR"})
- self.put_structured_out({"securityStateErrorInfo": str(e)})
- else:
- issues = []
- for cf in result_issues:
- issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
- self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
- self.put_structured_out({"securityState": "UNSECURED"})
-
-if __name__ == "__main__":
- HistoryServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapred_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapred_service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapred_service_check.py
deleted file mode 100644
index a9f4367..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapred_service_check.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-
-class MapReduce2ServiceCheck(Script):
- def service_check(self, env):
- import params
- env.set_params(params)
-
- jar_path = format("{hadoop_mapred2_jar_location}/{hadoopMapredExamplesJarName}")
- input_file = format("/user/{smokeuser}/mapredsmokeinput")
- output_file = format("/user/{smokeuser}/mapredsmokeoutput")
-
- cleanup_cmd = format("fs -rm -r -f {output_file} {input_file}")
- create_file_cmd = format("fs -put /etc/passwd {input_file}")
- test_cmd = format("fs -test -e {output_file}")
- run_wordcount_job = format("jar {jar_path} wordcount {input_file} {output_file}")
-
- if params.security_enabled:
- kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};")
-
- Execute(kinit_cmd,
- user=params.smokeuser
- )
-
- ExecuteHadoop(cleanup_cmd,
- tries=1,
- try_sleep=5,
- user=params.smokeuser,
- bin_dir=params.execute_path,
- conf_dir=params.hadoop_conf_dir
- )
-
- ExecuteHadoop(create_file_cmd,
- tries=1,
- try_sleep=5,
- user=params.smokeuser,
- bin_dir=params.execute_path,
- conf_dir=params.hadoop_conf_dir
- )
-
- ExecuteHadoop(run_wordcount_job,
- tries=1,
- try_sleep=5,
- user=params.smokeuser,
- bin_dir=params.execute_path,
- conf_dir=params.hadoop_conf_dir,
- logoutput=True
- )
-
- ExecuteHadoop(test_cmd,
- user=params.smokeuser,
- bin_dir=params.execute_path,
- conf_dir=params.hadoop_conf_dir
- )
-
-if __name__ == "__main__":
- MapReduce2ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapreduce2_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapreduce2_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapreduce2_client.py
deleted file mode 100644
index c4e2800..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/mapreduce2_client.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import sys
-from resource_management import *
-
-from yarn import yarn
-
-class MapReduce2Client(Script):
-
- def pre_rolling_restart(self, env):
- import params
- env.set_params(params)
-
- if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
- Execute(format("hdp-select set hadoop-client {version}"))
-
- def install(self, env):
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- yarn()
-
- def status(self, env):
- raise ClientComponentHasNoStatus()
-
-if __name__ == "__main__":
- MapReduce2Client().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py
deleted file mode 100644
index 87be214..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py
+++ /dev/null
@@ -1,137 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import nodemanager_upgrade
-
-from resource_management import *
-from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions.security_commons import build_expectations, \
- cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
- FILE_TYPE_XML
-
-from yarn import yarn
-from service import service
-
-class Nodemanager(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- yarn(name="nodemanager")
-
- def pre_rolling_restart(self, env):
- Logger.info("Executing NodeManager Rolling Upgrade pre-restart")
- import params
- env.set_params(params)
-
- if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
- Execute(format("hdp-select set hadoop-yarn-nodemanager {version}"))
-
- def start(self, env, rolling_restart=False):
- import params
- env.set_params(params)
- self.configure(env) # FOR SECURITY
- service('nodemanager',action='start')
-
- def post_rolling_restart(self, env):
- Logger.info("Executing NodeManager Rolling Upgrade post-restart")
- import params
- env.set_params(params)
-
- nodemanager_upgrade.post_upgrade_check()
-
- def stop(self, env, rolling_restart=False):
- import params
- env.set_params(params)
-
- service('nodemanager',action='stop')
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- check_process_status(status_params.nodemanager_pid_file)
-
- def security_status(self, env):
- import status_params
- env.set_params(status_params)
-
- props_value_check = {"yarn.timeline-service.enabled": "true",
- "yarn.timeline-service.http-authentication.type": "kerberos",
- "yarn.acl.enable": "true"}
- props_empty_check = ["yarn.nodemanager.principal",
- "yarn.nodemanager.keytab",
- "yarn.nodemanager.webapp.spnego-principal",
- "yarn.nodemanager.webapp.spnego-keytab-file"]
-
- props_read_check = ["yarn.nodemanager.keytab",
- "yarn.nodemanager.webapp.spnego-keytab-file"]
- yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
- props_read_check)
-
- yarn_expectations ={}
- yarn_expectations.update(yarn_site_props)
-
- security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
- {'yarn-site.xml': FILE_TYPE_XML})
- result_issues = validate_security_config_properties(security_params, yarn_site_props)
- if not result_issues: # If all validations passed successfully
- try:
- # Double check the dict before calling execute
- if ( 'yarn-site' not in security_params
- or 'yarn.nodemanager.keytab' not in security_params['yarn-site']
- or 'yarn.nodemanager.principal' not in security_params['yarn-site']) \
- or 'yarn.nodemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \
- or 'yarn.nodemanager.webapp.spnego-principal' not in security_params['yarn-site']:
- self.put_structured_out({"securityState": "UNSECURED"})
- self.put_structured_out(
- {"securityIssuesFound": "Keytab file or principal are not set property."})
- return
-
- cached_kinit_executor(status_params.kinit_path_local,
- status_params.yarn_user,
- security_params['yarn-site']['yarn.nodemanager.keytab'],
- security_params['yarn-site']['yarn.nodemanager.principal'],
- status_params.hostname,
- status_params.tmp_dir,
- 30)
- cached_kinit_executor(status_params.kinit_path_local,
- status_params.yarn_user,
- security_params['yarn-site']['yarn.nodemanager.webapp.spnego-keytab-file'],
- security_params['yarn-site']['yarn.nodemanager.webapp.spnego-principal'],
- status_params.hostname,
- status_params.tmp_dir,
- 30)
- self.put_structured_out({"securityState": "SECURED_KERBEROS"})
- except Exception as e:
- self.put_structured_out({"securityState": "ERROR"})
- self.put_structured_out({"securityStateErrorInfo": str(e)})
- else:
- issues = []
- for cf in result_issues:
- issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
- self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
- self.put_structured_out({"securityState": "UNSECURED"})
-
-if __name__ == "__main__":
- Nodemanager().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager_upgrade.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager_upgrade.py
deleted file mode 100644
index 54e0fae..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager_upgrade.py
+++ /dev/null
@@ -1,74 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import subprocess
-
-from resource_management.core.logger import Logger
-from resource_management.core.exceptions import Fail
-from resource_management.core.resources.system import Execute
-from resource_management.core import shell
-from resource_management.libraries.functions.decorator import retry
-
-
-def post_upgrade_check():
- '''
- Checks that the NodeManager has rejoined the cluster.
- This function will obtain the Kerberos ticket if security is enabled.
- :return:
- '''
- import params
-
- Logger.info('NodeManager executing "yarn node -list -states=RUNNING" to verify the node has rejoined the cluster...')
- if params.security_enabled and params.nodemanager_kinit_cmd:
- Execute(params.nodemanager_kinit_cmd, user = params.yarn_user)
-
- _check_nodemanager_startup()
-
-
-@retry(times=12, sleep_time=10, err_class=Fail)
-def _check_nodemanager_startup():
- '''
- Checks that a NodeManager is in a RUNNING state in the cluster via
- "yarn node -list -states=RUNNING" command. Once the NodeManager is found to be
- alive this method will return, otherwise it will raise a Fail(...) and retry
- automatically.
- :return:
- '''
- import params
-
- command = 'yarn node -list -states=RUNNING'
-
- try:
- # 'su - yarn -c "yarn node -status c6401.ambari.apache.org:45454"'
- return_code, yarn_output = shell.call(command, user=params.hdfs_user)
- except:
- raise Fail('Unable to determine if the NodeManager has started after upgrade.')
-
- if return_code == 0:
- hostname = params.hostname.lower()
- nodemanager_address = params.nm_address.lower()
- yarn_output = yarn_output.lower()
-
- if hostname in yarn_output or nodemanager_address in yarn_output:
- Logger.info('NodeManager with ID {0} has rejoined the cluster.'.format(nodemanager_address))
- return
- else:
- raise Fail('NodeManager with ID {0} was not found in the list of running NodeManagers'.format(nodemanager_address))
-
- raise Fail('Unable to determine if the NodeManager has started after upgrade (result code {0})'.format(str(return_code)))
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py
deleted file mode 100644
index d9c73bf..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py
+++ /dev/null
@@ -1,209 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-import os
-from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
-from resource_management.libraries.functions.default import default
-from resource_management import *
-import status_params
-
-# server configurations
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-# This is expected to be of the form #.#.#.#
-stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
-
-# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
-version = default("/commandParams/version", None)
-
-hostname = config['hostname']
-
-#hadoop params
-if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
- yarn_role_root = "hadoop-yarn-client"
- mapred_role_root = "hadoop-mapreduce-client"
-
- command_role = default("/role", "")
- if command_role == "APP_TIMELINE_SERVER":
- yarn_role_root = "hadoop-yarn-timelineserver"
- elif command_role == "HISTORYSERVER":
- mapred_role_root = "hadoop-mapreduce-historyserver"
- elif command_role == "MAPREDUCE2_CLIENT":
- mapred_role_root = "hadoop-mapreduce-client"
- elif command_role == "NODEMANAGER":
- yarn_role_root = "hadoop-yarn-nodemanager"
- elif command_role == "RESOURCEMANAGER":
- yarn_role_root = "hadoop-yarn-resourcemanager"
- elif command_role == "YARN_CLIENT":
- yarn_role_root = "hadoop-yarn-client"
-
- hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
- hadoop_bin = "/usr/hdp/current/hadoop-client/sbin"
- hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
-
- hadoop_mapred2_jar_location = format("/usr/hdp/current/{mapred_role_root}")
- mapred_bin = format("/usr/hdp/current/{mapred_role_root}/sbin")
-
- hadoop_yarn_home = format("/usr/hdp/current/{yarn_role_root}")
- yarn_bin = format("/usr/hdp/current/{yarn_role_root}/sbin")
- yarn_container_bin = format("/usr/hdp/current/{yarn_role_root}/bin")
-else:
- hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
- hadoop_bin = "/usr/lib/hadoop/sbin"
- hadoop_bin_dir = "/usr/bin"
- hadoop_yarn_home = '/usr/lib/hadoop-yarn'
- hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce"
- mapred_bin = "/usr/lib/hadoop-mapreduce/sbin"
- yarn_bin = "/usr/lib/hadoop-yarn/sbin"
- yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
-
-hadoop_conf_dir = "/etc/hadoop/conf"
-limits_conf_dir = "/etc/security/limits.d"
-execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir + os.pathsep + yarn_container_bin
-
-ulimit_cmd = "ulimit -c unlimited;"
-
-mapred_user = status_params.mapred_user
-yarn_user = status_params.yarn_user
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-
-smokeuser = config['configurations']['cluster-env']['smokeuser']
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group']
-kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-rm_hosts = config['clusterHostInfo']['rm_host']
-rm_host = rm_hosts[0]
-rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1]
-rm_https_port = "8090"
-# TODO UPGRADE default, update site during upgrade
-rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
-
-java64_home = config['hostLevelParams']['java_home']
-hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False)
-
-yarn_heapsize = config['configurations']['yarn-env']['yarn_heapsize']
-resourcemanager_heapsize = config['configurations']['yarn-env']['resourcemanager_heapsize']
-nodemanager_heapsize = config['configurations']['yarn-env']['nodemanager_heapsize']
-apptimelineserver_heapsize = default("/configurations/yarn-env/apptimelineserver_heapsize", 1024)
-ats_leveldb_dir = config['configurations']['yarn-site']['yarn.timeline-service.leveldb-timeline-store.path']
-yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix']
-yarn_pid_dir_prefix = status_params.yarn_pid_dir_prefix
-mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix
-mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix']
-mapred_env_sh_template = config['configurations']['mapred-env']['content']
-yarn_env_sh_template = config['configurations']['yarn-env']['content']
-
-if len(rm_hosts) > 1:
- additional_rm_host = rm_hosts[1]
- rm_webui_address = format("{rm_host}:{rm_port},{additional_rm_host}:{rm_port}")
- rm_webui_https_address = format("{rm_host}:{rm_https_port},{additional_rm_host}:{rm_https_port}")
-else:
- rm_webui_address = format("{rm_host}:{rm_port}")
- rm_webui_https_address = format("{rm_host}:{rm_https_port}")
-
-nm_webui_address = config['configurations']['yarn-site']['yarn.nodemanager.webapp.address']
-hs_webui_address = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address']
-nm_address = config['configurations']['yarn-site']['yarn.nodemanager.address'] # still contains 0.0.0.0
-if hostname and nm_address and nm_address.startswith("0.0.0.0:"):
- nm_address = nm_address.replace("0.0.0.0", hostname)
-
-nm_local_dirs = config['configurations']['yarn-site']['yarn.nodemanager.local-dirs']
-nm_log_dirs = config['configurations']['yarn-site']['yarn.nodemanager.log-dirs']
-
-distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar"
-hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
-
-yarn_pid_dir = status_params.yarn_pid_dir
-mapred_pid_dir = status_params.mapred_pid_dir
-
-mapred_log_dir = format("{mapred_log_dir_prefix}/{mapred_user}")
-yarn_log_dir = format("{yarn_log_dir_prefix}/{yarn_user}")
-mapred_job_summary_log = format("{mapred_log_dir_prefix}/{mapred_user}/hadoop-mapreduce.jobsummary.log")
-yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduce.jobsummary.log")
-
-user_group = config['configurations']['cluster-env']['user_group']
-
-#exclude file
-exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
-exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
-
-ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
-has_ats = not len(ats_host) == 0
-
-# default kinit commands
-rm_kinit_cmd = ""
-yarn_timelineservice_kinit_cmd = ""
-nodemanager_kinit_cmd = ""
-
-if security_enabled:
- _rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal']
- _rm_principal_name = _rm_principal_name.replace('_HOST',hostname.lower())
- _rm_keytab = config['configurations']['yarn-site']['yarn.resourcemanager.keytab']
- rm_kinit_cmd = format("{kinit_path_local} -kt {_rm_keytab} {_rm_principal_name};")
-
- # YARN timeline security options are only available in HDP Champlain
- if has_ats:
- _yarn_timelineservice_principal_name = config['configurations']['yarn-site']['yarn.timeline-service.principal']
- _yarn_timelineservice_principal_name = _yarn_timelineservice_principal_name.replace('_HOST', hostname.lower())
- _yarn_timelineservice_keytab = config['configurations']['yarn-site']['yarn.timeline-service.keytab']
- yarn_timelineservice_kinit_cmd = format("{kinit_path_local} -kt {_yarn_timelineservice_keytab} {_yarn_timelineservice_principal_name};")
-
- if 'yarn.nodemanager.principal' in config['configurations']['yarn-site']:
- _nodemanager_principal_name = config['configurations']['yarn-site']['yarn.nodemanager.principal']
- _nodemanager_keytab = config['configurations']['yarn-site']['yarn.nodemanager.keytab']
- nodemanager_kinit_cmd = format("{kinit_path_local} -kt {_nodemanager_keytab} {_nodemanager_principal_name};")
-
-
-yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable']
-yarn_nm_app_log_dir = config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir']
-mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir']
-mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir']
-jobhistory_heapsize = default("/configurations/mapred-env/jobhistory_heapsize", "900")
-
-#for create_hdfs_directory
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
- HdfsDirectory,
- conf_dir=hadoop_conf_dir,
- hdfs_user=hdfs_user,
- security_enabled = security_enabled,
- keytab = hdfs_user_keytab,
- kinit_path_local = kinit_path_local,
- bin_dir = hadoop_bin_dir
-)
-update_exclude_file_only = default("/commandParams/update_exclude_file_only",False)
-
-mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group)
-
-#taskcontroller.cfg
-
-mapred_local_dir = "/tmp/hadoop-mapred/mapred/local"
-hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
-min_user_id = config['configurations']['yarn-env']['min_user_id']
-
-# Node labels
-node_labels_dir = default("/configurations/yarn-site/yarn.node-labels.fs-store.root-dir", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py
deleted file mode 100644
index 3d6a00e..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py
+++ /dev/null
@@ -1,173 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
-from resource_management.libraries.functions.security_commons import build_expectations, \
- cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
- FILE_TYPE_XML
-
-from yarn import yarn
-from service import service
-
-
-class Resourcemanager(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env):
- import params
-
- env.set_params(params)
- yarn(name='resourcemanager')
-
- def pre_rolling_restart(self, env):
- Logger.info("Executing Rolling Upgrade post-restart")
- import params
- env.set_params(params)
-
- if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
- Execute(format("hdp-select set hadoop-yarn-resourcemanager {version}"))
-
- def start(self, env, rolling_restart=False):
- import params
-
- env.set_params(params)
- self.configure(env) # FOR SECURITY
- service('resourcemanager',
- action='start'
- )
-
- def stop(self, env, rolling_restart=False):
- import params
-
- env.set_params(params)
-
- service('resourcemanager',
- action='stop'
- )
-
- def status(self, env):
- import status_params
-
- env.set_params(status_params)
- check_process_status(status_params.resourcemanager_pid_file)
- pass
-
- def security_status(self, env):
- import status_params
- env.set_params(status_params)
-
- props_value_check = {"yarn.timeline-service.enabled": "true",
- "yarn.timeline-service.http-authentication.type": "kerberos",
- "yarn.acl.enable": "true"}
- props_empty_check = ["yarn.resourcemanager.principal",
- "yarn.resourcemanager.keytab",
- "yarn.resourcemanager.webapp.spnego-principal",
- "yarn.resourcemanager.webapp.spnego-keytab-file"]
-
- props_read_check = ["yarn.resourcemanager.keytab",
- "yarn.resourcemanager.webapp.spnego-keytab-file"]
- yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
- props_read_check)
-
- yarn_expectations ={}
- yarn_expectations.update(yarn_site_props)
-
- security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
- {'yarn-site.xml': FILE_TYPE_XML})
- result_issues = validate_security_config_properties(security_params, yarn_site_props)
- if not result_issues: # If all validations passed successfully
- try:
- # Double check the dict before calling execute
- if ( 'yarn-site' not in security_params
- or 'yarn.resourcemanager.keytab' not in security_params['yarn-site']
- or 'yarn.resourcemanager.principal' not in security_params['yarn-site']) \
- or 'yarn.resourcemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \
- or 'yarn.resourcemanager.webapp.spnego-principal' not in security_params['yarn-site']:
- self.put_structured_out({"securityState": "UNSECURED"})
- self.put_structured_out(
- {"securityIssuesFound": "Keytab file or principal are not set property."})
- return
-
- cached_kinit_executor(status_params.kinit_path_local,
- status_params.yarn_user,
- security_params['yarn-site']['yarn.resourcemanager.keytab'],
- security_params['yarn-site']['yarn.resourcemanager.principal'],
- status_params.hostname,
- status_params.tmp_dir,
- 30)
- cached_kinit_executor(status_params.kinit_path_local,
- status_params.yarn_user,
- security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-keytab-file'],
- security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-principal'],
- status_params.hostname,
- status_params.tmp_dir,
- 30)
- self.put_structured_out({"securityState": "SECURED_KERBEROS"})
- except Exception as e:
- self.put_structured_out({"securityState": "ERROR"})
- self.put_structured_out({"securityStateErrorInfo": str(e)})
- else:
- issues = []
- for cf in result_issues:
- issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
- self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
- self.put_structured_out({"securityState": "UNSECURED"})
-
-
- def refreshqueues(self, env):
- import params
-
- self.configure(env)
- env.set_params(params)
-
- service('resourcemanager',
- action='refreshQueues'
- )
-
- def decommission(self, env):
- import params
-
- env.set_params(params)
- rm_kinit_cmd = params.rm_kinit_cmd
- yarn_user = params.yarn_user
- conf_dir = params.hadoop_conf_dir
- user_group = params.user_group
-
- yarn_refresh_cmd = format("{rm_kinit_cmd} yarn --config {conf_dir} rmadmin -refreshNodes")
-
- File(params.exclude_file_path,
- content=Template("exclude_hosts_list.j2"),
- owner=yarn_user,
- group=user_group
- )
-
- if params.update_exclude_file_only == False:
- Execute(yarn_refresh_cmd,
- environment= {'PATH' : params.execute_path },
- user=yarn_user)
- pass
- pass
-
-
-if __name__ == "__main__":
- Resourcemanager().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service.py
deleted file mode 100644
index 1002094..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-
-
-def service(componentName, action='start', serviceName='yarn'):
-
- import params
-
- if serviceName == 'mapreduce' and componentName == 'historyserver':
- daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh")
- pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid")
- usr = params.mapred_user
- else:
- daemon = format("{yarn_bin}/yarn-daemon.sh")
- pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid")
- usr = params.yarn_user
-
- cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}")
-
- if action == 'start':
- daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}")
- check_process = format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1")
-
- # Remove the pid file if its corresponding process is not running.
- File(pid_file,
- action="delete",
- not_if=check_process)
-
- # Attempt to start the process. Internally, this is skipped if the process is already running.
- Execute(daemon_cmd,
- user=usr,
- not_if=check_process
- )
-
- # Ensure that the process with the expected PID exists.
- Execute(check_process,
- user=usr,
- not_if=check_process,
- initial_wait=5
- )
-
- elif action == 'stop':
- daemon_cmd = format("{cmd} stop {componentName}")
- Execute(daemon_cmd,
- user=usr)
-
- File(pid_file,
- action="delete")
-
- elif action == 'refreshQueues':
- rm_kinit_cmd = params.rm_kinit_cmd
- refresh_cmd = format("{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues")
-
- Execute(refresh_cmd,
- user=usr,
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service_check.py
deleted file mode 100644
index 7189664..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/service_check.py
+++ /dev/null
@@ -1,68 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-import sys
-
-class ServiceCheck(Script):
- def service_check(self, env):
- import params
- env.set_params(params)
-
- run_yarn_check_cmd = format("yarn --config {hadoop_conf_dir} node -list")
-
- component_type = 'rm'
- if params.hadoop_ssl_enabled:
- component_address = params.rm_webui_https_address
- else:
- component_address = params.rm_webui_address
-
- validateStatusFileName = "validateYarnComponentStatus.py"
- validateStatusFilePath = format("{tmp_dir}/{validateStatusFileName}")
- python_executable = sys.executable
- validateStatusCmd = format("{python_executable} {validateStatusFilePath} {component_type} -p {component_address} -s {hadoop_ssl_enabled}")
-
- if params.security_enabled:
- kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};")
- smoke_cmd = format("{kinit_cmd} {validateStatusCmd}")
- else:
- smoke_cmd = validateStatusCmd
-
- File(validateStatusFilePath,
- content=StaticFile(validateStatusFileName),
- mode=0755
- )
-
- Execute(smoke_cmd,
- tries=3,
- try_sleep=5,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=params.smokeuser,
- logoutput=True
- )
-
- Execute(run_yarn_check_cmd,
- path=params.execute_path,
- user=params.smokeuser
- )
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py
deleted file mode 100644
index 3cfff05..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/status_params.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-mapred_user = config['configurations']['mapred-env']['mapred_user']
-yarn_user = config['configurations']['yarn-env']['yarn_user']
-yarn_pid_dir_prefix = config['configurations']['yarn-env']['yarn_pid_dir_prefix']
-mapred_pid_dir_prefix = config['configurations']['mapred-env']['mapred_pid_dir_prefix']
-yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}")
-mapred_pid_dir = format("{mapred_pid_dir_prefix}/{mapred_user}")
-
-resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanager.pid")
-nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid")
-yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid")
-yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated
-mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid")
-
-# Security related/required params
-hadoop_conf_dir = "/etc/hadoop/conf"
-hostname = config['hostname']
-kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-security_enabled = config['configurations']['cluster-env']['security_enabled']
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
deleted file mode 100644
index cf0d211..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
+++ /dev/null
@@ -1,249 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-import sys
-import os
-
-
-def yarn(name = None):
- import params
-
-
- if name in ["nodemanager","historyserver"]:
- if params.yarn_log_aggregation_enabled:
- params.HdfsDirectory(params.yarn_nm_app_log_dir,
- action="create_delayed",
- owner=params.yarn_user,
- group=params.user_group,
- mode=0777,
- recursive_chmod=True
- )
- params.HdfsDirectory("/mapred",
- action="create_delayed",
- owner=params.mapred_user
- )
- params.HdfsDirectory("/mapred/system",
- action="create_delayed",
- owner=params.hdfs_user
- )
- params.HdfsDirectory(params.mapreduce_jobhistory_intermediate_done_dir,
- action="create_delayed",
- owner=params.mapred_user,
- group=params.user_group,
- mode=0777
- )
-
- params.HdfsDirectory(params.mapreduce_jobhistory_done_dir,
- action="create_delayed",
- owner=params.mapred_user,
- group=params.user_group,
- mode=01777
- )
- params.HdfsDirectory(None, action="create")
-
- if name == "nodemanager":
- Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
- owner=params.yarn_user,
- group=params.user_group,
- recursive=True,
- recursive_permission=True,
- ignore_failures=True,
- mode=0775
- )
-
- Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
- owner=params.yarn_user,
- group=params.user_group,
- recursive=True
- )
-
- Directory([params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir],
- owner=params.mapred_user,
- group=params.user_group,
- recursive=True
- )
- Directory([params.yarn_log_dir_prefix],
- owner=params.yarn_user,
- recursive=True,
- ignore_failures=True,
- )
-
- XmlConfig("core-site.xml",
- conf_dir=params.hadoop_conf_dir,
- configurations=params.config['configurations']['core-site'],
- configuration_attributes=params.config['configuration_attributes']['core-site'],
- owner=params.hdfs_user,
- group=params.user_group,
- mode=0644
- )
-
- XmlConfig("mapred-site.xml",
- conf_dir=params.hadoop_conf_dir,
- configurations=params.config['configurations']['mapred-site'],
- configuration_attributes=params.config['configuration_attributes']['mapred-site'],
- owner=params.yarn_user,
- group=params.user_group,
- mode=0644
- )
-
- XmlConfig("yarn-site.xml",
- conf_dir=params.hadoop_conf_dir,
- configurations=params.config['configurations']['yarn-site'],
- configuration_attributes=params.config['configuration_attributes']['yarn-site'],
- owner=params.yarn_user,
- group=params.user_group,
- mode=0644
- )
-
- XmlConfig("capacity-scheduler.xml",
- conf_dir=params.hadoop_conf_dir,
- configurations=params.config['configurations']['capacity-scheduler'],
- configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
- owner=params.yarn_user,
- group=params.user_group,
- mode=0644
- )
-
- if name == 'resourcemanager':
- File(params.yarn_job_summary_log,
- owner=params.yarn_user,
- group=params.user_group
- )
- if params.node_labels_dir:
- params.HdfsDirectory(params.node_labels_dir,
- action="create",
- owner=params.yarn_user,
- group=params.user_group,
- mode=0700
- )
- elif name == 'apptimelineserver':
- Directory(params.ats_leveldb_dir,
- owner=params.yarn_user,
- group=params.user_group,
- recursive=True,
- recursive_permission=True
- )
-
- File(params.rm_nodes_exclude_path,
- owner=params.yarn_user,
- group=params.user_group
- )
-
- File(format("{limits_conf_dir}/yarn.conf"),
- mode=0644,
- content=Template('yarn.conf.j2')
- )
-
- File(format("{limits_conf_dir}/mapreduce.conf"),
- mode=0644,
- content=Template('mapreduce.conf.j2')
- )
-
- File(format("{hadoop_conf_dir}/yarn-env.sh"),
- owner=params.yarn_user,
- group=params.user_group,
- mode=0755,
- content=InlineTemplate(params.yarn_env_sh_template)
- )
-
- if params.security_enabled:
- container_executor = format("{yarn_container_bin}/container-executor")
- File(container_executor,
- group=params.yarn_executor_container_group,
- mode=06050
- )
-
- File(format("{hadoop_conf_dir}/container-executor.cfg"),
- group=params.user_group,
- mode=0644,
- content=Template('container-executor.cfg.j2')
- )
-
-
- if params.security_enabled:
- tc_mode = 0644
- tc_owner = "root"
- else:
- tc_mode = None
- tc_owner = params.hdfs_user
-
- File(format("{hadoop_conf_dir}/mapred-env.sh"),
- owner=tc_owner,
- content=InlineTemplate(params.mapred_env_sh_template)
- )
-
- if params.security_enabled:
- File(os.path.join(params.hadoop_bin, "task-controller"),
- owner="root",
- group=params.mapred_tt_group,
- mode=06050
- )
- File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'),
- owner = tc_owner,
- mode = tc_mode,
- group = params.mapred_tt_group,
- content=Template("taskcontroller.cfg.j2")
- )
- else:
- File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'),
- owner=tc_owner,
- content=Template("taskcontroller.cfg.j2")
- )
-
- if "mapred-site" in params.config['configurations']:
- XmlConfig("mapred-site.xml",
- conf_dir=params.hadoop_conf_dir,
- configurations=params.config['configurations']['mapred-site'],
- configuration_attributes=params.config['configuration_attributes']['mapred-site'],
- owner=params.mapred_user,
- group=params.user_group
- )
-
- if "capacity-scheduler" in params.config['configurations']:
- XmlConfig("capacity-scheduler.xml",
- conf_dir=params.hadoop_conf_dir,
- configurations=params.config['configurations'][
- 'capacity-scheduler'],
- configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
- owner=params.hdfs_user,
- group=params.user_group
- )
-
- if os.path.exists(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml')):
- File(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml'),
- owner=params.mapred_user,
- group=params.user_group
- )
-
- if os.path.exists(
- os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example')):
- File(os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example'),
- owner=params.mapred_user,
- group=params.user_group
- )
-
- if os.path.exists(
- os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example')):
- File(os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example'),
- owner=params.mapred_user,
- group=params.user_group
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn_client.py
deleted file mode 100644
index 60b9bce..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn_client.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import sys
-from resource_management import *
-
-from yarn import yarn
-
-class YarnClient(Script):
-
- def pre_rolling_restart(self, env):
- import params
- env.set_params(params)
-
- if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
- Execute(format("hdp-select set hadoop-client {version}"))
-
- def install(self, env):
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- yarn()
-
- def status(self, env):
- raise ClientComponentHasNoStatus()
-
-if __name__ == "__main__":
- YarnClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/container-executor.cfg.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/container-executor.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/container-executor.cfg.j2
deleted file mode 100644
index c6f1ff6..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/container-executor.cfg.j2
+++ /dev/null
@@ -1,40 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-#/*
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-yarn.nodemanager.local-dirs={{nm_local_dirs}}
-yarn.nodemanager.log-dirs={{nm_log_dirs}}
-yarn.nodemanager.linux-container-executor.group={{yarn_executor_container_group}}
-banned.users=hdfs,yarn,mapred,bin
-min.user.id={{min_user_id}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/exclude_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/exclude_hosts_list.j2
deleted file mode 100644
index c7ce416..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/exclude_hosts_list.j2
+++ /dev/null
@@ -1,21 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-{% for host in exclude_hosts %}
-{{host}}
-{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapreduce.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapreduce.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapreduce.conf.j2
deleted file mode 100644
index b996645..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapreduce.conf.j2
+++ /dev/null
@@ -1,35 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{{mapred_user}} - nofile 32768
-{{mapred_user}} - nproc 65536
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/taskcontroller.cfg.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/taskcontroller.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/taskcontroller.cfg.j2
deleted file mode 100644
index 3d5f4f2..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/taskcontroller.cfg.j2
+++ /dev/null
@@ -1,38 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-#/*
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-mapred.local.dir={{mapred_local_dir}}
-mapreduce.tasktracker.group={{mapred_tt_group}}
-hadoop.log.dir={{hdfs_log_dir_prefix}}/{{mapred_user}}
[7/8] ambari git commit: AMBARI-8878: Common Services: Refactor
HDP-2.0.6 YARN service (Jayush Luniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metrics.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metrics.json b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metrics.json
new file mode 100644
index 0000000..5d8309d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metrics.json
@@ -0,0 +1,5360 @@
+{
+ "NODEMANAGER": {
+ "Component": [
+ {
+ "type": "ganglia",
+ "metrics": {
+ "metrics/memory/mem_total": {
+ "metric": "mem_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/memHeapCommittedM": {
+ "metric": "jvm.JvmMetrics.MemHeapCommittedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/mapred/ShuffleOutputsFailed": {
+ "metric": "mapred.ShuffleOutputsFailed",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsRunnable": {
+ "metric": "jvm.JvmMetrics.ThreadsRunnable",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsNew": {
+ "metric": "jvm.JvmMetrics.ThreadsNew",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthorizationFailures": {
+ "metric": "rpc.metrics.RpcAuthorizationFailures",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginSuccess_avg_time": {
+ "metric": "ugi.ugi.LoginSuccessAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcQueueTime_avg_time": {
+ "metric": "rpc.rpc.RpcQueueTimeAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersCompleted": {
+ "metric": "yarn.NodeManagerMetrics.ContainersCompleted",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/SentBytes": {
+ "metric": "rpc.rpc.SentBytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memNonHeapUsedM": {
+ "metric": "jvm.JvmMetrics.MemNonHeapUsedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersKilled": {
+ "metric": "yarn.NodeManagerMetrics.ContainersKilled",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/logWarn": {
+ "metric": "jvm.JvmMetrics.LogWarn",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsTimedWaiting": {
+ "metric": "jvm.JvmMetrics.ThreadsTimedWaiting",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcCount": {
+ "metric": "jvm.JvmMetrics.GcCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/process/proc_run": {
+ "metric": "proc_run",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/swap_total": {
+ "metric": "swap_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/ReceivedBytes": {
+ "metric": "rpc.rpc.ReceivedBytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_nice": {
+ "metric": "cpu_nice",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsBlocked": {
+ "metric": "jvm.JvmMetrics.ThreadsBlocked",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcQueueTime_num_ops": {
+ "metric": "rpc.rpc.RpcQueueTimeNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/process/proc_total": {
+ "metric": "proc_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/AllocatedGB": {
+ "metric": "yarn.NodeManagerMetrics.AllocatedGB",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/disk/part_max_used": {
+ "metric": "part_max_used",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/NumOpenConnections": {
+ "metric": "rpc.rpc.NumOpenConnections",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memHeapUsedM": {
+ "metric": "jvm.JvmMetrics.MemHeapUsedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsWaiting": {
+ "metric": "jvm.JvmMetrics.ThreadsWaiting",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/disk/disk_free": {
+ "metric": "disk_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/mapred/ShuffleOutputsOK": {
+ "metric": "mapred.ShuffleOutputsOK",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersFailed": {
+ "metric": "yarn.NodeManagerMetrics.ContainersFailed",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_buffers": {
+ "metric": "mem_buffers",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/ugi/loginSuccess_num_ops": {
+ "metric": "ugi.ugi.LoginSuccessNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcTimeMillis": {
+ "metric": "jvm.JvmMetrics.GcTimeMillis",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_idle": {
+ "metric": "cpu_idle",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/AllocatedContainers": {
+ "metric": "yarn.NodeManagerMetrics.AllocatedContainers",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsTerminated": {
+ "metric": "jvm.JvmMetrics.ThreadsTerminated",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/network/bytes_out": {
+ "metric": "bytes_out",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_aidle": {
+ "metric": "cpu_aidle",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_free": {
+ "metric": "mem_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_user": {
+ "metric": "cpu_user",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/swap_free": {
+ "metric": "swap_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_system": {
+ "metric": "cpu_system",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/bytes_in": {
+ "metric": "bytes_in",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/boottime": {
+ "metric": "boottime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/pkts_out": {
+ "metric": "pkts_out",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/memNonHeapCommittedM": {
+ "metric": "jvm.JvmMetrics.MemNonHeapCommittedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/callQueueLen": {
+ "metric": "rpc.rpc.CallQueueLength",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/memory/mem_cached": {
+ "metric": "mem_cached",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersRunning": {
+ "metric": "yarn.NodeManagerMetrics.ContainersRunning",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/disk/disk_total": {
+ "metric": "disk_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/logInfo": {
+ "metric": "jvm.JvmMetrics.LogInfo",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersLaunched": {
+ "metric": "yarn.NodeManagerMetrics.ContainersLaunched",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/RpcProcessingTime_num_ops": {
+ "metric": "rpc.rpc.RpcProcessingTimeNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginFailure_num_ops": {
+ "metric": "ugi.ugi.LoginFailureNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/memory/mem_shared": {
+ "metric": "mem_shared",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/AvailableGB": {
+ "metric": "yarn.NodeManagerMetrics.AvailableGB",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/mapred/ShuffleConnections": {
+ "metric": "mapred.ShuffleConnections",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_wio": {
+ "metric": "cpu_wio",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersIniting": {
+ "metric": "yarn.NodeManagerMetrics.ContainersIniting",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/logError": {
+ "metric": "jvm.JvmMetrics.LogError",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginFailure_avg_time": {
+ "metric": "ugi.ugi.LoginFailureAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_num": {
+ "metric": "cpu_num",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_speed": {
+ "metric": "cpu_speed",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthorizationSuccesses": {
+ "metric": "rpc.rpc.RpcAuthorizationSuccesses",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/logFatal": {
+ "metric": "jvm.JvmMetrics.LogFatal",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcProcessingTime_avg_time": {
+ "metric": "rpc.rpc.RpcProcessingTimeAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/mapred/ShuffleOutputBytes": {
+ "metric": "mapred.ShuffleOutputBytes",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthenticationSuccesses": {
+ "metric": "rpc.metrics.RpcAuthenticationSuccesses",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthenticationFailures": {
+ "metric": "rpc.metrics.RpcAuthenticationFailures",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/network/pkts_in": {
+ "metric": "pkts_in",
+ "pointInTime": true,
+ "temporal": true
+ }
+ }
+ }
+ ],
+ "HostComponent": [
+ {
+ "type": "ganglia",
+ "metrics": {
+ "metrics/memory/mem_total": {
+ "metric": "mem_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/memHeapCommittedM": {
+ "metric": "jvm.JvmMetrics.MemHeapCommittedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/mapred/ShuffleOutputsFailed": {
+ "metric": "mapred.ShuffleOutputsFailed",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsRunnable": {
+ "metric": "jvm.JvmMetrics.ThreadsRunnable",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsNew": {
+ "metric": "jvm.JvmMetrics.ThreadsNew",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthorizationFailures": {
+ "metric": "rpc.metrics.RpcAuthorizationFailures",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginSuccess_avg_time": {
+ "metric": "ugi.ugi.LoginSuccessAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcQueueTime_avg_time": {
+ "metric": "rpc.rpc.RpcQueueTimeAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersCompleted": {
+ "metric": "yarn.NodeManagerMetrics.ContainersCompleted",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/SentBytes": {
+ "metric": "rpc.rpc.SentBytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memNonHeapUsedM": {
+ "metric": "jvm.JvmMetrics.MemNonHeapUsedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersKilled": {
+ "metric": "yarn.NodeManagerMetrics.ContainersKilled",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/logWarn": {
+ "metric": "jvm.JvmMetrics.LogWarn",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsTimedWaiting": {
+ "metric": "jvm.JvmMetrics.ThreadsTimedWaiting",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcCount": {
+ "metric": "jvm.JvmMetrics.GcCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/process/proc_run": {
+ "metric": "proc_run",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/swap_total": {
+ "metric": "swap_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/ReceivedBytes": {
+ "metric": "rpc.rpc.ReceivedBytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_nice": {
+ "metric": "cpu_nice",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsBlocked": {
+ "metric": "jvm.JvmMetrics.ThreadsBlocked",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcQueueTime_num_ops": {
+ "metric": "rpc.rpc.RpcQueueTimeNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/process/proc_total": {
+ "metric": "proc_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/AllocatedGB": {
+ "metric": "yarn.NodeManagerMetrics.AllocatedGB",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/disk/part_max_used": {
+ "metric": "part_max_used",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/NumOpenConnections": {
+ "metric": "rpc.rpc.NumOpenConnections",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memHeapUsedM": {
+ "metric": "jvm.JvmMetrics.MemHeapUsedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsWaiting": {
+ "metric": "jvm.JvmMetrics.ThreadsWaiting",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/disk/disk_free": {
+ "metric": "disk_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/mapred/ShuffleOutputsOK": {
+ "metric": "mapred.ShuffleOutputsOK",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersFailed": {
+ "metric": "yarn.NodeManagerMetrics.ContainersFailed",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_buffers": {
+ "metric": "mem_buffers",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/ugi/loginSuccess_num_ops": {
+ "metric": "ugi.ugi.LoginSuccessNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcTimeMillis": {
+ "metric": "jvm.JvmMetrics.GcTimeMillis",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_idle": {
+ "metric": "cpu_idle",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/AllocatedContainers": {
+ "metric": "yarn.NodeManagerMetrics.AllocatedContainers",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsTerminated": {
+ "metric": "jvm.JvmMetrics.ThreadsTerminated",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/network/bytes_out": {
+ "metric": "bytes_out",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_aidle": {
+ "metric": "cpu_aidle",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_free": {
+ "metric": "mem_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_user": {
+ "metric": "cpu_user",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/swap_free": {
+ "metric": "swap_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_system": {
+ "metric": "cpu_system",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/bytes_in": {
+ "metric": "bytes_in",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/boottime": {
+ "metric": "boottime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/pkts_out": {
+ "metric": "pkts_out",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/memNonHeapCommittedM": {
+ "metric": "jvm.JvmMetrics.MemNonHeapCommittedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/callQueueLen": {
+ "metric": "rpc.rpc.CallQueueLength",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/memory/mem_cached": {
+ "metric": "mem_cached",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersRunning": {
+ "metric": "yarn.NodeManagerMetrics.ContainersRunning",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/disk/disk_total": {
+ "metric": "disk_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/logInfo": {
+ "metric": "jvm.JvmMetrics.LogInfo",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersLaunched": {
+ "metric": "yarn.NodeManagerMetrics.ContainersLaunched",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/RpcProcessingTime_num_ops": {
+ "metric": "rpc.rpc.RpcProcessingTimeNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginFailure_num_ops": {
+ "metric": "ugi.ugi.LoginFailureNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/memory/mem_shared": {
+ "metric": "mem_shared",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/AvailableGB": {
+ "metric": "yarn.NodeManagerMetrics.AvailableGB",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/mapred/ShuffleConnections": {
+ "metric": "mapred.ShuffleConnections",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_wio": {
+ "metric": "cpu_wio",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/ContainersIniting": {
+ "metric": "yarn.NodeManagerMetrics.ContainersIniting",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/logError": {
+ "metric": "jvm.JvmMetrics.LogError",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginFailure_avg_time": {
+ "metric": "ugi.ugi.LoginFailureAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_num": {
+ "metric": "cpu_num",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_speed": {
+ "metric": "cpu_speed",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthorizationSuccesses": {
+ "metric": "rpc.rpc.RpcAuthorizationSuccesses",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/logFatal": {
+ "metric": "jvm.JvmMetrics.LogFatal",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcProcessingTime_avg_time": {
+ "metric": "rpc.rpc.RpcProcessingTimeAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/mapred/ShuffleOutputBytes": {
+ "metric": "mapred.ShuffleOutputBytes",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthenticationSuccesses": {
+ "metric": "rpc.metrics.RpcAuthenticationSuccesses",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthenticationFailures": {
+ "metric": "rpc.metrics.RpcAuthenticationFailures",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/network/pkts_in": {
+ "metric": "pkts_in",
+ "pointInTime": true,
+ "temporal": true
+ }
+ }
+ },
+ {
+ "type": "jmx",
+ "metrics": {
+ "metrics/jvm/memHeapCommittedM": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.MemHeapCommittedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsRunnable": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsRunnable",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsNew": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsNew",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memNonHeapCommittedM": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.MemNonHeapCommittedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/callQueueLen": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.CallQueueLength",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/rpcAuthorizationFailures": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcAuthorizationFailures",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/RpcQueueTime_avg_time": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcQueueTimeAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/ugi/loginSuccess_avg_time": {
+ "metric": "Hadoop:service=NodeManager,name=UgiMetrics.LoginSuccessAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/SentBytes": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.SentBytes",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/logInfo": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.LogInfo",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memNonHeapUsedM": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.MemNonHeapUsedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/logWarn": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.LogWarn",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/ugi/loginFailure_num_ops": {
+ "metric": "Hadoop:service=NodeManager,name=UgiMetrics.LoginFailureNumOps",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsTimedWaiting": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsTimedWaiting",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/RpcProcessingTime_num_ops": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcProcessingTimeNumOps",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/gcCount": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.GcCount",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/ReceivedBytes": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.ReceivedBytes",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsBlocked": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsBlocked",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/RpcQueueTime_num_ops": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcQueueTimeNumOps",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/logError": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.LogError",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/ugi/loginFailure_avg_time": {
+ "metric": "Hadoop:service=NodeManager,name=UgiMetrics.LoginFailureAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/rpcAuthorizationSuccesses": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcAuthorizationSuccesses",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/NumOpenConnections": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.NumOpenConnections",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memHeapUsedM": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.MemHeapUsedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/logFatal": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.LogFatal",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/RpcProcessingTime_avg_time": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcProcessingTimeAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsWaiting": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsWaiting",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/gcTimeMillis": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.GcTimeMillis",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/ugi/loginSuccess_num_ops": {
+ "metric": "Hadoop:service=NodeManager,name=UgiMetrics.LoginSuccessNumOps",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/rpcAuthenticationSuccesses": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcAuthenticationSuccesses",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/rpcAuthenticationFailures": {
+ "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcAuthenticationFailures",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsTerminated": {
+ "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsTerminated",
+ "pointInTime": true,
+ "temporal": false
+ }
+ }
+ }
+ ]
+ },
+ "RESOURCEMANAGER": {
+ "Component": [
+ {
+ "type": "ganglia",
+ "metrics": {
+ "metrics/rpcdetailed/FinishApplicationMasterNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.FinishApplicationMasterNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_total": {
+ "metric": "mem_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsCompleted": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsCompleted",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumUnhealthyNMs": {
+ "metric": "yarn.ClusterMetrics.NumUnhealthyNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memHeapCommittedM": {
+ "metric": "jvm.JvmMetrics.MemHeapCommittedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsRunnable": {
+ "metric": "jvm.JvmMetrics.ThreadsRunnable",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumRebootedNMs": {
+ "metric": "yarn.ClusterMetrics.NumRebootedNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsNew": {
+ "metric": "jvm.JvmMetrics.ThreadsNew",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsSubmitted": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsSubmitted",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumLostNMs": {
+ "metric": "yarn.ClusterMetrics.NumLostNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginSuccess_avg_time": {
+ "metric": "ugi.ugi.LoginSuccessAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcQueueTime_avg_time": {
+ "metric": "rpc.rpc.RpcQueueTimeAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AllocatedContainers",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/SentBytes": {
+ "metric": "rpc.rpc.SentBytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsKilled": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsKilled",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumActiveNMs": {
+ "metric": "yarn.ClusterMetrics.NumActiveNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memNonHeapUsedM": {
+ "metric": "jvm.JvmMetrics.MemNonHeapUsedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/logWarn": {
+ "metric": "jvm.JvmMetrics.LogWarn",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsTimedWaiting": {
+ "metric": "jvm.JvmMetrics.ThreadsTimedWaiting",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsFailed": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsFailed",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcCount": {
+ "metric": "jvm.JvmMetrics.GcCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/process/proc_run": {
+ "metric": "proc_run",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/swap_total": {
+ "metric": "swap_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/ReceivedBytes": {
+ "metric": "rpc.rpc.ReceivedBytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/AllocateNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.AllocateNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_nice": {
+ "metric": "cpu_nice",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/gcCountMarkSweepCompact": {
+ "metric": "jvm.JvmMetrics.GcCountMarkSweepCompact",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsBlocked": {
+ "metric": "jvm.JvmMetrics.ThreadsBlocked",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsRunning": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsRunning",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumDecommissionedNMs": {
+ "metric": "yarn.ClusterMetrics.NumDecommissionedNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcQueueTime_num_ops": {
+ "metric": "rpc.rpc.RpcQueueTimeNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/process/proc_total": {
+ "metric": "proc_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/disk/part_max_used": {
+ "metric": "part_max_used",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/NumOpenConnections": {
+ "metric": "rpc.rpc.NumOpenConnections",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memHeapUsedM": {
+ "metric": "jvm.JvmMetrics.MemHeapUsedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/disk/disk_free": {
+ "metric": "disk_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsWaiting": {
+ "metric": "jvm.JvmMetrics.ThreadsWaiting",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/load_one": {
+ "metric": "load_one",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_buffers": {
+ "metric": "mem_buffers",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/ugi/loginSuccess_num_ops": {
+ "metric": "ugi.ugi.LoginSuccessNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcTimeMillisCopy": {
+ "metric": "jvm.JvmMetrics.GcTimeMillisCopy",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/gcTimeMillis": {
+ "metric": "jvm.JvmMetrics.GcTimeMillis",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_idle": {
+ "metric": "cpu_idle",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).PendingContainers",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memMaxM": {
+ "metric": "jvm.JvmMetrics.MemMaxM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsTerminated": {
+ "metric": "jvm.JvmMetrics.ThreadsTerminated",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/network/bytes_out": {
+ "metric": "bytes_out",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_aidle": {
+ "metric": "cpu_aidle",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_free": {
+ "metric": "mem_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/AllocateAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.AllocateAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_user": {
+ "metric": "cpu_user",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/swap_free": {
+ "metric": "swap_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_system": {
+ "metric": "cpu_system",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/load_five": {
+ "metric": "load_five",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/bytes_in": {
+ "metric": "bytes_in",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/boottime": {
+ "metric": "boottime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/pkts_out": {
+ "metric": "pkts_out",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/memNonHeapCommittedM": {
+ "metric": "jvm.JvmMetrics.MemNonHeapCommittedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/GetApplicationReportNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.GetApplicationReportNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/FinishApplicationMasterAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.FinishApplicationMasterAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/callQueueLen": {
+ "metric": "rpc.rpc.CallQueueLength",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/memory/mem_cached": {
+ "metric": "mem_cached",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/RegisterApplicationMasterNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.RegisterApplicationMasterNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/disk/disk_total": {
+ "metric": "disk_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AvailableMB",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).PendingMB",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/logInfo": {
+ "metric": "jvm.JvmMetrics.LogInfo",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcProcessingTime_num_ops": {
+ "metric": "rpc.rpc.RpcProcessingTimeNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginFailure_num_ops": {
+ "metric": "ugi.ugi.LoginFailureNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/memory/mem_shared": {
+ "metric": "mem_shared",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/SubmitApplicationAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.SubmitApplicationAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_wio": {
+ "metric": "cpu_wio",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/GetNewApplicationNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.GetNewApplicationNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsPending": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsPending",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcCountCopy": {
+ "metric": "jvm.JvmMetrics.GcCountCopy",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/load_fifteen": {
+ "metric": "load_fifteen",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/logError": {
+ "metric": "jvm.JvmMetrics.LogError",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginFailure_avg_time": {
+ "metric": "ugi.ugi.LoginFailureAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_num": {
+ "metric": "cpu_num",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/SubmitApplicationNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.SubmitApplicationNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/gcTimeMillisMarkSweepCompact": {
+ "metric": "jvm.JvmMetrics.GcTimeMillisMarkSweepCompact",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_speed": {
+ "metric": "cpu_speed",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthorizationSuccesses": {
+ "metric": "rpc.rpc.RpcAuthorizationSuccesses",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AllocatedMB",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/logFatal": {
+ "metric": "jvm.JvmMetrics.LogFatal",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcProcessingTime_avg_time": {
+ "metric": "rpc.rpc.RpcProcessingTimeAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/GetApplicationReportAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.GetApplicationReportAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/NodeHeartbeatAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.NodeHeartbeatAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/GetNewApplicationAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.GetNewApplicationAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/pkts_in": {
+ "metric": "pkts_in",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/RegisterApplicationMasterAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.RegisterApplicationMasterAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).ReservedContainers",
+ "pointInTime": false,
+ "temporal": true
+ }
+ }
+ },
+ {
+ "type": "jmx",
+ "metrics": {
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsFailed": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsFailed",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ServiceComponentInfo/rm_metrics/cluster/rebootedNMcount": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memHeapCommittedM": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemHeapCommittedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/ClusterMetrics/NumUnhealthyNMs": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/HeapMemoryMax":{
+ "metric": "java.lang:type=Memory.HeapMemoryUsage[max]",
+ "pointInTime" : true,
+ "temporal" : false
+ },
+ "metrics/jvm/HeapMemoryUsed":{
+ "metric": "java.lang:type=Memory.HeapMemoryUsage[used]",
+ "pointInTime" : true,
+ "temporal" : false
+ },
+ "metrics/jvm/NonHeapMemoryMax":{
+ "metric": "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+ "pointInTime" : true,
+ "temporal" : false
+ },
+ "metrics/jvm/NonHeapMemoryUsed":{
+ "metric": "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+ "pointInTime" : true,
+ "temporal" : false
+ },
+ "metrics/jvm/threadsRunnable": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsRunnable",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsNew": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsNew",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/ClusterMetrics/NumRebootedNMs": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/runtime/StartTime": {
+ "metric": "java.lang:type=Runtime.StartTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsKilled": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsKilled",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/rpcAuthorizationFailures": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthorizationFailures",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AggregateContainersAllocated": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AggregateContainersAllocated",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/ClusterMetrics/NumLostNMs": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/ugi/loginSuccess_avg_time": {
+ "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginSuccessAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ServiceComponentInfo/StartTime": {
+ "metric": "java.lang:type=Runtime.StartTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/RpcQueueTime_avg_time": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcQueueTimeAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ReservedContainers": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ReservedContainers",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsSubmitted": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsSubmitted",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/SentBytes": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.SentBytes",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/ClusterMetrics/NumActiveNMs": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_300": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_300",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memNonHeapUsedM": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemNonHeapUsedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/logWarn": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogWarn",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsTimedWaiting": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsTimedWaiting",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/gcCount": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.GcCount",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/ReceivedBytes": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.ReceivedBytes",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsBlocked": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsBlocked",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_60": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_60",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/RpcQueueTime_num_ops": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcQueueTimeNumOps",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/ClusterMetrics/NumDecommissionedNMs": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumDecommissionedNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AllocatedContainers": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AllocatedContainers",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/PendingContainers": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).PendingContainers",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/NumOpenConnections": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.NumOpenConnections",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memHeapUsedM": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemHeapUsedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsWaiting": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsWaiting",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/gcTimeMillis": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.GcTimeMillis",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/ugi/loginSuccess_num_ops": {
+ "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginSuccessNumOps",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsTerminated": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsTerminated",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memMaxM": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemMaxM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ServiceComponentInfo/rm_metrics/cluster/unhealthyNMcount": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ReservedVCores": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ReservedVCores",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ServiceComponentInfo/rm_metrics/cluster/decommissionedNMcount": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumDecommissionedNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/startTime": {
+ "metric": "java.lang:type=Runtime.StartTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ActiveApplications": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ActiveApplications",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AvailableMB": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AvailableMB",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memNonHeapCommittedM": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemNonHeapCommittedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ServiceComponentInfo/rm_metrics/cluster/nodeManagers": {
+ "metric": "Hadoop:service=ResourceManager,name=RMNMInfo.LiveNodeManagers",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/callQueueLen": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.CallQueueLength",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AllocatedVCores": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AllocatedVCores",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsPending": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsPending",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsCompleted": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsCompleted",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ActiveUsers": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ActiveUsers",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/logInfo": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogInfo",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsRunning": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsRunning",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_1440": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_1440",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AvailableVCores": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AvailableVCores",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/ugi/loginFailure_num_ops": {
+ "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginFailureNumOps",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/RpcProcessingTime_num_ops": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcProcessingTimeNumOps",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ReservedMB": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ReservedMB",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/logError": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogError",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/PendingMB": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).PendingMB",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/ugi/loginFailure_avg_time": {
+ "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginFailureAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/rpcAuthorizationSuccesses": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthorizationSuccesses",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/logFatal": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogFatal",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/RpcProcessingTime_avg_time": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcProcessingTimeAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ServiceComponentInfo/rm_metrics/cluster/activeNMcount": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/rpcAuthenticationSuccesses": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthenticationSuccesses",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AggregateContainersReleased": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AggregateContainersReleased",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/rpcAuthenticationFailures": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthenticationFailures",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "ServiceComponentInfo/rm_metrics/cluster/lostNMcount": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AllocatedMB": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AllocatedMB",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/PendingVCores": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).PendingVCores",
+ "pointInTime": true,
+ "temporal": false
+ }
+ }
+ }
+ ],
+ "HostComponent": [
+ {
+ "type": "ganglia",
+ "metrics": {
+ "metrics/rpcdetailed/FinishApplicationMasterNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.FinishApplicationMasterNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_total": {
+ "metric": "mem_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsCompleted": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsCompleted",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumUnhealthyNMs": {
+ "metric": "yarn.ClusterMetrics.NumUnhealthyNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memHeapCommittedM": {
+ "metric": "jvm.JvmMetrics.MemHeapCommittedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsRunnable": {
+ "metric": "jvm.JvmMetrics.ThreadsRunnable",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumRebootedNMs": {
+ "metric": "yarn.ClusterMetrics.NumRebootedNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsNew": {
+ "metric": "jvm.JvmMetrics.ThreadsNew",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsSubmitted": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsSubmitted",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumLostNMs": {
+ "metric": "yarn.ClusterMetrics.NumLostNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginSuccess_avg_time": {
+ "metric": "ugi.ugi.LoginSuccessAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcQueueTime_avg_time": {
+ "metric": "rpc.rpc.RpcQueueTimeAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AllocatedContainers",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/SentBytes": {
+ "metric": "rpc.rpc.SentBytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsKilled": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsKilled",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumActiveNMs": {
+ "metric": "yarn.ClusterMetrics.NumActiveNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memNonHeapUsedM": {
+ "metric": "jvm.JvmMetrics.MemNonHeapUsedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/logWarn": {
+ "metric": "jvm.JvmMetrics.LogWarn",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsTimedWaiting": {
+ "metric": "jvm.JvmMetrics.ThreadsTimedWaiting",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsFailed": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsFailed",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcCount": {
+ "metric": "jvm.JvmMetrics.GcCount",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/process/proc_run": {
+ "metric": "proc_run",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/swap_total": {
+ "metric": "swap_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/ReceivedBytes": {
+ "metric": "rpc.rpc.ReceivedBytes",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/AllocateNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.AllocateNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_nice": {
+ "metric": "cpu_nice",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/gcCountMarkSweepCompact": {
+ "metric": "jvm.JvmMetrics.GcCountMarkSweepCompact",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsBlocked": {
+ "metric": "jvm.JvmMetrics.ThreadsBlocked",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsRunning": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsRunning",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/ClusterMetrics/NumDecommissionedNMs": {
+ "metric": "yarn.ClusterMetrics.NumDecommissionedNMs",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcQueueTime_num_ops": {
+ "metric": "rpc.rpc.RpcQueueTimeNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/process/proc_total": {
+ "metric": "proc_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/disk/part_max_used": {
+ "metric": "part_max_used",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/NumOpenConnections": {
+ "metric": "rpc.rpc.NumOpenConnections",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memHeapUsedM": {
+ "metric": "jvm.JvmMetrics.MemHeapUsedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/disk/disk_free": {
+ "metric": "disk_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/threadsWaiting": {
+ "metric": "jvm.JvmMetrics.ThreadsWaiting",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/load_one": {
+ "metric": "load_one",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_buffers": {
+ "metric": "mem_buffers",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/ugi/loginSuccess_num_ops": {
+ "metric": "ugi.ugi.LoginSuccessNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcTimeMillisCopy": {
+ "metric": "jvm.JvmMetrics.GcTimeMillisCopy",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/gcTimeMillis": {
+ "metric": "jvm.JvmMetrics.GcTimeMillis",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_idle": {
+ "metric": "cpu_idle",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).PendingContainers",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/memMaxM": {
+ "metric": "jvm.JvmMetrics.MemMaxM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/threadsTerminated": {
+ "metric": "jvm.JvmMetrics.ThreadsTerminated",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/network/bytes_out": {
+ "metric": "bytes_out",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_aidle": {
+ "metric": "cpu_aidle",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/mem_free": {
+ "metric": "mem_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/AllocateAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.AllocateAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_user": {
+ "metric": "cpu_user",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/memory/swap_free": {
+ "metric": "swap_free",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_system": {
+ "metric": "cpu_system",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/load_five": {
+ "metric": "load_five",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/bytes_in": {
+ "metric": "bytes_in",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/boottime": {
+ "metric": "boottime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/pkts_out": {
+ "metric": "pkts_out",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/memNonHeapCommittedM": {
+ "metric": "jvm.JvmMetrics.MemNonHeapCommittedM",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/GetApplicationReportNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.GetApplicationReportNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/FinishApplicationMasterAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.FinishApplicationMasterAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/callQueueLen": {
+ "metric": "rpc.rpc.CallQueueLength",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/memory/mem_cached": {
+ "metric": "mem_cached",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/RegisterApplicationMasterNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.RegisterApplicationMasterNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/disk/disk_total": {
+ "metric": "disk_total",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AvailableMB",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).PendingMB",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/logInfo": {
+ "metric": "jvm.JvmMetrics.LogInfo",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcProcessingTime_num_ops": {
+ "metric": "rpc.rpc.RpcProcessingTimeNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginFailure_num_ops": {
+ "metric": "ugi.ugi.LoginFailureNumOps",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/memory/mem_shared": {
+ "metric": "mem_shared",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/SubmitApplicationAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.SubmitApplicationAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_wio": {
+ "metric": "cpu_wio",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/GetNewApplicationNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.GetNewApplicationNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsPending": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AppsPending",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/gcCountCopy": {
+ "metric": "jvm.JvmMetrics.GcCountCopy",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/load_fifteen": {
+ "metric": "load_fifteen",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/logError": {
+ "metric": "jvm.JvmMetrics.LogError",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/ugi/loginFailure_avg_time": {
+ "metric": "ugi.ugi.LoginFailureAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_num": {
+ "metric": "cpu_num",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/SubmitApplicationNumOps": {
+ "metric": "rpcdetailed.rpcdetailed.SubmitApplicationNumOps",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/jvm/gcTimeMillisMarkSweepCompact": {
+ "metric": "jvm.JvmMetrics.GcTimeMillisMarkSweepCompact",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/cpu/cpu_speed": {
+ "metric": "cpu_speed",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpc/rpcAuthorizationSuccesses": {
+ "metric": "rpc.rpc.RpcAuthorizationSuccesses",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).AllocatedMB",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/jvm/logFatal": {
+ "metric": "jvm.JvmMetrics.LogFatal",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpc/RpcProcessingTime_avg_time": {
+ "metric": "rpc.rpc.RpcProcessingTimeAvgTime",
+ "pointInTime": false,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/GetApplicationReportAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.GetApplicationReportAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/NodeHeartbeatAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.NodeHeartbeatAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/GetNewApplicationAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.GetNewApplicationAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/network/pkts_in": {
+ "metric": "pkts_in",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/rpcdetailed/RegisterApplicationMasterAvgTime": {
+ "metric": "rpcdetailed.rpcdetailed.RegisterApplicationMasterAvgTime",
+ "pointInTime": true,
+ "temporal": true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers": {
+ "metric": "yarn.QueueMetrics.Queue=(.+).ReservedContainers",
+ "pointInTime": false,
+ "temporal": true
+ }
+ }
+ },
+ {
+ "type": "jmx",
+ "metrics": {
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsFailed": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsFailed",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/memHeapCommittedM": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemHeapCommittedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/ClusterMetrics/NumUnhealthyNMs": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsRunnable": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsRunnable",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsNew": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsNew",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/ClusterMetrics/NumRebootedNMs": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsKilled": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsKilled",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/rpcAuthorizationFailures": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthorizationFailures",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AggregateContainersAllocated": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AggregateContainersAllocated",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/ClusterMetrics/NumLostNMs": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/RpcQueueTime_avg_time": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcQueueTimeAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/ugi/loginSuccess_avg_time": {
+ "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginSuccessAvgTime",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ReservedContainers": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ReservedContainers",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsSubmitted": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsSubmitted",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/SentBytes": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.SentBytes",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/ClusterMetrics/NumActiveNMs": {
+ "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_300": {
+ "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_300",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/HeapMemoryMax":{
+ "metric": "java.lang:type=Memory.HeapMemoryUsage[max]",
+ "pointInTime" : true,
+ "temporal" : false
+ },
+ "metrics/jvm/HeapMemoryUsed":{
+ "metric": "java.lang:type=Memory.HeapMemoryUsage[used]",
+ "pointInTime" : true,
+ "temporal" : false
+ },
+ "metrics/jvm/NonHeapMemoryMax":{
+ "metric": "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+ "pointInTime" : true,
+ "temporal" : false
+ },
+ "metrics/jvm/NonHeapMemoryUsed":{
+ "metric": "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+ "pointInTime" : true,
+ "temporal" : false
+ },
+ "metrics/jvm/memNonHeapUsedM": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemNonHeapUsedM",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/logWarn": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogWarn",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsTimedWaiting": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsTimedWaiting",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/gcCount": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.GcCount",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/rpc/ReceivedBytes": {
+ "metric": "Hadoop:service=ResourceManager,name=RpcActivity.ReceivedBytes",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/jvm/threadsBlocked": {
+ "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsBlocked",
+ "pointInTime": true,
+ "temporal": false
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_60": {
+ "metric": "Hadoop:service=Re
<TRUNCATED>
[4/8] ambari git commit: AMBARI-8878: Common Services: Refactor
HDP-2.0.6 YARN service (Jayush Luniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/metrics.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/metrics.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/metrics.json
deleted file mode 100644
index 5d8309d..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/metrics.json
+++ /dev/null
@@ -1,5360 +0,0 @@
-{
- "NODEMANAGER": {
- "Component": [
- {
- "type": "ganglia",
- "metrics": {
- "metrics/memory/mem_total": {
- "metric": "mem_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/memHeapCommittedM": {
- "metric": "jvm.JvmMetrics.MemHeapCommittedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/mapred/ShuffleOutputsFailed": {
- "metric": "mapred.ShuffleOutputsFailed",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsRunnable": {
- "metric": "jvm.JvmMetrics.ThreadsRunnable",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsNew": {
- "metric": "jvm.JvmMetrics.ThreadsNew",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/rpcAuthorizationFailures": {
- "metric": "rpc.metrics.RpcAuthorizationFailures",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginSuccess_avg_time": {
- "metric": "ugi.ugi.LoginSuccessAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcQueueTime_avg_time": {
- "metric": "rpc.rpc.RpcQueueTimeAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ContainersCompleted": {
- "metric": "yarn.NodeManagerMetrics.ContainersCompleted",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/SentBytes": {
- "metric": "rpc.rpc.SentBytes",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memNonHeapUsedM": {
- "metric": "jvm.JvmMetrics.MemNonHeapUsedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ContainersKilled": {
- "metric": "yarn.NodeManagerMetrics.ContainersKilled",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/logWarn": {
- "metric": "jvm.JvmMetrics.LogWarn",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsTimedWaiting": {
- "metric": "jvm.JvmMetrics.ThreadsTimedWaiting",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcCount": {
- "metric": "jvm.JvmMetrics.GcCount",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/process/proc_run": {
- "metric": "proc_run",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/swap_total": {
- "metric": "swap_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/ReceivedBytes": {
- "metric": "rpc.rpc.ReceivedBytes",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_nice": {
- "metric": "cpu_nice",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsBlocked": {
- "metric": "jvm.JvmMetrics.ThreadsBlocked",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcQueueTime_num_ops": {
- "metric": "rpc.rpc.RpcQueueTimeNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/process/proc_total": {
- "metric": "proc_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/AllocatedGB": {
- "metric": "yarn.NodeManagerMetrics.AllocatedGB",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/disk/part_max_used": {
- "metric": "part_max_used",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/NumOpenConnections": {
- "metric": "rpc.rpc.NumOpenConnections",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memHeapUsedM": {
- "metric": "jvm.JvmMetrics.MemHeapUsedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsWaiting": {
- "metric": "jvm.JvmMetrics.ThreadsWaiting",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/disk/disk_free": {
- "metric": "disk_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/mapred/ShuffleOutputsOK": {
- "metric": "mapred.ShuffleOutputsOK",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/ContainersFailed": {
- "metric": "yarn.NodeManagerMetrics.ContainersFailed",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_buffers": {
- "metric": "mem_buffers",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/ugi/loginSuccess_num_ops": {
- "metric": "ugi.ugi.LoginSuccessNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcTimeMillis": {
- "metric": "jvm.JvmMetrics.GcTimeMillis",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_idle": {
- "metric": "cpu_idle",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/AllocatedContainers": {
- "metric": "yarn.NodeManagerMetrics.AllocatedContainers",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsTerminated": {
- "metric": "jvm.JvmMetrics.ThreadsTerminated",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/network/bytes_out": {
- "metric": "bytes_out",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_aidle": {
- "metric": "cpu_aidle",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_free": {
- "metric": "mem_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_user": {
- "metric": "cpu_user",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/swap_free": {
- "metric": "swap_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_system": {
- "metric": "cpu_system",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/bytes_in": {
- "metric": "bytes_in",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/boottime": {
- "metric": "boottime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/pkts_out": {
- "metric": "pkts_out",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/memNonHeapCommittedM": {
- "metric": "jvm.JvmMetrics.MemNonHeapCommittedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/callQueueLen": {
- "metric": "rpc.rpc.CallQueueLength",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/memory/mem_cached": {
- "metric": "mem_cached",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/ContainersRunning": {
- "metric": "yarn.NodeManagerMetrics.ContainersRunning",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/disk/disk_total": {
- "metric": "disk_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/logInfo": {
- "metric": "jvm.JvmMetrics.LogInfo",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ContainersLaunched": {
- "metric": "yarn.NodeManagerMetrics.ContainersLaunched",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/RpcProcessingTime_num_ops": {
- "metric": "rpc.rpc.RpcProcessingTimeNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginFailure_num_ops": {
- "metric": "ugi.ugi.LoginFailureNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/memory/mem_shared": {
- "metric": "mem_shared",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/AvailableGB": {
- "metric": "yarn.NodeManagerMetrics.AvailableGB",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/mapred/ShuffleConnections": {
- "metric": "mapred.ShuffleConnections",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_wio": {
- "metric": "cpu_wio",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/ContainersIniting": {
- "metric": "yarn.NodeManagerMetrics.ContainersIniting",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/logError": {
- "metric": "jvm.JvmMetrics.LogError",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginFailure_avg_time": {
- "metric": "ugi.ugi.LoginFailureAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_num": {
- "metric": "cpu_num",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_speed": {
- "metric": "cpu_speed",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/rpcAuthorizationSuccesses": {
- "metric": "rpc.rpc.RpcAuthorizationSuccesses",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/logFatal": {
- "metric": "jvm.JvmMetrics.LogFatal",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcProcessingTime_avg_time": {
- "metric": "rpc.rpc.RpcProcessingTimeAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/mapred/ShuffleOutputBytes": {
- "metric": "mapred.ShuffleOutputBytes",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/rpcAuthenticationSuccesses": {
- "metric": "rpc.metrics.RpcAuthenticationSuccesses",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/rpcAuthenticationFailures": {
- "metric": "rpc.metrics.RpcAuthenticationFailures",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/network/pkts_in": {
- "metric": "pkts_in",
- "pointInTime": true,
- "temporal": true
- }
- }
- }
- ],
- "HostComponent": [
- {
- "type": "ganglia",
- "metrics": {
- "metrics/memory/mem_total": {
- "metric": "mem_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/memHeapCommittedM": {
- "metric": "jvm.JvmMetrics.MemHeapCommittedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/mapred/ShuffleOutputsFailed": {
- "metric": "mapred.ShuffleOutputsFailed",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsRunnable": {
- "metric": "jvm.JvmMetrics.ThreadsRunnable",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsNew": {
- "metric": "jvm.JvmMetrics.ThreadsNew",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/rpcAuthorizationFailures": {
- "metric": "rpc.metrics.RpcAuthorizationFailures",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginSuccess_avg_time": {
- "metric": "ugi.ugi.LoginSuccessAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcQueueTime_avg_time": {
- "metric": "rpc.rpc.RpcQueueTimeAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ContainersCompleted": {
- "metric": "yarn.NodeManagerMetrics.ContainersCompleted",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/SentBytes": {
- "metric": "rpc.rpc.SentBytes",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memNonHeapUsedM": {
- "metric": "jvm.JvmMetrics.MemNonHeapUsedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ContainersKilled": {
- "metric": "yarn.NodeManagerMetrics.ContainersKilled",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/logWarn": {
- "metric": "jvm.JvmMetrics.LogWarn",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsTimedWaiting": {
- "metric": "jvm.JvmMetrics.ThreadsTimedWaiting",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcCount": {
- "metric": "jvm.JvmMetrics.GcCount",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/process/proc_run": {
- "metric": "proc_run",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/swap_total": {
- "metric": "swap_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/ReceivedBytes": {
- "metric": "rpc.rpc.ReceivedBytes",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_nice": {
- "metric": "cpu_nice",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsBlocked": {
- "metric": "jvm.JvmMetrics.ThreadsBlocked",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcQueueTime_num_ops": {
- "metric": "rpc.rpc.RpcQueueTimeNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/process/proc_total": {
- "metric": "proc_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/AllocatedGB": {
- "metric": "yarn.NodeManagerMetrics.AllocatedGB",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/disk/part_max_used": {
- "metric": "part_max_used",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/NumOpenConnections": {
- "metric": "rpc.rpc.NumOpenConnections",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memHeapUsedM": {
- "metric": "jvm.JvmMetrics.MemHeapUsedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsWaiting": {
- "metric": "jvm.JvmMetrics.ThreadsWaiting",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/disk/disk_free": {
- "metric": "disk_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/mapred/ShuffleOutputsOK": {
- "metric": "mapred.ShuffleOutputsOK",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/ContainersFailed": {
- "metric": "yarn.NodeManagerMetrics.ContainersFailed",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_buffers": {
- "metric": "mem_buffers",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/ugi/loginSuccess_num_ops": {
- "metric": "ugi.ugi.LoginSuccessNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcTimeMillis": {
- "metric": "jvm.JvmMetrics.GcTimeMillis",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_idle": {
- "metric": "cpu_idle",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/AllocatedContainers": {
- "metric": "yarn.NodeManagerMetrics.AllocatedContainers",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsTerminated": {
- "metric": "jvm.JvmMetrics.ThreadsTerminated",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/network/bytes_out": {
- "metric": "bytes_out",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_aidle": {
- "metric": "cpu_aidle",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_free": {
- "metric": "mem_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_user": {
- "metric": "cpu_user",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/swap_free": {
- "metric": "swap_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_system": {
- "metric": "cpu_system",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/bytes_in": {
- "metric": "bytes_in",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/boottime": {
- "metric": "boottime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/pkts_out": {
- "metric": "pkts_out",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/memNonHeapCommittedM": {
- "metric": "jvm.JvmMetrics.MemNonHeapCommittedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/callQueueLen": {
- "metric": "rpc.rpc.CallQueueLength",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/memory/mem_cached": {
- "metric": "mem_cached",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/ContainersRunning": {
- "metric": "yarn.NodeManagerMetrics.ContainersRunning",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/disk/disk_total": {
- "metric": "disk_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/logInfo": {
- "metric": "jvm.JvmMetrics.LogInfo",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ContainersLaunched": {
- "metric": "yarn.NodeManagerMetrics.ContainersLaunched",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/RpcProcessingTime_num_ops": {
- "metric": "rpc.rpc.RpcProcessingTimeNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginFailure_num_ops": {
- "metric": "ugi.ugi.LoginFailureNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/memory/mem_shared": {
- "metric": "mem_shared",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/AvailableGB": {
- "metric": "yarn.NodeManagerMetrics.AvailableGB",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/mapred/ShuffleConnections": {
- "metric": "mapred.ShuffleConnections",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_wio": {
- "metric": "cpu_wio",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/ContainersIniting": {
- "metric": "yarn.NodeManagerMetrics.ContainersIniting",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/logError": {
- "metric": "jvm.JvmMetrics.LogError",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginFailure_avg_time": {
- "metric": "ugi.ugi.LoginFailureAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_num": {
- "metric": "cpu_num",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_speed": {
- "metric": "cpu_speed",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/rpcAuthorizationSuccesses": {
- "metric": "rpc.rpc.RpcAuthorizationSuccesses",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/logFatal": {
- "metric": "jvm.JvmMetrics.LogFatal",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcProcessingTime_avg_time": {
- "metric": "rpc.rpc.RpcProcessingTimeAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/mapred/ShuffleOutputBytes": {
- "metric": "mapred.ShuffleOutputBytes",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/rpcAuthenticationSuccesses": {
- "metric": "rpc.metrics.RpcAuthenticationSuccesses",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/rpcAuthenticationFailures": {
- "metric": "rpc.metrics.RpcAuthenticationFailures",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/network/pkts_in": {
- "metric": "pkts_in",
- "pointInTime": true,
- "temporal": true
- }
- }
- },
- {
- "type": "jmx",
- "metrics": {
- "metrics/jvm/memHeapCommittedM": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.MemHeapCommittedM",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsRunnable": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsRunnable",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsNew": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsNew",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memNonHeapCommittedM": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.MemNonHeapCommittedM",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/callQueueLen": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.CallQueueLength",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/rpcAuthorizationFailures": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcAuthorizationFailures",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/RpcQueueTime_avg_time": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcQueueTimeAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/ugi/loginSuccess_avg_time": {
- "metric": "Hadoop:service=NodeManager,name=UgiMetrics.LoginSuccessAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/SentBytes": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.SentBytes",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/logInfo": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.LogInfo",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memNonHeapUsedM": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.MemNonHeapUsedM",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/logWarn": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.LogWarn",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/ugi/loginFailure_num_ops": {
- "metric": "Hadoop:service=NodeManager,name=UgiMetrics.LoginFailureNumOps",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsTimedWaiting": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsTimedWaiting",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/RpcProcessingTime_num_ops": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcProcessingTimeNumOps",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/gcCount": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.GcCount",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/ReceivedBytes": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.ReceivedBytes",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsBlocked": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsBlocked",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/RpcQueueTime_num_ops": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcQueueTimeNumOps",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/logError": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.LogError",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/ugi/loginFailure_avg_time": {
- "metric": "Hadoop:service=NodeManager,name=UgiMetrics.LoginFailureAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/rpcAuthorizationSuccesses": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcAuthorizationSuccesses",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/NumOpenConnections": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.NumOpenConnections",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memHeapUsedM": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.MemHeapUsedM",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/logFatal": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.LogFatal",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/RpcProcessingTime_avg_time": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcProcessingTimeAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsWaiting": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsWaiting",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/gcTimeMillis": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.GcTimeMillis",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/ugi/loginSuccess_num_ops": {
- "metric": "Hadoop:service=NodeManager,name=UgiMetrics.LoginSuccessNumOps",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/rpcAuthenticationSuccesses": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcAuthenticationSuccesses",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/rpcAuthenticationFailures": {
- "metric": "Hadoop:service=NodeManager,name=RpcActivity.RpcAuthenticationFailures",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsTerminated": {
- "metric": "Hadoop:service=NodeManager,name=JvmMetrics.ThreadsTerminated",
- "pointInTime": true,
- "temporal": false
- }
- }
- }
- ]
- },
- "RESOURCEMANAGER": {
- "Component": [
- {
- "type": "ganglia",
- "metrics": {
- "metrics/rpcdetailed/FinishApplicationMasterNumOps": {
- "metric": "rpcdetailed.rpcdetailed.FinishApplicationMasterNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_total": {
- "metric": "mem_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsCompleted": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsCompleted",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumUnhealthyNMs": {
- "metric": "yarn.ClusterMetrics.NumUnhealthyNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memHeapCommittedM": {
- "metric": "jvm.JvmMetrics.MemHeapCommittedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsRunnable": {
- "metric": "jvm.JvmMetrics.ThreadsRunnable",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumRebootedNMs": {
- "metric": "yarn.ClusterMetrics.NumRebootedNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsNew": {
- "metric": "jvm.JvmMetrics.ThreadsNew",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsSubmitted": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsSubmitted",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumLostNMs": {
- "metric": "yarn.ClusterMetrics.NumLostNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginSuccess_avg_time": {
- "metric": "ugi.ugi.LoginSuccessAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcQueueTime_avg_time": {
- "metric": "rpc.rpc.RpcQueueTimeAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AllocatedContainers",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/SentBytes": {
- "metric": "rpc.rpc.SentBytes",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsKilled": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsKilled",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumActiveNMs": {
- "metric": "yarn.ClusterMetrics.NumActiveNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memNonHeapUsedM": {
- "metric": "jvm.JvmMetrics.MemNonHeapUsedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/logWarn": {
- "metric": "jvm.JvmMetrics.LogWarn",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsTimedWaiting": {
- "metric": "jvm.JvmMetrics.ThreadsTimedWaiting",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsFailed": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsFailed",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcCount": {
- "metric": "jvm.JvmMetrics.GcCount",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/process/proc_run": {
- "metric": "proc_run",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/swap_total": {
- "metric": "swap_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/ReceivedBytes": {
- "metric": "rpc.rpc.ReceivedBytes",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpcdetailed/AllocateNumOps": {
- "metric": "rpcdetailed.rpcdetailed.AllocateNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_nice": {
- "metric": "cpu_nice",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/gcCountMarkSweepCompact": {
- "metric": "jvm.JvmMetrics.GcCountMarkSweepCompact",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsBlocked": {
- "metric": "jvm.JvmMetrics.ThreadsBlocked",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsRunning": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsRunning",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumDecommissionedNMs": {
- "metric": "yarn.ClusterMetrics.NumDecommissionedNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcQueueTime_num_ops": {
- "metric": "rpc.rpc.RpcQueueTimeNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/process/proc_total": {
- "metric": "proc_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/disk/part_max_used": {
- "metric": "part_max_used",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/NumOpenConnections": {
- "metric": "rpc.rpc.NumOpenConnections",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memHeapUsedM": {
- "metric": "jvm.JvmMetrics.MemHeapUsedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/disk/disk_free": {
- "metric": "disk_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsWaiting": {
- "metric": "jvm.JvmMetrics.ThreadsWaiting",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/load_one": {
- "metric": "load_one",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_buffers": {
- "metric": "mem_buffers",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/ugi/loginSuccess_num_ops": {
- "metric": "ugi.ugi.LoginSuccessNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcTimeMillisCopy": {
- "metric": "jvm.JvmMetrics.GcTimeMillisCopy",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/gcTimeMillis": {
- "metric": "jvm.JvmMetrics.GcTimeMillis",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_idle": {
- "metric": "cpu_idle",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers": {
- "metric": "yarn.QueueMetrics.Queue=(.+).PendingContainers",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memMaxM": {
- "metric": "jvm.JvmMetrics.MemMaxM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsTerminated": {
- "metric": "jvm.JvmMetrics.ThreadsTerminated",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/network/bytes_out": {
- "metric": "bytes_out",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_aidle": {
- "metric": "cpu_aidle",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_free": {
- "metric": "mem_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/AllocateAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.AllocateAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_user": {
- "metric": "cpu_user",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/swap_free": {
- "metric": "swap_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_system": {
- "metric": "cpu_system",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/load_five": {
- "metric": "load_five",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/bytes_in": {
- "metric": "bytes_in",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/boottime": {
- "metric": "boottime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/pkts_out": {
- "metric": "pkts_out",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/memNonHeapCommittedM": {
- "metric": "jvm.JvmMetrics.MemNonHeapCommittedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpcdetailed/GetApplicationReportNumOps": {
- "metric": "rpcdetailed.rpcdetailed.GetApplicationReportNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/FinishApplicationMasterAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.FinishApplicationMasterAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/callQueueLen": {
- "metric": "rpc.rpc.CallQueueLength",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/memory/mem_cached": {
- "metric": "mem_cached",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/RegisterApplicationMasterNumOps": {
- "metric": "rpcdetailed.rpcdetailed.RegisterApplicationMasterNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/disk/disk_total": {
- "metric": "disk_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AvailableMB",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB": {
- "metric": "yarn.QueueMetrics.Queue=(.+).PendingMB",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/logInfo": {
- "metric": "jvm.JvmMetrics.LogInfo",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcProcessingTime_num_ops": {
- "metric": "rpc.rpc.RpcProcessingTimeNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginFailure_num_ops": {
- "metric": "ugi.ugi.LoginFailureNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/memory/mem_shared": {
- "metric": "mem_shared",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/SubmitApplicationAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.SubmitApplicationAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_wio": {
- "metric": "cpu_wio",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/GetNewApplicationNumOps": {
- "metric": "rpcdetailed.rpcdetailed.GetNewApplicationNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsPending": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsPending",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcCountCopy": {
- "metric": "jvm.JvmMetrics.GcCountCopy",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/load_fifteen": {
- "metric": "load_fifteen",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/logError": {
- "metric": "jvm.JvmMetrics.LogError",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginFailure_avg_time": {
- "metric": "ugi.ugi.LoginFailureAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_num": {
- "metric": "cpu_num",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/SubmitApplicationNumOps": {
- "metric": "rpcdetailed.rpcdetailed.SubmitApplicationNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/gcTimeMillisMarkSweepCompact": {
- "metric": "jvm.JvmMetrics.GcTimeMillisMarkSweepCompact",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_speed": {
- "metric": "cpu_speed",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/rpcAuthorizationSuccesses": {
- "metric": "rpc.rpc.RpcAuthorizationSuccesses",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AllocatedMB",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/logFatal": {
- "metric": "jvm.JvmMetrics.LogFatal",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcProcessingTime_avg_time": {
- "metric": "rpc.rpc.RpcProcessingTimeAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpcdetailed/GetApplicationReportAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.GetApplicationReportAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/NodeHeartbeatAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.NodeHeartbeatAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/GetNewApplicationAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.GetNewApplicationAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/pkts_in": {
- "metric": "pkts_in",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/RegisterApplicationMasterAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.RegisterApplicationMasterAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers": {
- "metric": "yarn.QueueMetrics.Queue=(.+).ReservedContainers",
- "pointInTime": false,
- "temporal": true
- }
- }
- },
- {
- "type": "jmx",
- "metrics": {
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsFailed": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsFailed",
- "pointInTime": true,
- "temporal": false
- },
- "ServiceComponentInfo/rm_metrics/cluster/rebootedNMcount": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memHeapCommittedM": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemHeapCommittedM",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/ClusterMetrics/NumUnhealthyNMs": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/HeapMemoryMax":{
- "metric": "java.lang:type=Memory.HeapMemoryUsage[max]",
- "pointInTime" : true,
- "temporal" : false
- },
- "metrics/jvm/HeapMemoryUsed":{
- "metric": "java.lang:type=Memory.HeapMemoryUsage[used]",
- "pointInTime" : true,
- "temporal" : false
- },
- "metrics/jvm/NonHeapMemoryMax":{
- "metric": "java.lang:type=Memory.NonHeapMemoryUsage[max]",
- "pointInTime" : true,
- "temporal" : false
- },
- "metrics/jvm/NonHeapMemoryUsed":{
- "metric": "java.lang:type=Memory.NonHeapMemoryUsage[used]",
- "pointInTime" : true,
- "temporal" : false
- },
- "metrics/jvm/threadsRunnable": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsRunnable",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsNew": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsNew",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/ClusterMetrics/NumRebootedNMs": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/runtime/StartTime": {
- "metric": "java.lang:type=Runtime.StartTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsKilled": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsKilled",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/rpcAuthorizationFailures": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthorizationFailures",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AggregateContainersAllocated": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AggregateContainersAllocated",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/ClusterMetrics/NumLostNMs": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/ugi/loginSuccess_avg_time": {
- "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginSuccessAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "ServiceComponentInfo/StartTime": {
- "metric": "java.lang:type=Runtime.StartTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/RpcQueueTime_avg_time": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcQueueTimeAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ReservedContainers": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ReservedContainers",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsSubmitted": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsSubmitted",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/SentBytes": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.SentBytes",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/ClusterMetrics/NumActiveNMs": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_300": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_300",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memNonHeapUsedM": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemNonHeapUsedM",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/logWarn": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogWarn",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsTimedWaiting": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsTimedWaiting",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/gcCount": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.GcCount",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/ReceivedBytes": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.ReceivedBytes",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsBlocked": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsBlocked",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_60": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_60",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/RpcQueueTime_num_ops": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcQueueTimeNumOps",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/ClusterMetrics/NumDecommissionedNMs": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumDecommissionedNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AllocatedContainers": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AllocatedContainers",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/PendingContainers": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).PendingContainers",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/NumOpenConnections": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.NumOpenConnections",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memHeapUsedM": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemHeapUsedM",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsWaiting": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsWaiting",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/gcTimeMillis": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.GcTimeMillis",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/ugi/loginSuccess_num_ops": {
- "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginSuccessNumOps",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsTerminated": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsTerminated",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memMaxM": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemMaxM",
- "pointInTime": true,
- "temporal": false
- },
- "ServiceComponentInfo/rm_metrics/cluster/unhealthyNMcount": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ReservedVCores": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ReservedVCores",
- "pointInTime": true,
- "temporal": false
- },
- "ServiceComponentInfo/rm_metrics/cluster/decommissionedNMcount": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumDecommissionedNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/startTime": {
- "metric": "java.lang:type=Runtime.StartTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ActiveApplications": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ActiveApplications",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AvailableMB": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AvailableMB",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memNonHeapCommittedM": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemNonHeapCommittedM",
- "pointInTime": true,
- "temporal": false
- },
- "ServiceComponentInfo/rm_metrics/cluster/nodeManagers": {
- "metric": "Hadoop:service=ResourceManager,name=RMNMInfo.LiveNodeManagers",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/callQueueLen": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.CallQueueLength",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AllocatedVCores": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AllocatedVCores",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsPending": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsPending",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsCompleted": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsCompleted",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ActiveUsers": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ActiveUsers",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/logInfo": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogInfo",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsRunning": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsRunning",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_1440": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_1440",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AvailableVCores": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AvailableVCores",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/ugi/loginFailure_num_ops": {
- "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginFailureNumOps",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/RpcProcessingTime_num_ops": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcProcessingTimeNumOps",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ReservedMB": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ReservedMB",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/logError": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogError",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/PendingMB": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).PendingMB",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/ugi/loginFailure_avg_time": {
- "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginFailureAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/rpcAuthorizationSuccesses": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthorizationSuccesses",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/logFatal": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogFatal",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/RpcProcessingTime_avg_time": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcProcessingTimeAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "ServiceComponentInfo/rm_metrics/cluster/activeNMcount": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/rpcAuthenticationSuccesses": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthenticationSuccesses",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AggregateContainersReleased": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AggregateContainersReleased",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/rpcAuthenticationFailures": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthenticationFailures",
- "pointInTime": true,
- "temporal": false
- },
- "ServiceComponentInfo/rm_metrics/cluster/lostNMcount": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AllocatedMB": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AllocatedMB",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/PendingVCores": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).PendingVCores",
- "pointInTime": true,
- "temporal": false
- }
- }
- }
- ],
- "HostComponent": [
- {
- "type": "ganglia",
- "metrics": {
- "metrics/rpcdetailed/FinishApplicationMasterNumOps": {
- "metric": "rpcdetailed.rpcdetailed.FinishApplicationMasterNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_total": {
- "metric": "mem_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsCompleted": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsCompleted",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumUnhealthyNMs": {
- "metric": "yarn.ClusterMetrics.NumUnhealthyNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memHeapCommittedM": {
- "metric": "jvm.JvmMetrics.MemHeapCommittedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsRunnable": {
- "metric": "jvm.JvmMetrics.ThreadsRunnable",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumRebootedNMs": {
- "metric": "yarn.ClusterMetrics.NumRebootedNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsNew": {
- "metric": "jvm.JvmMetrics.ThreadsNew",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsSubmitted": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsSubmitted",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumLostNMs": {
- "metric": "yarn.ClusterMetrics.NumLostNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginSuccess_avg_time": {
- "metric": "ugi.ugi.LoginSuccessAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcQueueTime_avg_time": {
- "metric": "rpc.rpc.RpcQueueTimeAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AllocatedContainers",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/SentBytes": {
- "metric": "rpc.rpc.SentBytes",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsKilled": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsKilled",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumActiveNMs": {
- "metric": "yarn.ClusterMetrics.NumActiveNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memNonHeapUsedM": {
- "metric": "jvm.JvmMetrics.MemNonHeapUsedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/logWarn": {
- "metric": "jvm.JvmMetrics.LogWarn",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsTimedWaiting": {
- "metric": "jvm.JvmMetrics.ThreadsTimedWaiting",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsFailed": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsFailed",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcCount": {
- "metric": "jvm.JvmMetrics.GcCount",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/process/proc_run": {
- "metric": "proc_run",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/swap_total": {
- "metric": "swap_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/ReceivedBytes": {
- "metric": "rpc.rpc.ReceivedBytes",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpcdetailed/AllocateNumOps": {
- "metric": "rpcdetailed.rpcdetailed.AllocateNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_nice": {
- "metric": "cpu_nice",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/gcCountMarkSweepCompact": {
- "metric": "jvm.JvmMetrics.GcCountMarkSweepCompact",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsBlocked": {
- "metric": "jvm.JvmMetrics.ThreadsBlocked",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsRunning": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsRunning",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/ClusterMetrics/NumDecommissionedNMs": {
- "metric": "yarn.ClusterMetrics.NumDecommissionedNMs",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcQueueTime_num_ops": {
- "metric": "rpc.rpc.RpcQueueTimeNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/process/proc_total": {
- "metric": "proc_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/disk/part_max_used": {
- "metric": "part_max_used",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/NumOpenConnections": {
- "metric": "rpc.rpc.NumOpenConnections",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memHeapUsedM": {
- "metric": "jvm.JvmMetrics.MemHeapUsedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/disk/disk_free": {
- "metric": "disk_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/threadsWaiting": {
- "metric": "jvm.JvmMetrics.ThreadsWaiting",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/load_one": {
- "metric": "load_one",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_buffers": {
- "metric": "mem_buffers",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/ugi/loginSuccess_num_ops": {
- "metric": "ugi.ugi.LoginSuccessNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcTimeMillisCopy": {
- "metric": "jvm.JvmMetrics.GcTimeMillisCopy",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/gcTimeMillis": {
- "metric": "jvm.JvmMetrics.GcTimeMillis",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_idle": {
- "metric": "cpu_idle",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers": {
- "metric": "yarn.QueueMetrics.Queue=(.+).PendingContainers",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/memMaxM": {
- "metric": "jvm.JvmMetrics.MemMaxM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/threadsTerminated": {
- "metric": "jvm.JvmMetrics.ThreadsTerminated",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/network/bytes_out": {
- "metric": "bytes_out",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_aidle": {
- "metric": "cpu_aidle",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/mem_free": {
- "metric": "mem_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/AllocateAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.AllocateAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_user": {
- "metric": "cpu_user",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/memory/swap_free": {
- "metric": "swap_free",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_system": {
- "metric": "cpu_system",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/load_five": {
- "metric": "load_five",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/bytes_in": {
- "metric": "bytes_in",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/boottime": {
- "metric": "boottime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/pkts_out": {
- "metric": "pkts_out",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/memNonHeapCommittedM": {
- "metric": "jvm.JvmMetrics.MemNonHeapCommittedM",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpcdetailed/GetApplicationReportNumOps": {
- "metric": "rpcdetailed.rpcdetailed.GetApplicationReportNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/FinishApplicationMasterAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.FinishApplicationMasterAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/callQueueLen": {
- "metric": "rpc.rpc.CallQueueLength",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/memory/mem_cached": {
- "metric": "mem_cached",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/RegisterApplicationMasterNumOps": {
- "metric": "rpcdetailed.rpcdetailed.RegisterApplicationMasterNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/disk/disk_total": {
- "metric": "disk_total",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AvailableMB",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB": {
- "metric": "yarn.QueueMetrics.Queue=(.+).PendingMB",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/logInfo": {
- "metric": "jvm.JvmMetrics.LogInfo",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcProcessingTime_num_ops": {
- "metric": "rpc.rpc.RpcProcessingTimeNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginFailure_num_ops": {
- "metric": "ugi.ugi.LoginFailureNumOps",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/memory/mem_shared": {
- "metric": "mem_shared",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/SubmitApplicationAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.SubmitApplicationAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_wio": {
- "metric": "cpu_wio",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/GetNewApplicationNumOps": {
- "metric": "rpcdetailed.rpcdetailed.GetNewApplicationNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AppsPending": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AppsPending",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/gcCountCopy": {
- "metric": "jvm.JvmMetrics.GcCountCopy",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/load_fifteen": {
- "metric": "load_fifteen",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/logError": {
- "metric": "jvm.JvmMetrics.LogError",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/ugi/loginFailure_avg_time": {
- "metric": "ugi.ugi.LoginFailureAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/cpu/cpu_num": {
- "metric": "cpu_num",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/SubmitApplicationNumOps": {
- "metric": "rpcdetailed.rpcdetailed.SubmitApplicationNumOps",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/jvm/gcTimeMillisMarkSweepCompact": {
- "metric": "jvm.JvmMetrics.GcTimeMillisMarkSweepCompact",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/cpu/cpu_speed": {
- "metric": "cpu_speed",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpc/rpcAuthorizationSuccesses": {
- "metric": "rpc.rpc.RpcAuthorizationSuccesses",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB": {
- "metric": "yarn.QueueMetrics.Queue=(.+).AllocatedMB",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/jvm/logFatal": {
- "metric": "jvm.JvmMetrics.LogFatal",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpc/RpcProcessingTime_avg_time": {
- "metric": "rpc.rpc.RpcProcessingTimeAvgTime",
- "pointInTime": false,
- "temporal": true
- },
- "metrics/rpcdetailed/GetApplicationReportAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.GetApplicationReportAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/NodeHeartbeatAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.NodeHeartbeatAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/GetNewApplicationAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.GetNewApplicationAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/network/pkts_in": {
- "metric": "pkts_in",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/rpcdetailed/RegisterApplicationMasterAvgTime": {
- "metric": "rpcdetailed.rpcdetailed.RegisterApplicationMasterAvgTime",
- "pointInTime": true,
- "temporal": true
- },
- "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers": {
- "metric": "yarn.QueueMetrics.Queue=(.+).ReservedContainers",
- "pointInTime": false,
- "temporal": true
- }
- }
- },
- {
- "type": "jmx",
- "metrics": {
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsFailed": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsFailed",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/memHeapCommittedM": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemHeapCommittedM",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/ClusterMetrics/NumUnhealthyNMs": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsRunnable": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsRunnable",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsNew": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsNew",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/ClusterMetrics/NumRebootedNMs": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsKilled": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsKilled",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/rpcAuthorizationFailures": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcAuthorizationFailures",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AggregateContainersAllocated": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AggregateContainersAllocated",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/ClusterMetrics/NumLostNMs": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/RpcQueueTime_avg_time": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.RpcQueueTimeAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/ugi/loginSuccess_avg_time": {
- "metric": "Hadoop:service=ResourceManager,name=UgiMetrics.LoginSuccessAvgTime",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/ReservedContainers": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).ReservedContainers",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/AppsSubmitted": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsSubmitted",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/SentBytes": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.SentBytes",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/ClusterMetrics/NumActiveNMs": {
- "metric": "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_300": {
- "metric": "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_300",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/HeapMemoryMax":{
- "metric": "java.lang:type=Memory.HeapMemoryUsage[max]",
- "pointInTime" : true,
- "temporal" : false
- },
- "metrics/jvm/HeapMemoryUsed":{
- "metric": "java.lang:type=Memory.HeapMemoryUsage[used]",
- "pointInTime" : true,
- "temporal" : false
- },
- "metrics/jvm/NonHeapMemoryMax":{
- "metric": "java.lang:type=Memory.NonHeapMemoryUsage[max]",
- "pointInTime" : true,
- "temporal" : false
- },
- "metrics/jvm/NonHeapMemoryUsed":{
- "metric": "java.lang:type=Memory.NonHeapMemoryUsage[used]",
- "pointInTime" : true,
- "temporal" : false
- },
- "metrics/jvm/memNonHeapUsedM": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.MemNonHeapUsedM",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/logWarn": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.LogWarn",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsTimedWaiting": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsTimedWaiting",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/gcCount": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.GcCount",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/rpc/ReceivedBytes": {
- "metric": "Hadoop:service=ResourceManager,name=RpcActivity.ReceivedBytes",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/jvm/threadsBlocked": {
- "metric": "Hadoop:service=ResourceManager,name=JvmMetrics.ThreadsBlocked",
- "pointInTime": true,
- "temporal": false
- },
- "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_60": {
- "metric": "Hadoop:servic
<TRUNCATED>
[6/8] ambari git commit: AMBARI-8878: Common Services: Refactor
HDP-2.0.6 YARN service (Jayush Luniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/alerts/alert_nodemanager_health.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/alerts/alert_nodemanager_health.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/alerts/alert_nodemanager_health.py
new file mode 100644
index 0000000..b1de951
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/alerts/alert_nodemanager_health.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import json
+import socket
+import urllib2
+
+RESULT_CODE_OK = 'OK'
+RESULT_CODE_CRITICAL = 'CRITICAL'
+RESULT_CODE_UNKNOWN = 'UNKNOWN'
+
+NODEMANAGER_HTTP_ADDRESS_KEY = '{{yarn-site/yarn.nodemanager.webapp.address}}'
+NODEMANAGER_HTTPS_ADDRESS_KEY = '{{yarn-site/yarn.nodemanager.webapp.https.address}}'
+YARN_HTTP_POLICY_KEY = '{{yarn-site/yarn.http.policy}}'
+
+OK_MESSAGE = 'NodeManager Healthy'
+CRITICAL_CONNECTION_MESSAGE = 'Connection failed to {0}'
+CRITICAL_NODEMANAGER_STATUS_MESSAGE = 'NodeManager returned an unexpected status of "{0}"'
+CRITICAL_NODEMANAGER_UNKNOWN_JSON_MESSAGE = 'Unable to determine NodeManager health from unexpected JSON response'
+
+NODEMANAGER_DEFAULT_PORT = 8042
+
+def get_tokens():
+ """
+ Returns a tuple of tokens in the format {{site/property}} that will be used
+ to build the dictionary passed into execute
+ """
+ return (NODEMANAGER_HTTP_ADDRESS_KEY,NODEMANAGER_HTTPS_ADDRESS_KEY,
+ YARN_HTTP_POLICY_KEY)
+
+
+def execute(parameters=None, host_name=None):
+ """
+ Returns a tuple containing the result code and a pre-formatted result label
+
+ Keyword arguments:
+ parameters (dictionary): a mapping of parameter key to value
+ host_name (string): the name of this host where the alert is running
+ """
+ result_code = RESULT_CODE_UNKNOWN
+
+ if parameters is None:
+ return (result_code, ['There were no parameters supplied to the script.'])
+
+ scheme = 'http'
+ http_uri = None
+ https_uri = None
+ http_policy = 'HTTP_ONLY'
+
+ if NODEMANAGER_HTTP_ADDRESS_KEY in parameters:
+ http_uri = parameters[NODEMANAGER_HTTP_ADDRESS_KEY]
+
+ if NODEMANAGER_HTTPS_ADDRESS_KEY in parameters:
+ https_uri = parameters[NODEMANAGER_HTTPS_ADDRESS_KEY]
+
+ if YARN_HTTP_POLICY_KEY in parameters:
+ http_policy = parameters[YARN_HTTP_POLICY_KEY]
+
+ # determine the right URI and whether to use SSL
+ uri = http_uri
+ if http_policy == 'HTTPS_ONLY':
+ scheme = 'https'
+
+ if https_uri is not None:
+ uri = https_uri
+
+ label = ''
+ url_response = None
+ node_healthy = 'false'
+ total_time = 0
+
+ # some yarn-site structures don't have the web ui address
+ if uri is None:
+ if host_name is None:
+ host_name = socket.getfqdn()
+
+ uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT)
+
+ try:
+ query = "{0}://{1}/ws/v1/node/info".format(scheme,uri)
+
+ # execute the query for the JSON that includes templeton status
+ url_response = urllib2.urlopen(query)
+ except:
+ label = CRITICAL_CONNECTION_MESSAGE.format(uri)
+ return (RESULT_CODE_CRITICAL, [label])
+
+ # URL response received, parse it
+ try:
+ json_response = json.loads(url_response.read())
+ node_healthy = json_response['nodeInfo']['nodeHealthy']
+
+ # convert boolean to string
+ node_healthy = str(node_healthy)
+ except:
+ return (RESULT_CODE_CRITICAL, [query])
+
+ # proper JSON received, compare against known value
+ if node_healthy.lower() == 'true':
+ result_code = RESULT_CODE_OK
+ label = OK_MESSAGE
+ else:
+ result_code = RESULT_CODE_CRITICAL
+ label = CRITICAL_NODEMANAGER_STATUS_MESSAGE.format(node_healthy)
+
+ return (result_code, [label])
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/files/validateYarnComponentStatus.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/files/validateYarnComponentStatus.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/files/validateYarnComponentStatus.py
new file mode 100644
index 0000000..33ed8b1
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/files/validateYarnComponentStatus.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import optparse
+import subprocess
+import json
+
+RESOURCEMANAGER = 'rm'
+NODEMANAGER = 'nm'
+HISTORYSERVER = 'hs'
+
+STARTED_STATE = 'STARTED'
+RUNNING_STATE = 'RUNNING'
+
+#Return reponse for given path and address
+def getResponse(path, address, ssl_enabled):
+
+ command = "curl"
+ httpGssnegotiate = "--negotiate"
+ userpswd = "-u:"
+ insecure = "-k"# This is smoke test, no need to check CA of server
+ if ssl_enabled:
+ url = 'https://' + address + path
+ else:
+ url = 'http://' + address + path
+
+ command_with_flags = [command,httpGssnegotiate,userpswd,insecure,url]
+
+ proc = subprocess.Popen(command_with_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (stdout, stderr) = proc.communicate()
+ response = json.loads(stdout)
+ if response == None:
+ print 'There is no response for url: ' + str(url)
+ raise Exception('There is no response for url: ' + str(url))
+ return response
+
+#Verify that REST api is available for given component
+def validateAvailability(component, path, addresses, ssl_enabled):
+ responses = {}
+ for address in addresses.split(','):
+ try:
+ responses[address] = getResponse(path, address, ssl_enabled)
+ except Exception as e:
+ print 'Error checking availability status of component.', e
+
+ if not responses:
+ exit(1)
+
+ is_valid = validateAvailabilityResponse(component, responses.values()[0])
+ if not is_valid:
+ exit(1)
+
+#Validate component-specific response
+def validateAvailabilityResponse(component, response):
+ try:
+ if component == RESOURCEMANAGER:
+ rm_state = response['clusterInfo']['state']
+ if rm_state == STARTED_STATE:
+ return True
+ else:
+ print 'Resourcemanager is not started'
+ return False
+
+ elif component == NODEMANAGER:
+ node_healthy = bool(response['nodeInfo']['nodeHealthy'])
+ if node_healthy:
+ return True
+ else:
+ return False
+ elif component == HISTORYSERVER:
+ hs_start_time = response['historyInfo']['startedOn']
+ if hs_start_time > 0:
+ return True
+ else:
+ return False
+ else:
+ return False
+ except Exception as e:
+ print 'Error validation of availability response for ' + str(component), e
+ return False
+
+#Verify that component has required resources to work
+def validateAbility(component, path, addresses, ssl_enabled):
+ responses = {}
+ for address in addresses.split(','):
+ try:
+ responses[address] = getResponse(path, address, ssl_enabled)
+ except Exception as e:
+ print 'Error checking ability of component.', e
+
+ if not responses:
+ exit(1)
+
+ is_valid = validateAbilityResponse(component, responses.values()[0])
+ if not is_valid:
+ exit(1)
+
+#Validate component-specific response that it has required resources to work
+def validateAbilityResponse(component, response):
+ try:
+ if component == RESOURCEMANAGER:
+ nodes = []
+ if response.has_key('nodes') and not response['nodes'] == None and response['nodes'].has_key('node'):
+ nodes = response['nodes']['node']
+ connected_nodes_count = len(nodes)
+ if connected_nodes_count == 0:
+ print 'There is no connected nodemanagers to resourcemanager'
+ return False
+ active_nodes = filter(lambda x: x['state'] == RUNNING_STATE, nodes)
+ active_nodes_count = len(active_nodes)
+
+ if connected_nodes_count == 0:
+ print 'There is no connected active nodemanagers to resourcemanager'
+ return False
+ else:
+ return True
+ else:
+ return False
+ except Exception as e:
+ print 'Error validation of ability response', e
+ return False
+
+#
+# Main.
+#
+def main():
+ parser = optparse.OptionParser(usage="usage: %prog [options] component ")
+ parser.add_option("-p", "--port", dest="address", help="Host:Port for REST API of a desired component")
+ parser.add_option("-s", "--ssl", dest="ssl_enabled", help="Is SSL enabled for UI of component")
+
+ (options, args) = parser.parse_args()
+
+ component = args[0]
+
+ address = options.address
+ ssl_enabled = (options.ssl_enabled) in 'true'
+ if component == RESOURCEMANAGER:
+ path = '/ws/v1/cluster/info'
+ elif component == NODEMANAGER:
+ path = '/ws/v1/node/info'
+ elif component == HISTORYSERVER:
+ path = '/ws/v1/history/info'
+ else:
+ parser.error("Invalid component")
+
+ validateAvailability(component, path, address, ssl_enabled)
+
+ if component == RESOURCEMANAGER:
+ path = '/ws/v1/cluster/nodes'
+ validateAbility(component, path, address, ssl_enabled)
+
+if __name__ == "__main__":
+ main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/__init__.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/__init__.py
new file mode 100644
index 0000000..35de4bb
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/__init__.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py
new file mode 100644
index 0000000..e9c0fa1
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py
@@ -0,0 +1,132 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.functions.security_commons import build_expectations, \
+ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties,\
+ FILE_TYPE_XML
+from resource_management.libraries.functions.format import format
+
+from yarn import yarn
+from service import service
+
+class ApplicationTimelineServer(Script):
+
+ def install(self, env):
+ self.install_packages(env)
+ #self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ yarn(name='apptimelineserver')
+
+ def pre_rolling_restart(self, env):
+ Logger.info("Executing Rolling Upgrade pre-restart")
+ import params
+ env.set_params(params)
+
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ Execute(format("hdp-select set hadoop-yarn-timelineserver {version}"))
+
+ def start(self, env, rolling_restart=False):
+ import params
+ env.set_params(params)
+ self.configure(env) # FOR SECURITY
+ service('timelineserver', action='start')
+
+ def stop(self, env, rolling_restart=False):
+ import params
+ env.set_params(params)
+ service('timelineserver', action='stop')
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ Execute(format("mv {yarn_historyserver_pid_file_old} {yarn_historyserver_pid_file}"),
+ only_if = format("test -e {yarn_historyserver_pid_file_old}", user=status_params.yarn_user))
+ functions.check_process_status(status_params.yarn_historyserver_pid_file)
+
+ def security_status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ props_value_check = {"yarn.timeline-service.enabled": "true",
+ "yarn.timeline-service.http-authentication.type": "kerberos",
+ "yarn.acl.enable": "true"}
+ props_empty_check = ["yarn.timeline-service.principal",
+ "yarn.timeline-service.keytab",
+ "yarn.timeline-service.http-authentication.kerberos.principal",
+ "yarn.timeline-service.http-authentication.kerberos.keytab"]
+
+ props_read_check = ["yarn.timeline-service.keytab",
+ "yarn.timeline-service.http-authentication.kerberos.keytab"]
+ yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
+ props_read_check)
+
+ yarn_expectations ={}
+ yarn_expectations.update(yarn_site_props)
+
+ security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
+ {'yarn-site.xml': FILE_TYPE_XML})
+ result_issues = validate_security_config_properties(security_params, yarn_expectations)
+ if not result_issues: # If all validations passed successfully
+ try:
+ # Double check the dict before calling execute
+ if ( 'yarn-site' not in security_params
+ or 'yarn.timeline-service.keytab' not in security_params['yarn-site']
+ or 'yarn.timeline-service.principal' not in security_params['yarn-site']) \
+ or 'yarn.timeline-service.http-authentication.kerberos.keytab' not in security_params['yarn-site'] \
+ or 'yarn.timeline-service.http-authentication.kerberos.principal' not in security_params['yarn-site']:
+ self.put_structured_out({"securityState": "UNSECURED"})
+ self.put_structured_out(
+ {"securityIssuesFound": "Keytab file or principal are not set property."})
+ return
+
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.yarn_user,
+ security_params['yarn-site']['yarn.timeline-service.keytab'],
+ security_params['yarn-site']['yarn.timeline-service.principal'],
+ status_params.hostname,
+ status_params.tmp_dir,
+ 30)
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.yarn_user,
+ security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.keytab'],
+ security_params['yarn-site']['yarn.timeline-service.http-authentication.kerberos.principal'],
+ status_params.hostname,
+ status_params.tmp_dir,
+ 30)
+ self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+ except Exception as e:
+ self.put_structured_out({"securityState": "ERROR"})
+ self.put_structured_out({"securityStateErrorInfo": str(e)})
+ else:
+ issues = []
+ for cf in result_issues:
+ issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+ self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+ self.put_structured_out({"securityState": "UNSECURED"})
+
+
+if __name__ == "__main__":
+ ApplicationTimelineServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
new file mode 100644
index 0000000..66c2143
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
@@ -0,0 +1,123 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.security_commons import build_expectations, \
+ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+ FILE_TYPE_XML
+
+from yarn import yarn
+from service import service
+
+class HistoryServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ yarn(name="historyserver")
+
+ def pre_rolling_restart(self, env):
+ Logger.info("Executing Rolling Upgrade pre-restart")
+ import params
+ env.set_params(params)
+
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ Execute(format("hdp-select set hadoop-mapreduce-historyserver {version}"))
+ copy_tarballs_to_hdfs('mapreduce', params.mapred_user, params.hdfs_user, params.user_group)
+
+ def start(self, env, rolling_restart=False):
+ import params
+ env.set_params(params)
+ self.configure(env) # FOR SECURITY
+ copy_tarballs_to_hdfs('mapreduce', params.mapred_user, params.hdfs_user, params.user_group)
+ service('historyserver', action='start', serviceName='mapreduce')
+
+ def stop(self, env, rolling_restart=False):
+ import params
+ env.set_params(params)
+ service('historyserver', action='stop', serviceName='mapreduce')
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.mapred_historyserver_pid_file)
+
+ def security_status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ expectations = {}
+ expectations.update(build_expectations('mapred-site',
+ None,
+ [
+ 'mapreduce.jobhistory.keytab',
+ 'mapreduce.jobhistory.principal',
+ 'mapreduce.jobhistory.webapp.spnego-keytab-file',
+ 'mapreduce.jobhistory.webapp.spnego-principal'
+ ],
+ None))
+
+ security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
+ {'mapred-site.xml': FILE_TYPE_XML})
+ result_issues = validate_security_config_properties(security_params, expectations)
+ if not result_issues: # If all validations passed successfully
+ try:
+ # Double check the dict before calling execute
+ if ( 'mapred-site' not in security_params or
+ 'mapreduce.jobhistory.keytab' not in security_params['mapred-site'] or
+ 'mapreduce.jobhistory.principal' not in security_params['mapred-site'] or
+ 'mapreduce.jobhistory.webapp.spnego-keytab-file' not in security_params['mapred-site'] or
+ 'mapreduce.jobhistory.webapp.spnego-principal' not in security_params['mapred-site']):
+ self.put_structured_out({"securityState": "UNSECURED"})
+ self.put_structured_out(
+ {"securityIssuesFound": "Keytab file or principal not set."})
+ return
+
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.mapred_user,
+ security_params['mapred-site']['mapreduce.jobhistory.keytab'],
+ security_params['mapred-site']['mapreduce.jobhistory.principal'],
+ status_params.hostname,
+ status_params.tmp_dir)
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.mapred_user,
+ security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-keytab-file'],
+ security_params['mapred-site']['mapreduce.jobhistory.webapp.spnego-principal'],
+ status_params.hostname,
+ status_params.tmp_dir)
+ self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+ except Exception as e:
+ self.put_structured_out({"securityState": "ERROR"})
+ self.put_structured_out({"securityStateErrorInfo": str(e)})
+ else:
+ issues = []
+ for cf in result_issues:
+ issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+ self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+ self.put_structured_out({"securityState": "UNSECURED"})
+
+if __name__ == "__main__":
+ HistoryServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapred_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapred_service_check.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapred_service_check.py
new file mode 100644
index 0000000..a9f4367
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapred_service_check.py
@@ -0,0 +1,77 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+class MapReduce2ServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ jar_path = format("{hadoop_mapred2_jar_location}/{hadoopMapredExamplesJarName}")
+ input_file = format("/user/{smokeuser}/mapredsmokeinput")
+ output_file = format("/user/{smokeuser}/mapredsmokeoutput")
+
+ cleanup_cmd = format("fs -rm -r -f {output_file} {input_file}")
+ create_file_cmd = format("fs -put /etc/passwd {input_file}")
+ test_cmd = format("fs -test -e {output_file}")
+ run_wordcount_job = format("jar {jar_path} wordcount {input_file} {output_file}")
+
+ if params.security_enabled:
+ kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};")
+
+ Execute(kinit_cmd,
+ user=params.smokeuser
+ )
+
+ ExecuteHadoop(cleanup_cmd,
+ tries=1,
+ try_sleep=5,
+ user=params.smokeuser,
+ bin_dir=params.execute_path,
+ conf_dir=params.hadoop_conf_dir
+ )
+
+ ExecuteHadoop(create_file_cmd,
+ tries=1,
+ try_sleep=5,
+ user=params.smokeuser,
+ bin_dir=params.execute_path,
+ conf_dir=params.hadoop_conf_dir
+ )
+
+ ExecuteHadoop(run_wordcount_job,
+ tries=1,
+ try_sleep=5,
+ user=params.smokeuser,
+ bin_dir=params.execute_path,
+ conf_dir=params.hadoop_conf_dir,
+ logoutput=True
+ )
+
+ ExecuteHadoop(test_cmd,
+ user=params.smokeuser,
+ bin_dir=params.execute_path,
+ conf_dir=params.hadoop_conf_dir
+ )
+
+if __name__ == "__main__":
+ MapReduce2ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py
new file mode 100644
index 0000000..c4e2800
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py
@@ -0,0 +1,49 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import sys
+from resource_management import *
+
+from yarn import yarn
+
+class MapReduce2Client(Script):
+
+ def pre_rolling_restart(self, env):
+ import params
+ env.set_params(params)
+
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ Execute(format("hdp-select set hadoop-client {version}"))
+
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ yarn()
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+if __name__ == "__main__":
+ MapReduce2Client().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
new file mode 100644
index 0000000..87be214
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
@@ -0,0 +1,137 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import nodemanager_upgrade
+
+from resource_management import *
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.security_commons import build_expectations, \
+ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+ FILE_TYPE_XML
+
+from yarn import yarn
+from service import service
+
+class Nodemanager(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ yarn(name="nodemanager")
+
+ def pre_rolling_restart(self, env):
+ Logger.info("Executing NodeManager Rolling Upgrade pre-restart")
+ import params
+ env.set_params(params)
+
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ Execute(format("hdp-select set hadoop-yarn-nodemanager {version}"))
+
+ def start(self, env, rolling_restart=False):
+ import params
+ env.set_params(params)
+ self.configure(env) # FOR SECURITY
+ service('nodemanager',action='start')
+
+ def post_rolling_restart(self, env):
+ Logger.info("Executing NodeManager Rolling Upgrade post-restart")
+ import params
+ env.set_params(params)
+
+ nodemanager_upgrade.post_upgrade_check()
+
+ def stop(self, env, rolling_restart=False):
+ import params
+ env.set_params(params)
+
+ service('nodemanager',action='stop')
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.nodemanager_pid_file)
+
+ def security_status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ props_value_check = {"yarn.timeline-service.enabled": "true",
+ "yarn.timeline-service.http-authentication.type": "kerberos",
+ "yarn.acl.enable": "true"}
+ props_empty_check = ["yarn.nodemanager.principal",
+ "yarn.nodemanager.keytab",
+ "yarn.nodemanager.webapp.spnego-principal",
+ "yarn.nodemanager.webapp.spnego-keytab-file"]
+
+ props_read_check = ["yarn.nodemanager.keytab",
+ "yarn.nodemanager.webapp.spnego-keytab-file"]
+ yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
+ props_read_check)
+
+ yarn_expectations ={}
+ yarn_expectations.update(yarn_site_props)
+
+ security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
+ {'yarn-site.xml': FILE_TYPE_XML})
+ result_issues = validate_security_config_properties(security_params, yarn_site_props)
+ if not result_issues: # If all validations passed successfully
+ try:
+ # Double check the dict before calling execute
+ if ( 'yarn-site' not in security_params
+ or 'yarn.nodemanager.keytab' not in security_params['yarn-site']
+ or 'yarn.nodemanager.principal' not in security_params['yarn-site']) \
+ or 'yarn.nodemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \
+ or 'yarn.nodemanager.webapp.spnego-principal' not in security_params['yarn-site']:
+ self.put_structured_out({"securityState": "UNSECURED"})
+ self.put_structured_out(
+ {"securityIssuesFound": "Keytab file or principal are not set property."})
+ return
+
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.yarn_user,
+ security_params['yarn-site']['yarn.nodemanager.keytab'],
+ security_params['yarn-site']['yarn.nodemanager.principal'],
+ status_params.hostname,
+ status_params.tmp_dir,
+ 30)
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.yarn_user,
+ security_params['yarn-site']['yarn.nodemanager.webapp.spnego-keytab-file'],
+ security_params['yarn-site']['yarn.nodemanager.webapp.spnego-principal'],
+ status_params.hostname,
+ status_params.tmp_dir,
+ 30)
+ self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+ except Exception as e:
+ self.put_structured_out({"securityState": "ERROR"})
+ self.put_structured_out({"securityStateErrorInfo": str(e)})
+ else:
+ issues = []
+ for cf in result_issues:
+ issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+ self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+ self.put_structured_out({"securityState": "UNSECURED"})
+
+if __name__ == "__main__":
+ Nodemanager().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
new file mode 100644
index 0000000..54e0fae
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
@@ -0,0 +1,74 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import subprocess
+
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+from resource_management.core.resources.system import Execute
+from resource_management.core import shell
+from resource_management.libraries.functions.decorator import retry
+
+
+def post_upgrade_check():
+ '''
+ Checks that the NodeManager has rejoined the cluster.
+ This function will obtain the Kerberos ticket if security is enabled.
+ :return:
+ '''
+ import params
+
+ Logger.info('NodeManager executing "yarn node -list -states=RUNNING" to verify the node has rejoined the cluster...')
+ if params.security_enabled and params.nodemanager_kinit_cmd:
+ Execute(params.nodemanager_kinit_cmd, user = params.yarn_user)
+
+ _check_nodemanager_startup()
+
+
+@retry(times=12, sleep_time=10, err_class=Fail)
+def _check_nodemanager_startup():
+ '''
+ Checks that a NodeManager is in a RUNNING state in the cluster via
+ "yarn node -list -states=RUNNING" command. Once the NodeManager is found to be
+ alive this method will return, otherwise it will raise a Fail(...) and retry
+ automatically.
+ :return:
+ '''
+ import params
+
+ command = 'yarn node -list -states=RUNNING'
+
+ try:
+ # 'su - yarn -c "yarn node -status c6401.ambari.apache.org:45454"'
+ return_code, yarn_output = shell.call(command, user=params.hdfs_user)
+ except:
+ raise Fail('Unable to determine if the NodeManager has started after upgrade.')
+
+ if return_code == 0:
+ hostname = params.hostname.lower()
+ nodemanager_address = params.nm_address.lower()
+ yarn_output = yarn_output.lower()
+
+ if hostname in yarn_output or nodemanager_address in yarn_output:
+ Logger.info('NodeManager with ID {0} has rejoined the cluster.'.format(nodemanager_address))
+ return
+ else:
+ raise Fail('NodeManager with ID {0} was not found in the list of running NodeManagers'.format(nodemanager_address))
+
+ raise Fail('Unable to determine if the NodeManager has started after upgrade (result code {0})'.format(str(return_code)))
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
new file mode 100644
index 0000000..d9c73bf
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
@@ -0,0 +1,209 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import os
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
+from resource_management import *
+import status_params
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+# This is expected to be of the form #.#.#.#
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+hostname = config['hostname']
+
+#hadoop params
+if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
+ yarn_role_root = "hadoop-yarn-client"
+ mapred_role_root = "hadoop-mapreduce-client"
+
+ command_role = default("/role", "")
+ if command_role == "APP_TIMELINE_SERVER":
+ yarn_role_root = "hadoop-yarn-timelineserver"
+ elif command_role == "HISTORYSERVER":
+ mapred_role_root = "hadoop-mapreduce-historyserver"
+ elif command_role == "MAPREDUCE2_CLIENT":
+ mapred_role_root = "hadoop-mapreduce-client"
+ elif command_role == "NODEMANAGER":
+ yarn_role_root = "hadoop-yarn-nodemanager"
+ elif command_role == "RESOURCEMANAGER":
+ yarn_role_root = "hadoop-yarn-resourcemanager"
+ elif command_role == "YARN_CLIENT":
+ yarn_role_root = "hadoop-yarn-client"
+
+ hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
+ hadoop_bin = "/usr/hdp/current/hadoop-client/sbin"
+ hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
+
+ hadoop_mapred2_jar_location = format("/usr/hdp/current/{mapred_role_root}")
+ mapred_bin = format("/usr/hdp/current/{mapred_role_root}/sbin")
+
+ hadoop_yarn_home = format("/usr/hdp/current/{yarn_role_root}")
+ yarn_bin = format("/usr/hdp/current/{yarn_role_root}/sbin")
+ yarn_container_bin = format("/usr/hdp/current/{yarn_role_root}/bin")
+else:
+ hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
+ hadoop_bin = "/usr/lib/hadoop/sbin"
+ hadoop_bin_dir = "/usr/bin"
+ hadoop_yarn_home = '/usr/lib/hadoop-yarn'
+ hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce"
+ mapred_bin = "/usr/lib/hadoop-mapreduce/sbin"
+ yarn_bin = "/usr/lib/hadoop-yarn/sbin"
+ yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
+
+hadoop_conf_dir = "/etc/hadoop/conf"
+limits_conf_dir = "/etc/security/limits.d"
+execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir + os.pathsep + yarn_container_bin
+
+ulimit_cmd = "ulimit -c unlimited;"
+
+mapred_user = status_params.mapred_user
+yarn_user = status_params.yarn_user
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+rm_hosts = config['clusterHostInfo']['rm_host']
+rm_host = rm_hosts[0]
+rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1]
+rm_https_port = "8090"
+# TODO UPGRADE default, update site during upgrade
+rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
+
+java64_home = config['hostLevelParams']['java_home']
+hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False)
+
+yarn_heapsize = config['configurations']['yarn-env']['yarn_heapsize']
+resourcemanager_heapsize = config['configurations']['yarn-env']['resourcemanager_heapsize']
+nodemanager_heapsize = config['configurations']['yarn-env']['nodemanager_heapsize']
+apptimelineserver_heapsize = default("/configurations/yarn-env/apptimelineserver_heapsize", 1024)
+ats_leveldb_dir = config['configurations']['yarn-site']['yarn.timeline-service.leveldb-timeline-store.path']
+yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix']
+yarn_pid_dir_prefix = status_params.yarn_pid_dir_prefix
+mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix
+mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix']
+mapred_env_sh_template = config['configurations']['mapred-env']['content']
+yarn_env_sh_template = config['configurations']['yarn-env']['content']
+
+if len(rm_hosts) > 1:
+ additional_rm_host = rm_hosts[1]
+ rm_webui_address = format("{rm_host}:{rm_port},{additional_rm_host}:{rm_port}")
+ rm_webui_https_address = format("{rm_host}:{rm_https_port},{additional_rm_host}:{rm_https_port}")
+else:
+ rm_webui_address = format("{rm_host}:{rm_port}")
+ rm_webui_https_address = format("{rm_host}:{rm_https_port}")
+
+nm_webui_address = config['configurations']['yarn-site']['yarn.nodemanager.webapp.address']
+hs_webui_address = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address']
+nm_address = config['configurations']['yarn-site']['yarn.nodemanager.address'] # still contains 0.0.0.0
+if hostname and nm_address and nm_address.startswith("0.0.0.0:"):
+ nm_address = nm_address.replace("0.0.0.0", hostname)
+
+nm_local_dirs = config['configurations']['yarn-site']['yarn.nodemanager.local-dirs']
+nm_log_dirs = config['configurations']['yarn-site']['yarn.nodemanager.log-dirs']
+
+distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar"
+hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
+
+yarn_pid_dir = status_params.yarn_pid_dir
+mapred_pid_dir = status_params.mapred_pid_dir
+
+mapred_log_dir = format("{mapred_log_dir_prefix}/{mapred_user}")
+yarn_log_dir = format("{yarn_log_dir_prefix}/{yarn_user}")
+mapred_job_summary_log = format("{mapred_log_dir_prefix}/{mapred_user}/hadoop-mapreduce.jobsummary.log")
+yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduce.jobsummary.log")
+
+user_group = config['configurations']['cluster-env']['user_group']
+
+#exclude file
+exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
+exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
+
+ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
+has_ats = not len(ats_host) == 0
+
+# default kinit commands
+rm_kinit_cmd = ""
+yarn_timelineservice_kinit_cmd = ""
+nodemanager_kinit_cmd = ""
+
+if security_enabled:
+ _rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal']
+ _rm_principal_name = _rm_principal_name.replace('_HOST',hostname.lower())
+ _rm_keytab = config['configurations']['yarn-site']['yarn.resourcemanager.keytab']
+ rm_kinit_cmd = format("{kinit_path_local} -kt {_rm_keytab} {_rm_principal_name};")
+
+ # YARN timeline security options are only available in HDP Champlain
+ if has_ats:
+ _yarn_timelineservice_principal_name = config['configurations']['yarn-site']['yarn.timeline-service.principal']
+ _yarn_timelineservice_principal_name = _yarn_timelineservice_principal_name.replace('_HOST', hostname.lower())
+ _yarn_timelineservice_keytab = config['configurations']['yarn-site']['yarn.timeline-service.keytab']
+ yarn_timelineservice_kinit_cmd = format("{kinit_path_local} -kt {_yarn_timelineservice_keytab} {_yarn_timelineservice_principal_name};")
+
+ if 'yarn.nodemanager.principal' in config['configurations']['yarn-site']:
+ _nodemanager_principal_name = config['configurations']['yarn-site']['yarn.nodemanager.principal']
+ _nodemanager_keytab = config['configurations']['yarn-site']['yarn.nodemanager.keytab']
+ nodemanager_kinit_cmd = format("{kinit_path_local} -kt {_nodemanager_keytab} {_nodemanager_principal_name};")
+
+
+yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable']
+yarn_nm_app_log_dir = config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir']
+mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir']
+mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir']
+jobhistory_heapsize = default("/configurations/mapred-env/jobhistory_heapsize", "900")
+
+#for create_hdfs_directory
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+import functools
+#create partial functions with common arguments for every HdfsDirectory call
+#to create hdfs directory we need to call params.HdfsDirectory in code
+HdfsDirectory = functools.partial(
+ HdfsDirectory,
+ conf_dir=hadoop_conf_dir,
+ hdfs_user=hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ bin_dir = hadoop_bin_dir
+)
+update_exclude_file_only = default("/commandParams/update_exclude_file_only",False)
+
+mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group)
+
+#taskcontroller.cfg
+
+mapred_local_dir = "/tmp/hadoop-mapred/mapred/local"
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+min_user_id = config['configurations']['yarn-env']['min_user_id']
+
+# Node labels
+node_labels_dir = default("/configurations/yarn-site/yarn.node-labels.fs-store.root-dir", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
new file mode 100644
index 0000000..3d6a00e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
@@ -0,0 +1,173 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.functions.security_commons import build_expectations, \
+ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+ FILE_TYPE_XML
+
+from yarn import yarn
+from service import service
+
+
+class Resourcemanager(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+
+ env.set_params(params)
+ yarn(name='resourcemanager')
+
+ def pre_rolling_restart(self, env):
+ Logger.info("Executing Rolling Upgrade post-restart")
+ import params
+ env.set_params(params)
+
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ Execute(format("hdp-select set hadoop-yarn-resourcemanager {version}"))
+
+ def start(self, env, rolling_restart=False):
+ import params
+
+ env.set_params(params)
+ self.configure(env) # FOR SECURITY
+ service('resourcemanager',
+ action='start'
+ )
+
+ def stop(self, env, rolling_restart=False):
+ import params
+
+ env.set_params(params)
+
+ service('resourcemanager',
+ action='stop'
+ )
+
+ def status(self, env):
+ import status_params
+
+ env.set_params(status_params)
+ check_process_status(status_params.resourcemanager_pid_file)
+ pass
+
+ def security_status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ props_value_check = {"yarn.timeline-service.enabled": "true",
+ "yarn.timeline-service.http-authentication.type": "kerberos",
+ "yarn.acl.enable": "true"}
+ props_empty_check = ["yarn.resourcemanager.principal",
+ "yarn.resourcemanager.keytab",
+ "yarn.resourcemanager.webapp.spnego-principal",
+ "yarn.resourcemanager.webapp.spnego-keytab-file"]
+
+ props_read_check = ["yarn.resourcemanager.keytab",
+ "yarn.resourcemanager.webapp.spnego-keytab-file"]
+ yarn_site_props = build_expectations('yarn-site', props_value_check, props_empty_check,
+ props_read_check)
+
+ yarn_expectations ={}
+ yarn_expectations.update(yarn_site_props)
+
+ security_params = get_params_from_filesystem(status_params.hadoop_conf_dir,
+ {'yarn-site.xml': FILE_TYPE_XML})
+ result_issues = validate_security_config_properties(security_params, yarn_site_props)
+ if not result_issues: # If all validations passed successfully
+ try:
+ # Double check the dict before calling execute
+ if ( 'yarn-site' not in security_params
+ or 'yarn.resourcemanager.keytab' not in security_params['yarn-site']
+ or 'yarn.resourcemanager.principal' not in security_params['yarn-site']) \
+ or 'yarn.resourcemanager.webapp.spnego-keytab-file' not in security_params['yarn-site'] \
+ or 'yarn.resourcemanager.webapp.spnego-principal' not in security_params['yarn-site']:
+ self.put_structured_out({"securityState": "UNSECURED"})
+ self.put_structured_out(
+ {"securityIssuesFound": "Keytab file or principal are not set property."})
+ return
+
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.yarn_user,
+ security_params['yarn-site']['yarn.resourcemanager.keytab'],
+ security_params['yarn-site']['yarn.resourcemanager.principal'],
+ status_params.hostname,
+ status_params.tmp_dir,
+ 30)
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.yarn_user,
+ security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-keytab-file'],
+ security_params['yarn-site']['yarn.resourcemanager.webapp.spnego-principal'],
+ status_params.hostname,
+ status_params.tmp_dir,
+ 30)
+ self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+ except Exception as e:
+ self.put_structured_out({"securityState": "ERROR"})
+ self.put_structured_out({"securityStateErrorInfo": str(e)})
+ else:
+ issues = []
+ for cf in result_issues:
+ issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+ self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+ self.put_structured_out({"securityState": "UNSECURED"})
+
+
+ def refreshqueues(self, env):
+ import params
+
+ self.configure(env)
+ env.set_params(params)
+
+ service('resourcemanager',
+ action='refreshQueues'
+ )
+
+ def decommission(self, env):
+ import params
+
+ env.set_params(params)
+ rm_kinit_cmd = params.rm_kinit_cmd
+ yarn_user = params.yarn_user
+ conf_dir = params.hadoop_conf_dir
+ user_group = params.user_group
+
+ yarn_refresh_cmd = format("{rm_kinit_cmd} yarn --config {conf_dir} rmadmin -refreshNodes")
+
+ File(params.exclude_file_path,
+ content=Template("exclude_hosts_list.j2"),
+ owner=yarn_user,
+ group=user_group
+ )
+
+ if params.update_exclude_file_only == False:
+ Execute(yarn_refresh_cmd,
+ environment= {'PATH' : params.execute_path },
+ user=yarn_user)
+ pass
+ pass
+
+
+if __name__ == "__main__":
+ Resourcemanager().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
new file mode 100644
index 0000000..1002094
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
@@ -0,0 +1,76 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+
+def service(componentName, action='start', serviceName='yarn'):
+
+ import params
+
+ if serviceName == 'mapreduce' and componentName == 'historyserver':
+ daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh")
+ pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid")
+ usr = params.mapred_user
+ else:
+ daemon = format("{yarn_bin}/yarn-daemon.sh")
+ pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-{componentName}.pid")
+ usr = params.yarn_user
+
+ cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}")
+
+ if action == 'start':
+ daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}")
+ check_process = format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1")
+
+ # Remove the pid file if its corresponding process is not running.
+ File(pid_file,
+ action="delete",
+ not_if=check_process)
+
+ # Attempt to start the process. Internally, this is skipped if the process is already running.
+ Execute(daemon_cmd,
+ user=usr,
+ not_if=check_process
+ )
+
+ # Ensure that the process with the expected PID exists.
+ Execute(check_process,
+ user=usr,
+ not_if=check_process,
+ initial_wait=5
+ )
+
+ elif action == 'stop':
+ daemon_cmd = format("{cmd} stop {componentName}")
+ Execute(daemon_cmd,
+ user=usr)
+
+ File(pid_file,
+ action="delete")
+
+ elif action == 'refreshQueues':
+ rm_kinit_cmd = params.rm_kinit_cmd
+ refresh_cmd = format("{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues")
+
+ Execute(refresh_cmd,
+ user=usr,
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
new file mode 100644
index 0000000..7189664
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
@@ -0,0 +1,68 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+import sys
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ run_yarn_check_cmd = format("yarn --config {hadoop_conf_dir} node -list")
+
+ component_type = 'rm'
+ if params.hadoop_ssl_enabled:
+ component_address = params.rm_webui_https_address
+ else:
+ component_address = params.rm_webui_address
+
+ validateStatusFileName = "validateYarnComponentStatus.py"
+ validateStatusFilePath = format("{tmp_dir}/{validateStatusFileName}")
+ python_executable = sys.executable
+ validateStatusCmd = format("{python_executable} {validateStatusFilePath} {component_type} -p {component_address} -s {hadoop_ssl_enabled}")
+
+ if params.security_enabled:
+ kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};")
+ smoke_cmd = format("{kinit_cmd} {validateStatusCmd}")
+ else:
+ smoke_cmd = validateStatusCmd
+
+ File(validateStatusFilePath,
+ content=StaticFile(validateStatusFileName),
+ mode=0755
+ )
+
+ Execute(smoke_cmd,
+ tries=3,
+ try_sleep=5,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ user=params.smokeuser,
+ logoutput=True
+ )
+
+ Execute(run_yarn_check_cmd,
+ path=params.execute_path,
+ user=params.smokeuser
+ )
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py
new file mode 100644
index 0000000..3cfff05
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+mapred_user = config['configurations']['mapred-env']['mapred_user']
+yarn_user = config['configurations']['yarn-env']['yarn_user']
+yarn_pid_dir_prefix = config['configurations']['yarn-env']['yarn_pid_dir_prefix']
+mapred_pid_dir_prefix = config['configurations']['mapred-env']['mapred_pid_dir_prefix']
+yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}")
+mapred_pid_dir = format("{mapred_pid_dir_prefix}/{mapred_user}")
+
+resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanager.pid")
+nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid")
+yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid")
+yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated
+mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid")
+
+# Security related/required params
+hadoop_conf_dir = "/etc/hadoop/conf"
+hostname = config['hostname']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+security_enabled = config['configurations']['cluster-env']['security_enabled']
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
new file mode 100644
index 0000000..cf0d211
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -0,0 +1,249 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+import sys
+import os
+
+
+def yarn(name = None):
+ import params
+
+
+ if name in ["nodemanager","historyserver"]:
+ if params.yarn_log_aggregation_enabled:
+ params.HdfsDirectory(params.yarn_nm_app_log_dir,
+ action="create_delayed",
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0777,
+ recursive_chmod=True
+ )
+ params.HdfsDirectory("/mapred",
+ action="create_delayed",
+ owner=params.mapred_user
+ )
+ params.HdfsDirectory("/mapred/system",
+ action="create_delayed",
+ owner=params.hdfs_user
+ )
+ params.HdfsDirectory(params.mapreduce_jobhistory_intermediate_done_dir,
+ action="create_delayed",
+ owner=params.mapred_user,
+ group=params.user_group,
+ mode=0777
+ )
+
+ params.HdfsDirectory(params.mapreduce_jobhistory_done_dir,
+ action="create_delayed",
+ owner=params.mapred_user,
+ group=params.user_group,
+ mode=01777
+ )
+ params.HdfsDirectory(None, action="create")
+
+ if name == "nodemanager":
+ Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
+ owner=params.yarn_user,
+ group=params.user_group,
+ recursive=True,
+ recursive_permission=True,
+ ignore_failures=True,
+ mode=0775
+ )
+
+ Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
+ owner=params.yarn_user,
+ group=params.user_group,
+ recursive=True
+ )
+
+ Directory([params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir],
+ owner=params.mapred_user,
+ group=params.user_group,
+ recursive=True
+ )
+ Directory([params.yarn_log_dir_prefix],
+ owner=params.yarn_user,
+ recursive=True,
+ ignore_failures=True,
+ )
+
+ XmlConfig("core-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ XmlConfig("mapred-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['mapred-site'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-site'],
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ XmlConfig("yarn-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['yarn-site'],
+ configuration_attributes=params.config['configuration_attributes']['yarn-site'],
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ XmlConfig("capacity-scheduler.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['capacity-scheduler'],
+ configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ if name == 'resourcemanager':
+ File(params.yarn_job_summary_log,
+ owner=params.yarn_user,
+ group=params.user_group
+ )
+ if params.node_labels_dir:
+ params.HdfsDirectory(params.node_labels_dir,
+ action="create",
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0700
+ )
+ elif name == 'apptimelineserver':
+ Directory(params.ats_leveldb_dir,
+ owner=params.yarn_user,
+ group=params.user_group,
+ recursive=True,
+ recursive_permission=True
+ )
+
+ File(params.rm_nodes_exclude_path,
+ owner=params.yarn_user,
+ group=params.user_group
+ )
+
+ File(format("{limits_conf_dir}/yarn.conf"),
+ mode=0644,
+ content=Template('yarn.conf.j2')
+ )
+
+ File(format("{limits_conf_dir}/mapreduce.conf"),
+ mode=0644,
+ content=Template('mapreduce.conf.j2')
+ )
+
+ File(format("{hadoop_conf_dir}/yarn-env.sh"),
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0755,
+ content=InlineTemplate(params.yarn_env_sh_template)
+ )
+
+ if params.security_enabled:
+ container_executor = format("{yarn_container_bin}/container-executor")
+ File(container_executor,
+ group=params.yarn_executor_container_group,
+ mode=06050
+ )
+
+ File(format("{hadoop_conf_dir}/container-executor.cfg"),
+ group=params.user_group,
+ mode=0644,
+ content=Template('container-executor.cfg.j2')
+ )
+
+
+ if params.security_enabled:
+ tc_mode = 0644
+ tc_owner = "root"
+ else:
+ tc_mode = None
+ tc_owner = params.hdfs_user
+
+ File(format("{hadoop_conf_dir}/mapred-env.sh"),
+ owner=tc_owner,
+ content=InlineTemplate(params.mapred_env_sh_template)
+ )
+
+ if params.security_enabled:
+ File(os.path.join(params.hadoop_bin, "task-controller"),
+ owner="root",
+ group=params.mapred_tt_group,
+ mode=06050
+ )
+ File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'),
+ owner = tc_owner,
+ mode = tc_mode,
+ group = params.mapred_tt_group,
+ content=Template("taskcontroller.cfg.j2")
+ )
+ else:
+ File(os.path.join(params.hadoop_conf_dir, 'taskcontroller.cfg'),
+ owner=tc_owner,
+ content=Template("taskcontroller.cfg.j2")
+ )
+
+ if "mapred-site" in params.config['configurations']:
+ XmlConfig("mapred-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['mapred-site'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-site'],
+ owner=params.mapred_user,
+ group=params.user_group
+ )
+
+ if "capacity-scheduler" in params.config['configurations']:
+ XmlConfig("capacity-scheduler.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations'][
+ 'capacity-scheduler'],
+ configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+
+ if os.path.exists(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml')):
+ File(os.path.join(params.hadoop_conf_dir, 'fair-scheduler.xml'),
+ owner=params.mapred_user,
+ group=params.user_group
+ )
+
+ if os.path.exists(
+ os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example')):
+ File(os.path.join(params.hadoop_conf_dir, 'ssl-client.xml.example'),
+ owner=params.mapred_user,
+ group=params.user_group
+ )
+
+ if os.path.exists(
+ os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example')):
+ File(os.path.join(params.hadoop_conf_dir, 'ssl-server.xml.example'),
+ owner=params.mapred_user,
+ group=params.user_group
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py
new file mode 100644
index 0000000..60b9bce
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py
@@ -0,0 +1,49 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import sys
+from resource_management import *
+
+from yarn import yarn
+
+class YarnClient(Script):
+
+ def pre_rolling_restart(self, env):
+ import params
+ env.set_params(params)
+
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ Execute(format("hdp-select set hadoop-client {version}"))
+
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ yarn()
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+if __name__ == "__main__":
+ YarnClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/container-executor.cfg.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/container-executor.cfg.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/container-executor.cfg.j2
new file mode 100644
index 0000000..c6f1ff6
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/container-executor.cfg.j2
@@ -0,0 +1,40 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+yarn.nodemanager.local-dirs={{nm_local_dirs}}
+yarn.nodemanager.log-dirs={{nm_log_dirs}}
+yarn.nodemanager.linux-container-executor.group={{yarn_executor_container_group}}
+banned.users=hdfs,yarn,mapred,bin
+min.user.id={{min_user_id}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/exclude_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/exclude_hosts_list.j2
new file mode 100644
index 0000000..c7ce416
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/exclude_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in exclude_hosts %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/mapreduce.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/mapreduce.conf.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/mapreduce.conf.j2
new file mode 100644
index 0000000..b996645
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/mapreduce.conf.j2
@@ -0,0 +1,35 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{mapred_user}} - nofile 32768
+{{mapred_user}} - nproc 65536
http://git-wip-us.apache.org/repos/asf/ambari/blob/53c39154/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/taskcontroller.cfg.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/taskcontroller.cfg.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/taskcontroller.cfg.j2
new file mode 100644
index 0000000..3d5f4f2
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/taskcontroller.cfg.j2
@@ -0,0 +1,38 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+mapred.local.dir={{mapred_local_dir}}
+mapreduce.tasktracker.group={{mapred_tt_group}}
+hadoop.log.dir={{hdfs_log_dir_prefix}}/{{mapred_user}}