You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2017/10/09 18:34:33 UTC
[19/50] [abbrv] ambari git commit: AMBARI-21729 Owner of yarn.include
stays to be root:root where as dfs.include gets changed to the correct one
upon restart services (dsen)
AMBARI-21729 Owner of yarn.include stays to be root:root where as dfs.include gets changed to the correct one upon restart services (dsen)
(cherry picked from commit 640fefb21cc73b7ccbd4dc714c0415dcbb311700)
Change-Id: I102293ceaf8e9815d3a5efb7222449835f64ee52
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d2bab16
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d2bab16
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d2bab16
Branch: refs/heads/2.5-maint
Commit: 6d2bab1615598753b642e35ae2f1148641eac16b
Parents: fd3379f
Author: Dmytro Sen <ds...@apache.org>
Authored: Wed Aug 16 12:35:45 2017 +0300
Committer: Zuul <re...@hortonworks.com>
Committed: Wed Aug 16 11:31:41 2017 -0700
----------------------------------------------------------------------
.../2.1.0.2.0/package/scripts/params_linux.py | 5 +-
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 14 +-
.../stacks/2.0.6/YARN/test_resourcemanager.py | 160 ++-
.../default_yarn_include_file_dont_manage.json | 1260 ++++++++++++++++++
.../default_yarn_include_file_manage.json | 1260 ++++++++++++++++++
.../secured_yarn_include_file_dont_manage.json | 1078 +++++++++++++++
.../secured_yarn_include_file_manage.json | 1078 +++++++++++++++
.../YARN/package/scripts/params_linux.py | 5 +-
.../2.0/services/YARN/package/scripts/yarn.py | 16 +-
9 files changed, 4865 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d2bab16/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index fad1fcb..f217192 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -169,9 +169,6 @@ rm_hosts = config['clusterHostInfo']['rm_host']
rm_host = rm_hosts[0]
rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1]
rm_https_port = default('/configurations/yarn-site/yarn.resourcemanager.webapp.https.address', ":8090").split(':')[-1]
-# TODO UPGRADE default, update site during upgrade
-rm_nodes_exclude_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
-rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path)
java64_home = config['hostLevelParams']['java_home']
java_exec = format("{java64_home}/bin/java")
@@ -234,6 +231,7 @@ user_group = config['configurations']['cluster-env']['user_group']
#exclude file
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
+rm_nodes_exclude_dir = os.path.dirname(exclude_file_path)
nm_hosts = default("/clusterHostInfo/nm_hosts", [])
#incude file
@@ -241,6 +239,7 @@ include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.node
include_hosts = None
manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
if include_file_path and manage_include_files:
+ rm_nodes_include_dir = os.path.dirname(include_file_path)
include_hosts = list(set(nm_hosts) - set(exclude_hosts))
ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d2bab16/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index 3300875..f6e8391 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -296,10 +296,22 @@ def yarn(name=None, config_dir=None):
create_parents=True,
cd_access='a',
)
- File(params.rm_nodes_exclude_path,
+ File(params.exclude_file_path,
+ content=Template("exclude_hosts_list.j2"),
owner=params.yarn_user,
group=params.user_group
)
+ if params.include_hosts:
+ Directory(params.rm_nodes_include_dir,
+ mode=0755,
+ create_parents=True,
+ cd_access='a',
+ )
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=params.yarn_user,
+ group=params.user_group
+ )
File(params.yarn_job_summary_log,
owner=params.yarn_user,
group=params.user_group
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d2bab16/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
index e8b5f78..01b5282 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
@@ -48,6 +48,28 @@ class TestResourceManager(RMFTestCase):
self.assert_configure_default()
self.assertNoMoreResources()
+ def test_configure_default_with_include_file_dont_manage(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+ classname="Resourcemanager",
+ command="configure",
+ config_file="default_yarn_include_file_dont_manage.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default(is_include_file_configured=True, manage_include_files=False)
+ self.assertNoMoreResources()
+
+ def test_configure_default_with_include_file_manage(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+ classname="Resourcemanager",
+ command="configure",
+ config_file="default_yarn_include_file_manage.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default(is_include_file_configured=True, manage_include_files=True)
+ self.assertNoMoreResources()
+
def test_start_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
@@ -98,6 +120,28 @@ class TestResourceManager(RMFTestCase):
)
self.assert_configure_secured()
+ def test_configure_secured_with_include_file_dont_manage(self):
+
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+ classname="Resourcemanager",
+ command="configure",
+ config_file="secured_yarn_include_file_dont_manage.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_secured(is_include_file_configured=True, manage_include_files=False)
+
+ def test_configure_secured_with_include_file_manage(self):
+
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+ classname="Resourcemanager",
+ command="configure",
+ config_file="secured_yarn_include_file_manage.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_secured(is_include_file_configured=True, manage_include_files=True)
+
def test_start_secured(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
classname="Resourcemanager",
@@ -177,7 +221,95 @@ class TestResourceManager(RMFTestCase):
self.assertNoMoreResources()
- def assert_configure_default(self):
+ def test_decommission_default_with_include_file_dont_manage(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+ classname = "Resourcemanager",
+ command = "decommission",
+ config_file="default_yarn_include_file_dont_manage.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+ owner = 'yarn',
+ content = Template('exclude_hosts_list.j2'),
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
+ environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
+ user = 'yarn',
+ )
+ self.assertNoMoreResources()
+
+ def test_decommission_default_with_include_file_manage(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+ classname = "Resourcemanager",
+ command = "decommission",
+ config_file="default_yarn_include_file_manage.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+ owner = 'yarn',
+ content = Template('exclude_hosts_list.j2'),
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
+ owner = 'yarn',
+ content = Template('include_hosts_list.j2'),
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
+ environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
+ user = 'yarn',
+ )
+ self.assertNoMoreResources()
+
+ def test_decommission_secured_with_include_file_dont_manage(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+ classname = "Resourcemanager",
+ command = "decommission",
+ config_file="secured_yarn_include_file_dont_manage.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+ owner = 'yarn',
+ content = Template('exclude_hosts_list.j2'),
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/c6401.ambari.apache.org@EXAMPLE.COM; yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
+ environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
+ user = 'yarn',
+ )
+
+ self.assertNoMoreResources()
+
+ def test_decommission_secured_with_include_file_manage(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/resourcemanager.py",
+ classname = "Resourcemanager",
+ command = "decommission",
+ config_file="secured_yarn_include_file_manage.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+ owner = 'yarn',
+ content = Template('exclude_hosts_list.j2'),
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
+ owner = 'yarn',
+ content = Template('include_hosts_list.j2'),
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/c6401.ambari.apache.org@EXAMPLE.COM; yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
+ environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
+ user = 'yarn',
+ )
+
+ self.assertNoMoreResources()
+
+ def assert_configure_default(self, is_include_file_configured = False, manage_include_files=False):
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
@@ -274,9 +406,21 @@ class TestResourceManager(RMFTestCase):
cd_access = 'a',
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+ content = Template('exclude_hosts_list.j2'),
owner = 'yarn',
group = 'hadoop',
)
+ if is_include_file_configured and manage_include_files:
+ self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include',
+ mode = 0755,
+ create_parents = True,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
+ content = Template('include_hosts_list.j2'),
+ owner = 'yarn',
+ group = 'hadoop',
+ )
self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
owner = 'yarn',
group = 'hadoop',
@@ -346,7 +490,7 @@ class TestResourceManager(RMFTestCase):
group = 'hadoop',
)
- def assert_configure_secured(self):
+ def assert_configure_secured(self, is_include_file_configured = False, manage_include_files=False):
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
@@ -443,9 +587,21 @@ class TestResourceManager(RMFTestCase):
cd_access = 'a',
)
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+ content = Template('exclude_hosts_list.j2'),
owner = 'yarn',
group = 'hadoop',
)
+ if is_include_file_configured and manage_include_files:
+ self.assertResourceCalled('Directory', '/etc/hadoop/conf_for_include',
+ mode = 0755,
+ create_parents = True,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf_for_include/yarn.include',
+ content = Template('include_hosts_list.j2'),
+ owner = 'yarn',
+ group = 'hadoop',
+ )
self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
owner = 'yarn',
group = 'hadoop',