You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/08/31 14:14:01 UTC
ambari git commit: AMBARI-12935 Ambari is overwriting ownership of
yarn.local.dir with yarn:hadoop in Kerberized cluster (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 1d2e6e667 -> 0e675ff28
AMBARI-12935 Ambari is overwriting ownership of yarn.local.dir with yarn:hadoop in Kerberized cluster (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0e675ff2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0e675ff2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0e675ff2
Branch: refs/heads/trunk
Commit: 0e675ff28914f461ec19b0b98539c5e6583cb33e
Parents: 1d2e6e6
Author: Dmytro Sen <ds...@apache.org>
Authored: Mon Aug 31 15:13:48 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Mon Aug 31 15:13:48 2015 +0300
----------------------------------------------------------------------
.../2.1.0.2.0/package/scripts/params_linux.py | 7 ++
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 72 ++++++++++++++------
.../stacks/2.0.6/YARN/test_nodemanager.py | 22 +++---
.../stacks/2.0.6/YARN/test_resourcemanager.py | 6 ++
4 files changed, 73 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e675ff2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 3d45d21..70ae4b8 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -118,6 +118,12 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
smokeuser = config['configurations']['cluster-env']['smokeuser']
smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
security_enabled = config['configurations']['cluster-env']['security_enabled']
+nm_security_marker = '/var/lib/hadoop-yarn/nm_security_enabled'
+current_nm_security_state = os.path.isfile(nm_security_marker)
+toggle_nm_security = (current_nm_security_state and not security_enabled) or (not current_nm_security_state and security_enabled)
+rm_security_marker = "/var/lib/hadoop-yarn/rm_security_enabled"
+current_rm_security_state = os.path.isfile(rm_security_marker)
+toggle_rm_security = (current_rm_security_state and not security_enabled) or (not current_rm_security_state and security_enabled)
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group']
yarn_nodemanager_container_executor_class = config['configurations']['yarn-site']['yarn.nodemanager.container-executor.class']
@@ -145,6 +151,7 @@ mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix
mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix']
mapred_env_sh_template = config['configurations']['mapred-env']['content']
yarn_env_sh_template = config['configurations']['yarn-env']['content']
+yarn_nodemanager_recovery_dir = default('/configurations/yarn-site/yarn.nodemanager.recovery.dir', None)
if len(rm_hosts) > 1:
additional_rm_host = rm_hosts[1]
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e675ff2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index 30629be..bea6954 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -98,31 +98,47 @@ def yarn(name = None):
)
if name == "nodemanager":
- Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
- owner=params.yarn_user,
- group=params.user_group,
- recursive=True,
- cd_access="a",
- ignore_failures=True,
- mode=0775
- )
- smokeuser_directories = [os.path.join(dir, 'usercache' ,params.smokeuser)
- for dir in params.nm_local_dirs.split(',')]
+ # First start after enabling/disabling security
+ if params.toggle_nm_security:
+ Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
+ action='delete'
+ )
- if not params.security_enabled:
- for directory in smokeuser_directories:
- Execute(('chown', '-R', params.yarn_user, directory),
- only_if=format("test -d {directory}"),
- sudo=True)
+ # If yarn.nodemanager.recovery.dir exists, remove this dir
+ if params.yarn_nodemanager_recovery_dir:
+ Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
+ action='delete'
+ )
+ # Setting NM marker file
+ if params.security_enabled:
+ File(params.nm_security_marker,
+ content="Marker file to track first start after enabling/disabling security. "
+ "During first start yarn local, log dirs are removed and recreated"
+ )
+ elif not params.security_enabled:
+ File(params.nm_security_marker, action="delete")
+
+
+ if not params.security_enabled or params.toggle_nm_security:
+ Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
+ owner=params.yarn_user,
+ group=params.user_group,
+ recursive=True,
+ cd_access="a",
+ ignore_failures=True,
+ mode=0775
+ )
+
+ if params.yarn_nodemanager_recovery_dir:
+ Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
+ owner=params.yarn_user,
+ group=params.user_group,
+ recursive=True,
+ mode=0755
+ )
- if params.security_enabled:
- for directory in smokeuser_directories:
- Execute(('chown', '-R', params.smokeuser, directory),
- only_if=format("test -d {directory}"),
- sudo=True,
- )
Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
owner=params.yarn_user,
group=params.user_group,
@@ -206,6 +222,20 @@ def yarn(name = None):
mode=0700
)
params.HdfsResource(None, action="execute")
+ if params.toggle_rm_security:
+ Execute('yarn resourcemanager -format-state-store', user = params.yarn_user,
+ )
+ # Setting RM marker file
+ if params.security_enabled:
+ File(params.rm_security_marker,
+ content="Marker file to track first start after enabling/disabling security. "
+ "During first start ResourceManager state store is formatted"
+ )
+ elif not params.security_enabled:
+ File(params.rm_security_marker, action="delete")
+
+
+
elif name == 'apptimelineserver':
Directory(params.ats_leveldb_dir,
owner=params.yarn_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e675ff2/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
index 613ab69..6794097 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
@@ -167,15 +167,6 @@ class TestNodeManager(RMFTestCase):
mode = 0775,
cd_access='a'
)
- self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local/usercache/ambari-qa'),
- sudo = True,
- only_if = 'test -d /hadoop/yarn/local/usercache/ambari-qa',
- )
-
- self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local1/usercache/ambari-qa'),
- sudo = True,
- only_if = 'test -d /hadoop/yarn/local1/usercache/ambari-qa',
- )
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
@@ -334,6 +325,15 @@ class TestNodeManager(RMFTestCase):
def assert_configure_secured(self):
self.assertResourceCalled('Directory', '/hadoop/yarn/local',
+ action = ['delete']
+ )
+ self.assertResourceCalled('Directory', '/hadoop/yarn/log',
+ action = ['delete']
+ )
+ self.assertResourceCalled('File', '/var/lib/hadoop-yarn/nm_security_enabled',
+ content= 'Marker file to track first start after enabling/disabling security. During first start yarn local, log dirs are removed and recreated'
+ )
+ self.assertResourceCalled('Directory', '/hadoop/yarn/local',
owner = 'yarn',
group = 'hadoop',
recursive = True,
@@ -350,10 +350,6 @@ class TestNodeManager(RMFTestCase):
cd_access='a'
)
- self.assertResourceCalled('Execute', ('chown', '-R', u'ambari-qa', u'/hadoop/yarn/local/usercache/ambari-qa'),
- sudo = True,
- only_if = 'test -d /hadoop/yarn/local/usercache/ambari-qa',
- )
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e675ff2/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
index 428655b..fb7d847 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
@@ -434,6 +434,12 @@ class TestResourceManager(RMFTestCase):
owner = 'yarn',
group = 'hadoop',
)
+ self.assertResourceCalled('Execute', 'yarn resourcemanager -format-state-store',
+ user = 'yarn',
+ )
+ self.assertResourceCalled('File', '/var/lib/hadoop-yarn/rm_security_enabled',
+ content = 'Marker file to track first start after enabling/disabling security. During first start ResourceManager state store is formatted',
+ )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',