You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2015/09/14 14:04:24 UTC
ambari git commit: AMBARI-13073. Backport Request - Ambari is
overwriting ownership of yarn.local.dir with yarn:hadoop in Kerberized
cluster (dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-2.0.maint 4342a0602 -> 113eb2362
AMBARI-13073. Backport Request - Ambari is overwriting ownership of yarn.local.dir with yarn:hadoop in Kerberized cluster (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/113eb236
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/113eb236
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/113eb236
Branch: refs/heads/branch-2.0.maint
Commit: 113eb2362101cbca8d0f7bcd7ce8e94d307b08a8
Parents: 4342a06
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Mon Sep 14 15:03:56 2015 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Mon Sep 14 15:03:56 2015 +0300
----------------------------------------------------------------------
.../YARN/2.1.0.2.0/package/scripts/params.py | 7 ++
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 69 ++++++++++++++------
.../stacks/2.0.6/YARN/test_nodemanager.py | 31 ++++-----
.../stacks/2.0.6/YARN/test_resourcemanager.py | 8 ++-
4 files changed, 77 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/113eb236/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
index 29e8dfb..c396650 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py
@@ -91,6 +91,12 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
smokeuser = config['configurations']['cluster-env']['smokeuser']
smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
security_enabled = config['configurations']['cluster-env']['security_enabled']
+nm_security_marker = '/var/lib/hadoop-yarn/nm_security_enabled'
+current_nm_security_state = os.path.isfile(nm_security_marker)
+toggle_nm_security = (current_nm_security_state and not security_enabled) or (not current_nm_security_state and security_enabled)
+rm_security_marker = "/var/lib/hadoop-yarn/rm_security_enabled"
+current_rm_security_state = os.path.isfile(rm_security_marker)
+toggle_rm_security = (current_rm_security_state and not security_enabled) or (not current_rm_security_state and security_enabled)
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group']
kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
@@ -115,6 +121,7 @@ mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix
mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix']
mapred_env_sh_template = config['configurations']['mapred-env']['content']
yarn_env_sh_template = config['configurations']['yarn-env']['content']
+yarn_nodemanager_recovery_dir = default('/configurations/yarn-site/yarn.nodemanager.recovery.dir', None)
if len(rm_hosts) > 1:
additional_rm_host = rm_hosts[1]
http://git-wip-us.apache.org/repos/asf/ambari/blob/113eb236/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index 0736663..bdea2d7 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -61,28 +61,47 @@ def yarn(name = None):
params.HdfsDirectory(None, action="create")
if name == "nodemanager":
- Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
- owner=params.yarn_user,
- group=params.user_group,
- recursive=True,
- cd_access="a",
- ignore_failures=True,
- mode=0775
- )
-
- Execute(('chown', '-R', params.yarn_user, params.nm_local_dirs),
- only_if=format("test -d {nm_local_dirs}"),
- sudo=True)
+ # First start after enabling/disabling security
+ if params.toggle_nm_security:
+ Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
+ action='delete'
+ )
+ # If yarn.nodemanager.recovery.dir exists, remove this dir
+ if params.yarn_nodemanager_recovery_dir:
+ Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
+ action='delete'
+ )
- if params.security_enabled:
- smokeuser_directories = [os.path.join(dir, 'usercache' ,params.smokeuser)
- for dir in params.nm_local_dirs.split(',')]
- for directory in smokeuser_directories:
- Execute(('chown', '-R', params.smokeuser, directory),
- only_if=format("test -d {directory}"),
- sudo=True,
+ # Setting NM marker file
+ if params.security_enabled:
+ File(params.nm_security_marker,
+ content="Marker file to track first start after enabling/disabling security. "
+ "During first start yarn local, log dirs are removed and recreated"
)
+ elif not params.security_enabled:
+ File(params.nm_security_marker, action="delete")
+
+
+ if not params.security_enabled or params.toggle_nm_security:
+ Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
+ owner=params.yarn_user,
+ group=params.user_group,
+ recursive=True,
+ cd_access="a",
+ ignore_failures=True,
+ mode=0775
+ )
+
+ if params.yarn_nodemanager_recovery_dir:
+ Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
+ owner=params.yarn_user,
+ group=params.user_group,
+ recursive=True,
+ mode=0755,
+ cd_access = 'a',
+ )
+
Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
owner=params.yarn_user,
group=params.user_group,
@@ -151,6 +170,18 @@ def yarn(name = None):
group=params.user_group,
mode=0700
)
+ if params.toggle_rm_security:
+ Execute('yarn resourcemanager -format-state-store', user = params.yarn_user,
+ )
+ # Setting RM marker file
+ if params.security_enabled:
+ File(params.rm_security_marker,
+ content="Marker file to track first start after enabling/disabling security. "
+ "During first start ResourceManager state store is formatted"
+ )
+ elif not params.security_enabled:
+ File(params.rm_security_marker, action="delete")
+
elif name == 'apptimelineserver':
Directory(params.ats_leveldb_dir,
owner=params.yarn_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/113eb236/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
index 0956565..9ac71e7 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
@@ -230,11 +230,6 @@ class TestNodeManager(RMFTestCase):
mode = 0775,
cd_access='a'
)
- self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local,/hadoop/yarn/local1'),
- sudo = True,
- only_if = 'test -d /hadoop/yarn/local,/hadoop/yarn/local1',
- )
-
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
@@ -436,30 +431,30 @@ class TestNodeManager(RMFTestCase):
action = ['create'],
)
self.assertResourceCalled('Directory', '/hadoop/yarn/local',
- owner = 'yarn',
+ action = ['delete'],
+ )
+ self.assertResourceCalled('Directory', '/hadoop/yarn/log',
+ action = ['delete'],
+ )
+ self.assertResourceCalled('File', '/var/lib/hadoop-yarn/nm_security_enabled',
+ content = 'Marker file to track first start after enabling/disabling security. During first start yarn local, log dirs are removed and recreated',
+ )
+ self.assertResourceCalled('Directory', '/hadoop/yarn/local',
group = 'hadoop',
recursive = True,
+ cd_access = 'a',
ignore_failures = True,
mode = 0775,
- cd_access='a'
+ owner = 'yarn',
)
self.assertResourceCalled('Directory', '/hadoop/yarn/log',
- owner = 'yarn',
group = 'hadoop',
recursive = True,
+ cd_access = 'a',
ignore_failures = True,
mode = 0775,
- cd_access='a'
- )
- self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local'),
- sudo = True,
- only_if = 'test -d /hadoop/yarn/local',
+ owner = 'yarn',
)
-
- self.assertResourceCalled('Execute', ('chown', '-R', u'ambari-qa', u'/hadoop/yarn/local/usercache/ambari-qa'),
- sudo = True,
- only_if = 'test -d /hadoop/yarn/local/usercache/ambari-qa',
- )
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
http://git-wip-us.apache.org/repos/asf/ambari/blob/113eb236/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
index d32fc1a..ffef460 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
@@ -396,6 +396,12 @@ class TestResourceManager(RMFTestCase):
owner = 'yarn',
group = 'hadoop',
)
+ self.assertResourceCalled('Execute', 'yarn resourcemanager -format-state-store',
+ user = 'yarn',
+ )
+ self.assertResourceCalled('File', '/var/lib/hadoop-yarn/rm_security_enabled',
+ content = 'Marker file to track first start after enabling/disabling security. During first start ResourceManager state store is formatted',
+ )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
@@ -571,4 +577,4 @@ class TestResourceManager(RMFTestCase):
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
\ No newline at end of file
+ put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})