You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/09/28 05:44:42 UTC
ambari git commit: AMBARI-18471. Refactor yarn() function in YARN
service. Part 1. (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk c265ae622 -> 6fb1ceef6
AMBARI-18471. Refactor yarn() function in YARN service. Part 1. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6fb1ceef
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6fb1ceef
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6fb1ceef
Branch: refs/heads/trunk
Commit: 6fb1ceef66b8d63c66603eebf1cd9e6d91c815aa
Parents: c265ae6
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Sep 28 08:44:44 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Wed Sep 28 08:44:44 2016 +0300
----------------------------------------------------------------------
.../2.1.0.2.0/package/scripts/params_linux.py | 7 +
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 548 +++++++++----------
.../stacks/2.0.6/YARN/test_historyserver.py | 1 -
.../stacks/2.0.6/YARN/test_resourcemanager.py | 54 +-
.../stacks/2.1/YARN/test_apptimelineserver.py | 40 +-
.../test/python/stacks/2.3/YARN/test_ats_1_5.py | 188 +++----
6 files changed, 421 insertions(+), 417 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 4d42861..0d46069 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -193,6 +193,13 @@ else:
rm_webui_address = format("{rm_host}:{rm_port}")
rm_webui_https_address = format("{rm_host}:{rm_https_port}")
+if security_enabled:
+ tc_mode = 0644
+ tc_owner = "root"
+else:
+ tc_mode = None
+ tc_owner = hdfs_user
+
nm_webui_address = config['configurations']['yarn-site']['yarn.nodemanager.webapp.address']
hs_webui_address = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address']
nm_address = config['configurations']['yarn-site']['yarn.nodemanager.address'] # still contains 0.0.0.0
http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index a3a3a06..70ed5b3 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -37,63 +37,6 @@ from ambari_commons import OSConst
from resource_management.libraries.functions.mounted_dirs_helper import handle_mounted_dirs
-# Local Imports
-
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def yarn(name = None):
- import params
- XmlConfig("mapred-site.xml",
- conf_dir=params.config_dir,
- configurations=params.config['configurations']['mapred-site'],
- owner=params.yarn_user,
- mode='f'
- )
- XmlConfig("yarn-site.xml",
- conf_dir=params.config_dir,
- configurations=params.config['configurations']['yarn-site'],
- owner=params.yarn_user,
- mode='f',
- configuration_attributes=params.config['configuration_attributes']['yarn-site']
- )
- XmlConfig("capacity-scheduler.xml",
- conf_dir=params.config_dir,
- configurations=params.config['configurations']['capacity-scheduler'],
- owner=params.yarn_user,
- mode='f'
- )
-
- if params.service_map.has_key(name):
- service_name = params.service_map[name]
-
- ServiceConfig(service_name,
- action="change_user",
- username = params.yarn_user,
- password = Script.get_password(params.yarn_user))
-
-def create_log_dir(dir_name):
- import params
- Directory(dir_name,
- create_parents = True,
- cd_access="a",
- mode=0775,
- owner=params.yarn_user,
- group=params.user_group,
- ignore_failures=True,
- )
-
-def create_local_dir(dir_name):
- import params
- Directory(dir_name,
- create_parents = True,
- cd_access="a",
- mode=0755,
- owner=params.yarn_user,
- group=params.user_group,
- ignore_failures=True,
- recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'},
- )
-
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def yarn(name=None, config_dir=None):
"""
@@ -102,107 +45,18 @@ def yarn(name=None, config_dir=None):
"""
import params
+ if name == 'resourcemanager':
+ setup_resourcemanager()
+ elif name == 'nodemanager':
+ setup_nodemanager()
+ elif name == 'apptimelineserver':
+ setup_ats()
+ elif name == 'historyserver':
+ setup_historyserver()
+
if config_dir is None:
config_dir = params.hadoop_conf_dir
- if name == "historyserver":
- if params.yarn_log_aggregation_enabled:
- params.HdfsResource(params.yarn_nm_app_log_dir,
- action="create_on_execute",
- type="directory",
- owner=params.yarn_user,
- group=params.user_group,
- mode=01777,
- recursive_chmod=True
- )
-
- # create the /tmp folder with proper permissions if it doesn't exist yet
- if params.entity_file_history_directory.startswith('/tmp'):
- params.HdfsResource(params.hdfs_tmp_dir,
- action="create_on_execute",
- type="directory",
- owner=params.hdfs_user,
- mode=0777,
- )
-
- params.HdfsResource(params.entity_file_history_directory,
- action="create_on_execute",
- type="directory",
- owner=params.yarn_user,
- group=params.user_group
- )
- params.HdfsResource("/mapred",
- type="directory",
- action="create_on_execute",
- owner=params.mapred_user
- )
- params.HdfsResource("/mapred/system",
- type="directory",
- action="create_on_execute",
- owner=params.hdfs_user
- )
- params.HdfsResource(params.mapreduce_jobhistory_done_dir,
- type="directory",
- action="create_on_execute",
- owner=params.mapred_user,
- group=params.user_group,
- change_permissions_for_parents=True,
- mode=0777
- )
- params.HdfsResource(None, action="execute")
- Directory(params.jhs_leveldb_state_store_dir,
- owner=params.mapred_user,
- group=params.user_group,
- create_parents = True,
- cd_access="a",
- recursive_ownership = True,
- )
-
- #<editor-fold desc="Node Manager Section">
- if name == "nodemanager":
-
- # First start after enabling/disabling security
- if params.toggle_nm_security:
- Directory(params.nm_local_dirs_list + params.nm_log_dirs_list,
- action='delete'
- )
-
- # If yarn.nodemanager.recovery.dir exists, remove this dir
- if params.yarn_nodemanager_recovery_dir:
- Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
- action='delete'
- )
-
- # Setting NM marker file
- if params.security_enabled:
- Directory(params.nm_security_marker_dir)
- File(params.nm_security_marker,
- content="Marker file to track first start after enabling/disabling security. "
- "During first start yarn local, log dirs are removed and recreated"
- )
- elif not params.security_enabled:
- File(params.nm_security_marker, action="delete")
-
-
- if not params.security_enabled or params.toggle_nm_security:
- # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
- nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params)
- # create a history file used by handle_mounted_dirs
- File(params.nm_log_dir_to_mount_file,
- owner=params.hdfs_user,
- group=params.user_group,
- mode=0644,
- content=nm_log_dir_to_mount_file_content
- )
- nm_local_dir_to_mount_file_content = handle_mounted_dirs(create_local_dir, params.nm_local_dirs, params.nm_local_dir_to_mount_file, params)
- File(params.nm_local_dir_to_mount_file,
- owner=params.hdfs_user,
- group=params.user_group,
- mode=0644,
- content=nm_local_dir_to_mount_file_content
- )
- #</editor-fold>
-
if params.yarn_nodemanager_recovery_dir:
Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
owner=params.yarn_user,
@@ -245,15 +99,14 @@ def yarn(name=None, config_dir=None):
# During RU, Core Masters and Slaves need hdfs-site.xml
# TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added,
# RU should rely on all available in <stack-root>/<version>/hadoop/conf
- if 'hdfs-site' in params.config['configurations']:
- XmlConfig("hdfs-site.xml",
- conf_dir=config_dir,
- configurations=params.config['configurations']['hdfs-site'],
- configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
- owner=params.hdfs_user,
- group=params.user_group,
- mode=0644
- )
+ XmlConfig("hdfs-site.xml",
+ conf_dir=config_dir,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=0644
+ )
XmlConfig("mapred-site.xml",
conf_dir=config_dir,
@@ -282,85 +135,6 @@ def yarn(name=None, config_dir=None):
mode=0644
)
- if name == 'resourcemanager':
- Directory(params.rm_nodes_exclude_dir,
- mode=0755,
- create_parents=True,
- cd_access='a',
- )
- File(params.rm_nodes_exclude_path,
- owner=params.yarn_user,
- group=params.user_group
- )
- File(params.yarn_job_summary_log,
- owner=params.yarn_user,
- group=params.user_group
- )
- if not is_empty(params.node_label_enable) and params.node_label_enable or is_empty(params.node_label_enable) and params.node_labels_dir:
- params.HdfsResource(params.node_labels_dir,
- type="directory",
- action="create_on_execute",
- change_permissions_for_parents=True,
- owner=params.yarn_user,
- group=params.user_group,
- mode=0700
- )
- params.HdfsResource(None, action="execute")
-
-
- elif name == 'apptimelineserver':
- Directory(params.ats_leveldb_dir,
- owner=params.yarn_user,
- group=params.user_group,
- create_parents = True,
- cd_access="a",
- )
-
- # if stack support application timeline-service state store property (timeline_state_store stack feature)
- if params.stack_supports_timeline_state_store:
- Directory(params.ats_leveldb_state_store_dir,
- owner=params.yarn_user,
- group=params.user_group,
- create_parents = True,
- cd_access="a",
- )
- # app timeline server 1.5 directories
- if not is_empty(params.entity_groupfs_store_dir):
- parent_path = os.path.dirname(params.entity_groupfs_store_dir)
- params.HdfsResource(parent_path,
- type="directory",
- action="create_on_execute",
- change_permissions_for_parents=True,
- owner=params.yarn_user,
- group=params.user_group,
- mode=0755
- )
- params.HdfsResource(params.entity_groupfs_store_dir,
- type="directory",
- action="create_on_execute",
- owner=params.yarn_user,
- group=params.user_group,
- mode=params.entity_groupfs_store_dir_mode
- )
- if not is_empty(params.entity_groupfs_active_dir):
- parent_path = os.path.dirname(params.entity_groupfs_active_dir)
- params.HdfsResource(parent_path,
- type="directory",
- action="create_on_execute",
- change_permissions_for_parents=True,
- owner=params.yarn_user,
- group=params.user_group,
- mode=0755
- )
- params.HdfsResource(params.entity_groupfs_active_dir,
- type="directory",
- action="create_on_execute",
- owner=params.yarn_user,
- group=params.user_group,
- mode=params.entity_groupfs_active_dir_mode
- )
- params.HdfsResource(None, action="execute")
-
File(format("{limits_conf_dir}/yarn.conf"),
mode=0644,
content=Template('yarn.conf.j2')
@@ -378,8 +152,7 @@ def yarn(name=None, config_dir=None):
content=InlineTemplate(params.yarn_env_sh_template)
)
- container_executor = format("{yarn_container_bin}/container-executor")
- File(container_executor,
+ File(format("{yarn_container_bin}/container-executor"),
group=params.yarn_executor_container_group,
mode=params.container_executor_mode
)
@@ -396,15 +169,8 @@ def yarn(name=None, config_dir=None):
mode=0755,
cd_access="a")
- if params.security_enabled:
- tc_mode = 0644
- tc_owner = "root"
- else:
- tc_mode = None
- tc_owner = params.hdfs_user
-
File(os.path.join(config_dir, "mapred-env.sh"),
- owner=tc_owner,
+ owner=params.tc_owner,
mode=0755,
content=InlineTemplate(params.mapred_env_sh_template)
)
@@ -416,35 +182,34 @@ def yarn(name=None, config_dir=None):
mode=06050
)
File(os.path.join(config_dir, 'taskcontroller.cfg'),
- owner = tc_owner,
- mode = tc_mode,
+ owner = params.tc_owner,
+ mode = params.tc_mode,
group = params.mapred_tt_group,
content=Template("taskcontroller.cfg.j2")
)
else:
File(os.path.join(config_dir, 'taskcontroller.cfg'),
- owner=tc_owner,
+ owner=params.tc_owner,
content=Template("taskcontroller.cfg.j2")
)
- if "mapred-site" in params.config['configurations']:
- XmlConfig("mapred-site.xml",
- conf_dir=config_dir,
- configurations=params.config['configurations']['mapred-site'],
- configuration_attributes=params.config['configuration_attributes']['mapred-site'],
- owner=params.mapred_user,
- group=params.user_group
- )
+ XmlConfig("mapred-site.xml",
+ conf_dir=config_dir,
+ configurations=params.config['configurations']['mapred-site'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-site'],
+ owner=params.mapred_user,
+ group=params.user_group
+ )
+
+ XmlConfig("capacity-scheduler.xml",
+ conf_dir=config_dir,
+ configurations=params.config['configurations'][
+ 'capacity-scheduler'],
+ configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
- if "capacity-scheduler" in params.config['configurations']:
- XmlConfig("capacity-scheduler.xml",
- conf_dir=config_dir,
- configurations=params.config['configurations'][
- 'capacity-scheduler'],
- configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
- owner=params.hdfs_user,
- group=params.user_group
- )
if "ssl-client" in params.config['configurations']:
XmlConfig("ssl-client.xml",
conf_dir=config_dir,
@@ -495,4 +260,239 @@ def yarn(name=None, config_dir=None):
File(os.path.join(config_dir, 'ssl-server.xml.example'),
owner=params.mapred_user,
group=params.user_group
- )
\ No newline at end of file
+ )
+
+def setup_historyserver():
+ import params
+
+ if params.yarn_log_aggregation_enabled:
+ params.HdfsResource(params.yarn_nm_app_log_dir,
+ action="create_on_execute",
+ type="directory",
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=01777,
+ recursive_chmod=True
+ )
+
+ # create the /tmp folder with proper permissions if it doesn't exist yet
+ if params.entity_file_history_directory.startswith('/tmp'):
+ params.HdfsResource(params.hdfs_tmp_dir,
+ action="create_on_execute",
+ type="directory",
+ owner=params.hdfs_user,
+ mode=0777,
+ )
+
+ params.HdfsResource(params.entity_file_history_directory,
+ action="create_on_execute",
+ type="directory",
+ owner=params.yarn_user,
+ group=params.user_group
+ )
+ params.HdfsResource("/mapred",
+ type="directory",
+ action="create_on_execute",
+ owner=params.mapred_user
+ )
+ params.HdfsResource("/mapred/system",
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user
+ )
+ params.HdfsResource(params.mapreduce_jobhistory_done_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.mapred_user,
+ group=params.user_group,
+ change_permissions_for_parents=True,
+ mode=0777
+ )
+ params.HdfsResource(None, action="execute")
+ Directory(params.jhs_leveldb_state_store_dir,
+ owner=params.mapred_user,
+ group=params.user_group,
+ create_parents = True,
+ cd_access="a",
+ recursive_ownership = True,
+ )
+
+def setup_nodemanager():
+ import params
+
+ # First start after enabling/disabling security
+ if params.toggle_nm_security:
+ Directory(params.nm_local_dirs_list + params.nm_log_dirs_list,
+ action='delete'
+ )
+
+ # If yarn.nodemanager.recovery.dir exists, remove this dir
+ if params.yarn_nodemanager_recovery_dir:
+ Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
+ action='delete'
+ )
+
+ # Setting NM marker file
+ if params.security_enabled:
+ Directory(params.nm_security_marker_dir)
+ File(params.nm_security_marker,
+ content="Marker file to track first start after enabling/disabling security. "
+ "During first start yarn local, log dirs are removed and recreated"
+ )
+ elif not params.security_enabled:
+ File(params.nm_security_marker, action="delete")
+
+
+ if not params.security_enabled or params.toggle_nm_security:
+ # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
+ nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params)
+ # create a history file used by handle_mounted_dirs
+ File(params.nm_log_dir_to_mount_file,
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=0644,
+ content=nm_log_dir_to_mount_file_content
+ )
+ nm_local_dir_to_mount_file_content = handle_mounted_dirs(create_local_dir, params.nm_local_dirs, params.nm_local_dir_to_mount_file, params)
+ File(params.nm_local_dir_to_mount_file,
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=0644,
+ content=nm_local_dir_to_mount_file_content
+ )
+
+def setup_resourcemanager():
+ import params
+
+ Directory(params.rm_nodes_exclude_dir,
+ mode=0755,
+ create_parents=True,
+ cd_access='a',
+ )
+ File(params.rm_nodes_exclude_path,
+ owner=params.yarn_user,
+ group=params.user_group
+ )
+ File(params.yarn_job_summary_log,
+ owner=params.yarn_user,
+ group=params.user_group
+ )
+ if not is_empty(params.node_label_enable) and params.node_label_enable or is_empty(params.node_label_enable) and params.node_labels_dir:
+ params.HdfsResource(params.node_labels_dir,
+ type="directory",
+ action="create_on_execute",
+ change_permissions_for_parents=True,
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0700
+ )
+ params.HdfsResource(None, action="execute")
+
+def setup_ats():
+ import params
+
+ Directory(params.ats_leveldb_dir,
+ owner=params.yarn_user,
+ group=params.user_group,
+ create_parents = True,
+ cd_access="a",
+ )
+
+ # if stack support application timeline-service state store property (timeline_state_store stack feature)
+ if params.stack_supports_timeline_state_store:
+ Directory(params.ats_leveldb_state_store_dir,
+ owner=params.yarn_user,
+ group=params.user_group,
+ create_parents = True,
+ cd_access="a",
+ )
+ # app timeline server 1.5 directories
+ if not is_empty(params.entity_groupfs_store_dir):
+ parent_path = os.path.dirname(params.entity_groupfs_store_dir)
+ params.HdfsResource(parent_path,
+ type="directory",
+ action="create_on_execute",
+ change_permissions_for_parents=True,
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0755
+ )
+ params.HdfsResource(params.entity_groupfs_store_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=params.entity_groupfs_store_dir_mode
+ )
+ if not is_empty(params.entity_groupfs_active_dir):
+ parent_path = os.path.dirname(params.entity_groupfs_active_dir)
+ params.HdfsResource(parent_path,
+ type="directory",
+ action="create_on_execute",
+ change_permissions_for_parents=True,
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0755
+ )
+ params.HdfsResource(params.entity_groupfs_active_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=params.entity_groupfs_active_dir_mode
+ )
+ params.HdfsResource(None, action="execute")
+
+def create_log_dir(dir_name):
+ import params
+ Directory(dir_name,
+ create_parents = True,
+ cd_access="a",
+ mode=0775,
+ owner=params.yarn_user,
+ group=params.user_group,
+ ignore_failures=True,
+ )
+
+def create_local_dir(dir_name):
+ import params
+ Directory(dir_name,
+ create_parents = True,
+ cd_access="a",
+ mode=0755,
+ owner=params.yarn_user,
+ group=params.user_group,
+ ignore_failures=True,
+ recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'},
+ )
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def yarn(name = None):
+ import params
+ XmlConfig("mapred-site.xml",
+ conf_dir=params.config_dir,
+ configurations=params.config['configurations']['mapred-site'],
+ owner=params.yarn_user,
+ mode='f'
+ )
+ XmlConfig("yarn-site.xml",
+ conf_dir=params.config_dir,
+ configurations=params.config['configurations']['yarn-site'],
+ owner=params.yarn_user,
+ mode='f',
+ configuration_attributes=params.config['configuration_attributes']['yarn-site']
+ )
+ XmlConfig("capacity-scheduler.xml",
+ conf_dir=params.config_dir,
+ configurations=params.config['configurations']['capacity-scheduler'],
+ owner=params.yarn_user,
+ mode='f'
+ )
+
+ if params.service_map.has_key(name):
+ service_name = params.service_map[name]
+
+ ServiceConfig(service_name,
+ action="change_user",
+ username = params.yarn_user,
+ password = Script.get_password(params.yarn_user))
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
index 643f946..119dcf0 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
@@ -464,7 +464,6 @@ class TestHistoryServer(RMFTestCase):
)
def assert_configure_secured(self):
-
self.assertResourceCalled('HdfsResource', '/app-logs',
immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = True,
http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
index 7b5ce18..c98a64d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
@@ -178,7 +178,19 @@ class TestResourceManager(RMFTestCase):
self.assertNoMoreResources()
def assert_configure_default(self):
-
+ self.assertResourceCalled('Directory', '/etc/hadoop/conf',
+ mode = 0755,
+ create_parents = True,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+ owner = 'yarn',
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
+ owner = 'yarn',
+ group = 'hadoop',
+ )
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
@@ -268,19 +280,6 @@ class TestResourceManager(RMFTestCase):
configurations = self.getConfig()['configurations']['capacity-scheduler'],
configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler']
)
- self.assertResourceCalled('Directory', '/etc/hadoop/conf',
- mode = 0755,
- create_parents = True,
- cd_access = 'a',
- )
- self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
- owner = 'yarn',
- group = 'hadoop',
- )
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
@@ -347,7 +346,19 @@ class TestResourceManager(RMFTestCase):
)
def assert_configure_secured(self):
-
+ self.assertResourceCalled('Directory', '/etc/hadoop/conf',
+ mode = 0755,
+ create_parents = True,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
+ owner = 'yarn',
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
+ owner = 'yarn',
+ group = 'hadoop',
+ )
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
@@ -437,19 +448,6 @@ class TestResourceManager(RMFTestCase):
configurations = self.getConfig()['configurations']['capacity-scheduler'],
configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler']
)
- self.assertResourceCalled('Directory', '/etc/hadoop/conf',
- mode = 0755,
- create_parents = True,
- cd_access = 'a',
- )
- self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
- owner = 'yarn',
- group = 'hadoop',
- )
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
index dc11ba9..40db813 100644
--- a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
+++ b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
@@ -92,6 +92,26 @@ class TestAppTimelineServer(RMFTestCase):
self.assertNoMoreResources()
def assert_configure_default(self):
+ self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline',
+ owner = 'yarn',
+ group = 'hadoop',
+ create_parents = True,
+ cd_access='a'
+ )
+ self.assertResourceCalled('HdfsResource', None,
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ dfs_type = '',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ hadoop_conf_dir = '/etc/hadoop/conf',
+ )
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
@@ -181,26 +201,6 @@ class TestAppTimelineServer(RMFTestCase):
configurations = self.getConfig()['configurations']['capacity-scheduler'],
configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler']
)
- self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline',
- owner = 'yarn',
- group = 'hadoop',
- create_parents = True,
- cd_access='a'
- )
- self.assertResourceCalled('HdfsResource', None,
- immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
- security_enabled = False,
- hadoop_bin_dir = '/usr/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- dfs_type = '',
- hdfs_site = self.getConfig()['configurations']['hdfs-site'],
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'hdfs',
- action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
- hadoop_conf_dir = '/etc/hadoop/conf',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/6fb1ceef/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py b/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py
index abfe756..b523412 100644
--- a/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py
+++ b/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py
@@ -50,6 +50,100 @@ class TestAts(RMFTestCase):
self.assertNoMoreResources()
def assert_configure_default(self):
+ self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline',
+ owner = 'yarn',
+ group = 'hadoop',
+ create_parents = True,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('HdfsResource', '/ats',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ dfs_type = '',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ change_permissions_for_parents = True,
+ owner = 'yarn',
+ group = 'hadoop',
+ hadoop_conf_dir = '/etc/hadoop/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ mode = 0755,
+ )
+ self.assertResourceCalled('HdfsResource', '/ats/done',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ dfs_type = '',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ owner = 'yarn',
+ group = 'hadoop',
+ hadoop_conf_dir = '/etc/hadoop/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ mode = 0700,
+ )
+ self.assertResourceCalled('HdfsResource', '/ats',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ dfs_type = '',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ change_permissions_for_parents = True,
+ owner = 'yarn',
+ group = 'hadoop',
+ hadoop_conf_dir = '/etc/hadoop/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ mode = 0755,
+ )
+ self.assertResourceCalled('HdfsResource', '/ats/active',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/bin',
+ keytab = UnknownConfigurationMock(),
+ dfs_type = '',
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ owner = 'yarn',
+ group = 'hadoop',
+ hadoop_conf_dir = '/etc/hadoop/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ mode = 01777,
+ )
+ self.assertResourceCalled('HdfsResource', None,
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ dfs_type = '',
+ hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ hadoop_conf_dir = '/etc/hadoop/conf',
+ )
self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
owner = 'yarn',
group = 'hadoop',
@@ -146,100 +240,6 @@ class TestAts(RMFTestCase):
owner = 'yarn',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline',
- owner = 'yarn',
- group = 'hadoop',
- create_parents = True,
- cd_access = 'a',
- )
- self.assertResourceCalled('HdfsResource', '/ats',
- immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
- security_enabled = False,
- hadoop_bin_dir = '/usr/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- dfs_type = '',
- hdfs_site = self.getConfig()['configurations']['hdfs-site'],
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'hdfs',
- change_permissions_for_parents = True,
- owner = 'yarn',
- group = 'hadoop',
- hadoop_conf_dir = '/etc/hadoop/conf',
- type = 'directory',
- action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
- mode = 0755,
- )
- self.assertResourceCalled('HdfsResource', '/ats/done',
- immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
- security_enabled = False,
- hadoop_bin_dir = '/usr/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- dfs_type = '',
- hdfs_site = self.getConfig()['configurations']['hdfs-site'],
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'hdfs',
- owner = 'yarn',
- group = 'hadoop',
- hadoop_conf_dir = '/etc/hadoop/conf',
- type = 'directory',
- action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
- mode = 0700,
- )
- self.assertResourceCalled('HdfsResource', '/ats',
- immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
- security_enabled = False,
- hadoop_bin_dir = '/usr/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- dfs_type = '',
- hdfs_site = self.getConfig()['configurations']['hdfs-site'],
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'hdfs',
- change_permissions_for_parents = True,
- owner = 'yarn',
- group = 'hadoop',
- hadoop_conf_dir = '/etc/hadoop/conf',
- type = 'directory',
- action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
- mode = 0755,
- )
- self.assertResourceCalled('HdfsResource', '/ats/active',
- immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
- security_enabled = False,
- hadoop_bin_dir = '/usr/bin',
- keytab = UnknownConfigurationMock(),
- dfs_type = '',
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- hdfs_site = self.getConfig()['configurations']['hdfs-site'],
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'hdfs',
- owner = 'yarn',
- group = 'hadoop',
- hadoop_conf_dir = '/etc/hadoop/conf',
- type = 'directory',
- action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
- mode = 01777,
- )
- self.assertResourceCalled('HdfsResource', None,
- immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
- security_enabled = False,
- hadoop_bin_dir = '/usr/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- dfs_type = '',
- hdfs_site = self.getConfig()['configurations']['hdfs-site'],
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'hdfs',
- action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
- hadoop_conf_dir = '/etc/hadoop/conf',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,