You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2017/01/27 19:03:06 UTC
ambari git commit: AMBARI-19737. Add a validation check in HSI about
Capacity Scheduler preemption requirements.
Repository: ambari
Updated Branches:
refs/heads/trunk 9963ac819 -> dcdf95b28
AMBARI-19737. Add a validation check in HSI about Capacity Scheduler preemption requirements.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dcdf95b2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dcdf95b2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dcdf95b2
Branch: refs/heads/trunk
Commit: dcdf95b28a98e47062727f270adf2b4c69816f93
Parents: 9963ac8
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Fri Jan 27 11:02:48 2017 -0800
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Fri Jan 27 11:02:48 2017 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.5/services/stack_advisor.py | 12 ++++++++++++
.../2.5/common/services-normal-his-2-hosts.json | 8 +++++++-
.../2.5/common/services-normal-his-valid.json | 8 +++++++-
.../stacks/2.5/common/test_stack_advisor.py | 18 +++++++++++-------
4 files changed, 37 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcdf95b2/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 8e377da..ca4a968 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -388,8 +388,11 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
def validateHiveInteractiveEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
hive_site_env_properties = self.getSiteProperties(configurations, "hive-interactive-env")
+ yarn_site_properties = self.getSiteProperties(configurations, "yarn-site")
validationItems = []
hsi_hosts = self.getHostsForComponent(services, "HIVE", "HIVE_SERVER_INTERACTIVE")
+
+ # Check for expecting 'enable_hive_interactive' is ON given that there is HSI on atleast one host present.
if len(hsi_hosts) > 0:
# HIVE_SERVER_INTERACTIVE is mapped to a host
if 'enable_hive_interactive' not in hive_site_env_properties or (
@@ -407,6 +410,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
"item": self.getErrorItem(
"enable_hive_interactive in hive-interactive-env should be set to false.")})
+ # Check for 'yarn.resourcemanager.scheduler.monitor.enable' config to be true if HSI is ON.
+ if yarn_site_properties and 'yarn.resourcemanager.scheduler.monitor.enable' in yarn_site_properties:
+ scheduler_monitor_enabled = yarn_site_properties['yarn.resourcemanager.scheduler.monitor.enable']
+ if scheduler_monitor_enabled.lower() == 'false' and hive_site_env_properties and 'enable_hive_interactive' in hive_site_env_properties and \
+ hive_site_env_properties['enable_hive_interactive'].lower() == 'true':
+ validationItems.append({"config-name": "enable_hive_interactive",
+ "item": self.getWarnItem(
+ "When enabling LLAP, set 'yarn.resourcemanager.scheduler.monitor.enable' to true to ensure that LLAP gets the full allocated capacity.")})
+
validationProblems = self.toConfigurationValidationProblems(validationItems, "hive-interactive-env")
return validationProblems
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcdf95b2/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json
index 44698ed..0044bf1 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json
+++ b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json
@@ -1108,7 +1108,8 @@
"hive.llap.daemon.queue.name": "llap",
"hive.server2.enable.doAs": "true",
"hive.llap.daemon.num.executors": "0",
- "hive.server2.tez.sessions.per.default.queue": "32"
+ "hive.server2.tez.sessions.per.default.queue": "32",
+ "hive.tez.container.size" : "341"
}
},
"hive-interactive-env" : {
@@ -1159,6 +1160,11 @@
"properties": {
"tez.am.resource.memory.mb" : "341"
}
+ },
+ "tez-interactive-site": {
+ "properties": {
+ "tez.am.resource.memory.mb" : "341"
+ }
}
},
"changed-configurations" : [ ],
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcdf95b2/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
index 7955333..fad4a55 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
+++ b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
@@ -1049,7 +1049,8 @@
"configurations" : {
"hive-interactive-site" : {
"properties": {
- "hive.llap.daemon.queue.name": "llap"
+ "hive.llap.daemon.queue.name": "llap",
+ "hive.tez.container.size" : "341"
}
},
"capacity-scheduler" : {
@@ -1095,6 +1096,11 @@
"properties": {
"tez.am.resource.memory.mb" : "341"
}
+ },
+ "tez-interactive-site": {
+ "properties": {
+ "tez.am.resource.memory.mb" : "341"
+ }
}
},
"changed-configurations" : [ ],
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcdf95b2/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index a53cb25..ac00bab 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -492,7 +492,8 @@ class TestHDP25StackAdvisor(TestCase):
"properties": {"hive_security_authorization": "None"}
},
"yarn-site": {
- "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"}
+ "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true",
+ "yarn.resourcemanager.scheduler.monitor.enable": "false"}
}
}
configurations2 = {
@@ -506,17 +507,22 @@ class TestHDP25StackAdvisor(TestCase):
"properties": {"hive_security_authorization": "None"}
},
"yarn-site": {
- "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true"}
+ "properties": {"yarn.resourcemanager.work-preserving-recovery.enabled": "true",
+ "yarn.resourcemanager.scheduler.monitor.enable": "true"}
}
}
services = self.load_json("services-normal-his-valid.json")
+ # Checks for WARN message that 'yarn.resourcemanager.scheduler.monitor.enable' should be true.
res_expected = [
+ {'config-type': 'hive-interactive-env', 'message': "When enabling LLAP, set 'yarn.resourcemanager.scheduler.monitor.enable' to true to ensure that LLAP gets the full allocated capacity.", 'type': 'configuration', 'config-name': 'enable_hive_interactive', 'level': 'WARN'}
]
# the above error is not what we are checking for - just to keep test happy without having to test
res = self.stackAdvisor.validateHiveInteractiveEnvConfigurations(properties, recommendedDefaults, configurations, services, {})
self.assertEquals(res, res_expected)
+ # (1). Checks for ERROR message for 'enable_hive_interactive' to be true.
+ # (2). Further, no message regarding 'yarn.resourcemanager.scheduler.monitor.enable' as it is true already.
res_expected = [
{'config-type': 'hive-interactive-env', 'message': 'HIVE_SERVER_INTERACTIVE requires enable_hive_interactive in hive-interactive-env set to true.', 'type': 'configuration', 'config-name': 'enable_hive_interactive', 'level': 'ERROR'}
]
@@ -525,7 +531,6 @@ class TestHDP25StackAdvisor(TestCase):
pass
- ''' TODO: Commenting, Need to fix validations in 2.5/stack_advisor and then fix the test code
"""
Tests validations for Hive Server Interactive site.
"""
@@ -558,7 +563,7 @@ class TestHDP25StackAdvisor(TestCase):
# Expected : Error telling about the current size compared to minimum required size.
services1 = self.load_json("services-normal-his-valid.json")
res_expected1 = [
- {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49%) is less than minimum required "
+ {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49.0%) is less than minimum required "
"capacity (50%) for LLAP app to run", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'},
]
res1 = self.stackAdvisor.validateHiveInteractiveSiteConfigurations({}, {}, {}, services1, hosts)
@@ -575,7 +580,7 @@ class TestHDP25StackAdvisor(TestCase):
# than 50% of queue capacity for LLAP.
services2 = self.load_json("services-normal-his-2-hosts.json")
res_expected2 = [
- {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49%) is less than minimum required "
+ {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49.0%) is less than minimum required "
"capacity (50%) for LLAP app to run", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'},
{'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' current state is : 'STOPPED'. It is required to be in "
"'RUNNING' state for LLAP to run", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'},
@@ -636,7 +641,7 @@ class TestHDP25StackAdvisor(TestCase):
},
}
res_expected4 = [
- {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49%) is less than minimum required capacity (200%) for LLAP app to run",
+ {'config-type': 'hive-interactive-site', 'message': "Selected queue 'llap' capacity (49.0%) is less than minimum required capacity (200%) for LLAP app to run",
'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'ERROR'},
{'config-type': 'hive-interactive-site', 'message': "Capacity used by 'llap' queue is '250.88'. Service checks may not run as remaining available capacity "
"(261.12) in cluster is less than 512 MB.", 'type': 'configuration', 'config-name': 'hive.llap.daemon.queue.name', 'level': 'WARN'}]
@@ -645,7 +650,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(res4, res_expected4)
pass
- '''