You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/08/17 22:25:18 UTC
ambari git commit: AMBARI-12796 Cannot add service after upgrade and
enabling ranger (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk d4c0780ba -> 5b38d8766
AMBARI-12796 Cannot add service after upgrade and enabling ranger (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5b38d876
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5b38d876
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5b38d876
Branch: refs/heads/trunk
Commit: 5b38d8766be4535e64da600682a604d969ef17ee
Parents: d4c0780
Author: Dmytro Sen <ds...@apache.org>
Authored: Mon Aug 17 23:25:09 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Mon Aug 17 23:25:09 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 21 ++++-
.../stacks/HDP/2.2/services/stack_advisor.py | 40 +++++++---
.../stacks/HDP/2.3/services/stack_advisor.py | 4 +-
.../src/main/resources/stacks/stack_advisor.py | 7 +-
.../stacks/2.3/common/test_stack_advisor.py | 81 ++++++++++++++++++++
5 files changed, 134 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5b38d876/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 8af6d91..3fd010a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -381,8 +381,21 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
print("SiteName: %s, method: %s\n" % (siteName, method.__name__))
print("Site properties: %s\n" % str(siteProperties))
print("Recommendations: %s\n********\n" % str(siteRecommendations))
- resultItems = method(siteProperties, siteRecommendations, configurations, services, hosts)
- items.extend(resultItems)
+ try:
+ resultItems = method(siteProperties, siteRecommendations, configurations, services, hosts)
+ items.extend(resultItems)
+ except (AttributeError, TypeError, LookupError) as e:
+ msg = "Failed to validate configuration "
+ print msg
+ print e
+ items.extend([{
+ 'message': msg,
+ 'level': 'ERROR',
+ 'config-type': siteName,
+ 'config-name': '',
+ 'type': 'configuration'
+ }])
+
clusterWideItems = self.validateClusterConfigurations(configurations, services, hosts)
items.extend(clusterWideItems)
self.validateMinMax(items, recommendedDefaults, configurations)
@@ -482,7 +495,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
distributed_item = None
distributed = properties.get("hbase.cluster.distributed")
- if hbase_rootdir.startswith("hdfs://") and not distributed.lower() == "true":
+ if hbase_rootdir and hbase_rootdir.startswith("hdfs://") and not distributed.lower() == "true":
distributed_item = self.getErrorItem("Distributed property should be set to true if hbase.rootdir points to HDFS.")
validationItems.extend([{"config-name":'hbase.rootdir', "item": rootdir_item },
@@ -532,7 +545,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
hbase_site = getSiteProperties(configurations, "ams-hbase-site")
hbase_rootdir = hbase_site.get("hbase.rootdir")
regionServerMinMemItem = None
- if hbase_rootdir.startswith("hdfs://"):
+ if hbase_rootdir and hbase_rootdir.startswith("hdfs://"):
regionServerMinMemItem = self.validateMinMemorySetting(properties, 1024, 'hbase_regionserver_heapsize')
validationItems = [{"config-name": "hbase_regionserver_heapsize", "item": regionServerItem},
http://git-wip-us.apache.org/repos/asf/ambari/blob/5b38d876/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 16f7e74..529df83 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -750,19 +750,22 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
{"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
{"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')}]
- if checkXmxValueFormat(properties['mapreduce.map.java.opts']):
+ if 'mapreduce.map.java.opts' in properties and \
+ checkXmxValueFormat(properties['mapreduce.map.java.opts']):
mapreduceMapJavaOpts = formatXmxSizeToBytes(getXmxSize(properties['mapreduce.map.java.opts'])) / (1024.0 * 1024)
mapreduceMapMemoryMb = to_number(properties['mapreduce.map.memory.mb'])
if mapreduceMapJavaOpts > mapreduceMapMemoryMb:
validationItems.append({"config-name": 'mapreduce.map.java.opts', "item": self.getWarnItem("mapreduce.map.java.opts Xmx should be less than mapreduce.map.memory.mb ({0})".format(mapreduceMapMemoryMb))})
- if checkXmxValueFormat(properties['mapreduce.reduce.java.opts']):
+ if 'mapreduce.reduce.java.opts' in properties and \
+ checkXmxValueFormat(properties['mapreduce.reduce.java.opts']):
mapreduceReduceJavaOpts = formatXmxSizeToBytes(getXmxSize(properties['mapreduce.reduce.java.opts'])) / (1024.0 * 1024)
mapreduceReduceMemoryMb = to_number(properties['mapreduce.reduce.memory.mb'])
if mapreduceReduceJavaOpts > mapreduceReduceMemoryMb:
validationItems.append({"config-name": 'mapreduce.reduce.java.opts', "item": self.getWarnItem("mapreduce.reduce.java.opts Xmx should be less than mapreduce.reduce.memory.mb ({0})".format(mapreduceReduceMemoryMb))})
- if checkXmxValueFormat(properties['yarn.app.mapreduce.am.command-opts']):
+ if 'yarn.app.mapreduce.am.command-opts' in properties and \
+ checkXmxValueFormat(properties['yarn.app.mapreduce.am.command-opts']):
yarnAppMapreduceAmCommandOpts = formatXmxSizeToBytes(getXmxSize(properties['yarn.app.mapreduce.am.command-opts'])) / (1024.0 * 1024)
yarnAppMapreduceAmResourceMb = to_number(properties['yarn.app.mapreduce.am.resource.mb'])
if yarnAppMapreduceAmCommandOpts > yarnAppMapreduceAmResourceMb:
@@ -822,13 +825,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
ranger_plugin_enabled = ranger_plugin_properties['ranger-hdfs-plugin-enabled'] if ranger_plugin_properties else 'No'
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()):
- if hdfs_site['dfs.permissions.enabled'] != 'true':
+ if 'dfs.permissions.enabled' in hdfs_site and \
+ hdfs_site['dfs.permissions.enabled'] != 'true':
validationItems.append({"config-name": 'dfs.permissions.enabled',
"item": self.getWarnItem(
"dfs.permissions.enabled needs to be set to true if Ranger HDFS Plugin is enabled.")})
if (not wire_encryption_enabled and # If wire encryption is enabled at Hadoop, it disables all our checks
+ 'hadoop.security.authentication' in core_site and
core_site['hadoop.security.authentication'] == 'kerberos' and
+ 'hadoop.security.authorization' in core_site and
core_site['hadoop.security.authorization'] == 'true'):
# security is enabled
@@ -961,7 +967,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
validationItems = []
hive_env = properties
hive_site = getSiteProperties(configurations, "hive-site")
- if str(hive_env["hive_security_authorization"]).lower() == "none" \
+ if "hive_security_authorization" in hive_env and \
+ str(hive_env["hive_security_authorization"]).lower() == "none" \
and str(hive_site["hive.security.authorization.enabled"]).lower() == "true":
authorization_item = self.getErrorItem("hive_security_authorization should not be None "
"if hive.security.authorization.enabled is set")
@@ -976,7 +983,9 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
#Adding Ranger Plugin logic here
ranger_plugin_properties = getSiteProperties(configurations, "ranger-hive-plugin-properties")
hive_env_properties = getSiteProperties(configurations, "hive-env")
- ranger_plugin_enabled = 'hive_security_authorization' in hive_env_properties and hive_env_properties['hive_security_authorization'].lower() == 'ranger'
+ ranger_plugin_enabled = hive_env_properties \
+ and 'hive_security_authorization' in hive_env_properties \
+ and hive_env_properties['hive_security_authorization'].lower() == 'ranger'
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
##Add stack validations only if Ranger is enabled.
if ("RANGER" in servicesList):
@@ -984,7 +993,9 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
if ranger_plugin_enabled:
prop_name = 'hive.security.authorization.enabled'
prop_val = 'true'
- if hive_site[prop_name] != prop_val:
+ if hive_site and \
+ prop_name in hive_site and \
+ hive_site[prop_name] != prop_val:
validationItems.append({"config-name": prop_name,
"item": self.getWarnItem(
"If Ranger Hive Plugin is enabled." \
@@ -992,7 +1003,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
prop_name = 'hive.conf.restricted.list'
prop_vals = 'hive.security.authorization.enabled,hive.security.authorization.manager,hive.security.authenticator.manager'.split(',')
- current_vals = hive_site[prop_name].split(',')
+ current_vals = []
+ if hive_site and prop_name in hive_site:
+ current_vals = hive_site[prop_name].split(',')
+
missing_vals = []
for val in prop_vals:
@@ -1006,7 +1020,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
" {0} needs to contain {1}".format(prop_name, ','.join(missing_vals)))})
stripe_size_values = [8388608, 16777216, 33554432, 67108864, 134217728, 268435456]
stripe_size_property = "hive.exec.orc.default.stripe.size"
- if int(properties[stripe_size_property]) not in stripe_size_values:
+ if stripe_size_property in properties and \
+ int(properties[stripe_size_property]) not in stripe_size_values:
validationItems.append({"config-name": stripe_size_property,
"item": self.getWarnItem("Correct values are {0}".format(stripe_size_values))
}
@@ -1023,15 +1038,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
prop_name2 = 'hfile.block.cache.size'
props_max_sum = 0.8
- if not is_number(hbase_site[prop_name1]):
+ if prop_name1 in hbase_site and not is_number(hbase_site[prop_name1]):
validationItems.append({"config-name": prop_name1,
"item": self.getWarnItem(
"{0} should be float value".format(prop_name1))})
- elif not is_number(hbase_site[prop_name2]):
+ elif prop_name2 in hbase_site and not is_number(hbase_site[prop_name2]):
validationItems.append({"config-name": prop_name2,
"item": self.getWarnItem(
"{0} should be float value".format(prop_name2))})
- elif float(hbase_site[prop_name1]) + float(hbase_site[prop_name2]) > props_max_sum:
+ elif prop_name1 in hbase_site and prop_name2 in hbase_site and \
+ float(hbase_site[prop_name1]) + float(hbase_site[prop_name2]) > props_max_sum:
validationItems.append({"config-name": prop_name1,
"item": self.getWarnItem(
"{0} and {1} sum should not exceed {2}".format(prop_name1, prop_name2, props_max_sum))})
http://git-wip-us.apache.org/repos/asf/ambari/blob/5b38d876/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 2f84be1..9329032 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -242,14 +242,14 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
if ranger_plugin_enabled:
prop_name = 'hive.security.authorization.manager'
prop_val = "org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory"
- if hive_server2[prop_name] != prop_val:
+ if prop_name in hive_server2 and hive_server2[prop_name] != prop_val:
validationItems.append({"config-name": prop_name,
"item": self.getWarnItem(
"If Ranger Hive Plugin is enabled."\
" {0} needs to be set to {1}".format(prop_name,prop_val))})
prop_name = 'hive.security.authenticator.manager'
prop_val = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator"
- if hive_server2[prop_name] != prop_val:
+ if prop_name in hive_server2 and hive_server2[prop_name] != prop_val:
validationItems.append({"config-name": prop_name,
"item": self.getWarnItem(
"If Ranger Hive Plugin is enabled."\
http://git-wip-us.apache.org/repos/asf/ambari/blob/5b38d876/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 0ef953d..708aaf7 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -566,7 +566,12 @@ class DefaultStackAdvisor(StackAdvisor):
for service in servicesList:
calculation = self.getServiceConfigurationRecommender(service)
if calculation is not None:
- calculation(configurations, clusterSummary, services, hosts)
+ try:
+ calculation(configurations, clusterSummary, services, hosts)
+ except (AttributeError, TypeError, LookupError) as e:
+ # NOP
+ print "Failed to recommend configuration "
+ print e
return recommendations
http://git-wip-us.apache.org/repos/asf/ambari/blob/5b38d876/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 8a37391..7f948bd 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -731,3 +731,84 @@ class TestHDP23StackAdvisor(TestCase):
res = self.stackAdvisor.validateHiveConfigurations(properties, recommendedDefaults, configurations, services, {})
self.assertEquals(res, res_expected)
+ # This test intentionally calls all validate methods with
+ # incorrect parameters (empty configs)
+ def test_noRiskyDictLookups(self):
+ properties = {}
+ recommendedDefaults = {}
+ configurations = {"core-site": {"properties": {}}}
+ services = {
+ "services": [],
+ "Versions": {
+ "stack_name": "HDP",
+ "stack_version": "2.3"
+ },
+ "configurations": configurations
+ }
+
+ hosts = {
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "disk_info" : [
+ {
+ "available" : "4564632",
+ "used" : "5230344",
+ "percent" : "54%",
+ "size" : "10319160",
+ "type" : "ext4",
+ "mountpoint" : "/"
+ },
+ {
+ "available" : "1832436",
+ "used" : "0",
+ "percent" : "0%",
+ "size" : "1832436",
+ "type" : "tmpfs",
+ "mountpoint" : "/dev/shm"
+ }
+ ],
+ "host_name" : "c6401.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ }
+ ]
+ }
+
+ def return_c6401_hostname(services, service_name, component_name):
+ return ["c6401.ambari.apache.org"]
+ self.stackAdvisor.getComponentHostNames = return_c6401_hostname
+
+ validators = self.stackAdvisor.getServiceConfigurationValidators()
+
+ # Setting up empty configs and services info
+ for serviceName, validator in validators.items():
+ services["services"].extend([{"StackServices": {"service_name": serviceName},
+ "components": []}])
+ for siteName in validator.keys():
+ configurations[siteName] = {"properties": {}}
+
+ # Emulate enabled RANGER
+ services["services"].extend([{"StackServices": {"service_name": "RANGER"},
+ "components": []}])
+ configurations["ranger-hbase-plugin-properties"] = {
+ "ranger-hbase-plugin-enabled": "Yes"
+ }
+
+ recommendations = self.stackAdvisor.recommendConfigurations(services, hosts)
+
+ for serviceName, validator in validators.items():
+ recommendedDefaults = recommendations["recommendations"]["blueprint"]["configurations"]
+ for siteName, method in validator.items():
+ if siteName in recommendedDefaults:
+ siteRecommendations = recommendedDefaults[siteName]["properties"]
+ method(properties, siteRecommendations, configurations, services, hosts)
+
+ pass