You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/12/28 14:59:01 UTC
[2/2] ambari git commit: AMBARI-19296. When adding service,
custom core-site properties get reverted back (echekanskiy via
dlisnichenko)
AMBARI-19296. When adding service, custom core-site properties get reverted back (echekanskiy via dlisnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d84aa3bc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d84aa3bc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d84aa3bc
Branch: refs/heads/trunk
Commit: d84aa3bcb184db60e8d1a68429e1c7abb3fa3646
Parents: 1bfffc0
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Wed Dec 28 16:58:27 2016 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Wed Dec 28 16:58:27 2016 +0200
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 51 +++++++++++++++-----
.../stacks/HDP/2.1/services/stack_advisor.py | 20 +-------
.../stacks/HDP/2.3/services/stack_advisor.py | 2 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 20 +-------
4 files changed, 43 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d84aa3bc/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index a352cdb..77ed527 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -205,11 +205,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if "HDFS" in servicesList:
ambari_user = self.getAmbariUser(services)
ambariHostName = socket.getfqdn()
- is_wildcard_value, hosts = self.get_hosts_for_proxyuser(ambari_user, services)
- if not is_wildcard_value:
- hosts.add(ambariHostName)
- putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(ambari_user), ",".join(hosts))
- putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(ambari_user), "*")
+ self.put_proxyuser_value(ambari_user, ambariHostName, services=services, put_function=putCoreSiteProperty)
+ self.put_proxyuser_value(ambari_user, "*", is_groups=True, services=services, put_function=putCoreSiteProperty)
old_ambari_user = self.getOldAmbariUser(services)
if old_ambari_user is not None:
putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
@@ -254,7 +251,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
for proxyPropertyName, hostSelector in hostSelectorMap.iteritems():
componentHostNamesString = hostSelector if isinstance(hostSelector, basestring) else '*'
if isinstance(hostSelector, (list, tuple)):
- _, componentHostNames = self.get_hosts_for_proxyuser(user, services) # preserve old values
+ _, componentHostNames = self.get_data_for_proxyuser(user, services) # preserve old values
for component in hostSelector:
componentHosts = self.getHostsWithComponent(serviceName, component, services, hosts)
if componentHosts is not None:
@@ -288,13 +285,17 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return users
- def get_hosts_for_proxyuser(self, user_name, services):
+ def get_data_for_proxyuser(self, user_name, services, groups=False):
if "core-site" in services["configurations"]:
coreSite = services["configurations"]["core-site"]['properties']
else:
coreSite = {}
- property_name = "hadoop.proxyuser.{0}.hosts".format(user_name)
+ if groups:
+ property_name = "hadoop.proxyuser.{0}.groups".format(user_name)
+ else:
+ property_name = "hadoop.proxyuser.{0}.hosts".format(user_name)
+
if property_name in coreSite:
property_value = coreSite[property_name]
if property_value == "*":
@@ -303,6 +304,32 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return False, set(property_value.split(","))
return False, set()
+ def put_proxyuser_value(self, user_name, value, is_groups=False, services=None, put_function=None):
+ is_wildcard_value, current_value = self.get_data_for_proxyuser(user_name, services, is_groups)
+ result_value = "*"
+ result_values_set = self.merge_proxyusers_values(current_value, value)
+ if len(result_values_set) > 0:
+ result_value = ",".join(sorted([val for val in result_values_set if val]))
+
+ if is_groups:
+ property_name = "hadoop.proxyuser.{0}.groups".format(user_name)
+ else:
+ property_name = "hadoop.proxyuser.{0}.hosts".format(user_name)
+
+ put_function(property_name, result_value)
+
+ def merge_proxyusers_values(self, first, second):
+ result = set()
+ def append(data):
+ if isinstance(data, str) or isinstance(data, unicode):
+ if data != "*":
+ result.update(data.split(","))
+ else:
+ result.update(data)
+ append(first)
+ append(second)
+ return result
+
def getServiceHadoopProxyUsersConfigurationDict(self):
"""
Returns a map that is used by 'getHadoopProxyUsers' to determine service
@@ -365,14 +392,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.hosts".format(hive_user)})
for user_name, user_properties in users.iteritems():
- is_wildcard_value, _ = self.get_hosts_for_proxyuser(user_name, services)
# Add properties "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" to core-site for all users
- if not is_wildcard_value:
- putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(user_name) , user_properties["propertyHosts"])
- Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(user_name, user_properties["propertyHosts"]))
+ self.put_proxyuser_value(user_name, user_properties["propertyHosts"], services=services, put_function=putCoreSiteProperty)
+ Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(user_name, user_properties["propertyHosts"]))
if "propertyGroups" in user_properties:
- putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(user_name) , user_properties["propertyGroups"])
+ self.put_proxyuser_value(user_name, user_properties["propertyGroups"], is_groups=True, services=services, put_function=putCoreSiteProperty)
# Remove old properties if user was renamed
userOldValue = getOldValue(self, services, user_properties["config"], user_properties["propertyName"])
http://git-wip-us.apache.org/repos/asf/ambari/blob/d84aa3bc/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 17225d0..f34f5a1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -139,27 +139,11 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
if self.is_secured_cluster(services):
- appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services)
-
- def updateCallback(originalValue, newValue):
- """
- :type originalValue str
- :type newValue list
- """
- if originalValue and not originalValue.isspace():
- hosts = originalValue.split(',')
-
- if newValue:
- hosts.extend(newValue)
-
- result = ','.join(set(hosts))
- return result
- else:
- return ','.join(set(newValue))
+ putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services)
if "hostnames" in meta:
- appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', meta["hostnames"], updateCallback)
+ self.put_proxyuser_value("HTTP", meta["hostnames"], services=services, put_function=putCoreSiteProperty)
def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
putTezProperty = self.putProperty(configurations, "tez-site")
http://git-wip-us.apache.org/repos/asf/ambari/blob/d84aa3bc/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index d97efe2..eb81655 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -485,7 +485,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList:
kmsUser = kmsEnvProperties['kms_user']
kmsUserOld = getOldValue(self, services, 'kms-env', 'kms_user')
- putCoreSiteProperty('hadoop.proxyuser.{0}.groups'.format(kmsUser), '*')
+ self.put_proxyuser_value(kmsUser, '*', is_groups=True, services=services, put_function=putCoreSiteProperty)
if kmsUserOld is not None and kmsUser != kmsUserOld:
putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true')
services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)})
http://git-wip-us.apache.org/repos/asf/ambari/blob/d84aa3bc/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index a130566..b52d753 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -618,7 +618,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP25StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts)
putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
- appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services)
+ putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
if "cluster-env" in services["configurations"] \
and "security_enabled" in services["configurations"]["cluster-env"]["properties"] \
@@ -632,24 +632,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
Logger.debug("Attempting to update hadoop.proxyuser.HTTP.hosts with %s" % str(phoenix_query_server_hosts))
# The PQS hosts we want to ensure are set
new_value = ','.join(phoenix_query_server_hosts)
- # Compute the unique set of hosts for the property
- def updateCallback(originalValue, newValue):
- Logger.debug("Original hadoop.proxyuser.HTTP.hosts value %s, appending %s" % (originalValue, newValue))
- # Only update the original value if it's not whitespace only
- if originalValue and not originalValue.isspace():
- hosts = originalValue.split(',')
- # Add in the new hosts if we have some
- if newValue and not newValue.isspace():
- hosts.extend(newValue.split(','))
- # Return the combined (uniqued) list of hosts
- result = ','.join(set(hosts))
- Logger.debug("Setting final to %s" % result)
- return result
- else:
- Logger.debug("Setting final value to %s" % newValue)
- return newValue
# Update the proxyuser setting, deferring to out callback to merge results together
- appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', new_value, updateCallback)
+ self.put_proxyuser_value("HTTP", new_value, services=services, put_function=putCoreSiteProperty)
else:
Logger.debug("No phoenix query server hosts to update")
else: