You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/10/06 17:02:27 UTC
[4/4] ambari git commit: AMBARI-18547. No validation errors recieved
for empty or removed hadoop.proxyuser.* properties (dlysnichenko)
AMBARI-18547. No validation errors recieved for empty or removed hadoop.proxyuser.* properties (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/55c1dc5c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/55c1dc5c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/55c1dc5c
Branch: refs/heads/trunk
Commit: 55c1dc5cb8015e3d114c186792dacd883b7aa820
Parents: cceb7d2
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Thu Oct 6 20:01:13 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Thu Oct 6 20:01:13 2016 +0300
----------------------------------------------------------------------
.../TestDataStructureUtils.py | 47 ++++
.../libraries/functions/data_structure_utils.py | 57 ++++
.../stacks/HDP/2.0.6/services/stack_advisor.py | 269 ++++++++++++-------
.../stacks/HDP/2.3/services/stack_advisor.py | 13 +-
.../stacks/2.0.6/common/test_stack_advisor.py | 215 +++++++++++++++
5 files changed, 495 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-agent/src/test/python/resource_management/TestDataStructureUtils.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestDataStructureUtils.py b/ambari-agent/src/test/python/resource_management/TestDataStructureUtils.py
new file mode 100644
index 0000000..a4bdc1c
--- /dev/null
+++ b/ambari-agent/src/test/python/resource_management/TestDataStructureUtils.py
@@ -0,0 +1,47 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+
+from resource_management.libraries.functions.data_structure_utils import get_from_dict
+from resource_management.libraries.functions.data_structure_utils import KeyNotFound
+
+
+class TestDictUtils(TestCase):
+
+ def test_get_nested(self):
+ dict_ = {1: {2: {3: 'data'}}}
+ empty_dict = {}
+
+ self.assertEquals('data', get_from_dict(dict_, (1, 2, 3)))
+ self.assertEquals('data', get_from_dict(dict_, [1, 2, 3]))
+
+ self.assertEquals({3: 'data'}, get_from_dict(dict_, (1, 2)))
+
+ self.assertEquals({2: {3: 'data'}}, get_from_dict(dict_, 1))
+
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, (1, 2, 0)))
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, [1, 2, 0]))
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, (1, 0, 3)))
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, (1, 2, 3, 4)))
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, (0, 2)))
+
+ self.assertEquals('default', get_from_dict(dict_, (0, 2, 3), default_value='default'))
+ self.assertEquals('default', get_from_dict(empty_dict, (0, 2, 3), default_value='default'))
+
+ self.assertEquals(KeyNotFound, get_from_dict(empty_dict, [1]))
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-common/src/main/python/resource_management/libraries/functions/data_structure_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/data_structure_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/data_structure_utils.py
new file mode 100644
index 0000000..59e01aa
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/data_structure_utils.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+__all__ = ['get_from_dict', 'convert_to_list']
+
+class KeyNotFound(object):
+ # Prohibit instantiation
+ def __new__(cls):
+ raise AttributeError("Instance creation is not supported.")
+
+def get_from_dict(dict_, keys, default_value=KeyNotFound):
+ """
+ Tries to get an element from the nested dictionaries (dict_) using the key sequence (keys).
+ For example, if keys == ['a', 'b', 'c'], will return dict_[a][b][c] if all keys are present in the map.
+ If one of the keys is absent, it returns 'default_value'.
+ """
+
+ keys = convert_to_list(keys)
+
+ curr_dict = dict_
+ if not curr_dict:
+ return default_value
+ for x in keys:
+ if isinstance(curr_dict, dict) and x in curr_dict:
+ curr_dict = curr_dict[x]
+ else:
+ return default_value
+
+ return curr_dict
+
+def convert_to_list(keys):
+ """
+ If 'keys' is a list or a tuple, this function returns it unchanged.
+ Otherwise returns a list that contains 'keys' as the only element.
+ """
+ if not isinstance(keys, (list, tuple)):
+ return [keys]
+ return keys
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 897a654..76c3112 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -26,6 +26,7 @@ from math import ceil, floor, log
from resource_management.core.logger import Logger
from resource_management.libraries.functions.mounted_dirs_helper import get_mounts_with_multiple_data_dirs
+from resource_management.libraries.functions.data_structure_utils import get_from_dict
from stack_advisor import DefaultStackAdvisor
@@ -171,118 +172,154 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return ambari_user
def recommendAmbariProxyUsersForHDFS(self, services, servicesList, putCoreSiteProperty, putCoreSitePropertyAttribute):
- if "HDFS" in servicesList:
- ambari_user = self.getAmbariUser(services)
- ambariHostName = socket.getfqdn()
- putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(ambari_user), ambariHostName)
- putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(ambari_user), "*")
- old_ambari_user = self.getOldAmbariUser(services)
- if old_ambari_user is not None:
- putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
- putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
-
- def recommendHadoopProxyUsers (self, configurations, services, hosts):
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if "HDFS" in servicesList:
+ ambari_user = self.getAmbariUser(services)
+ ambariHostName = socket.getfqdn()
+ putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(ambari_user), ambariHostName)
+ putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(ambari_user), "*")
+ old_ambari_user = self.getOldAmbariUser(services)
+ if old_ambari_user is not None:
+ putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
+ putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
+
+ def getAmbariProxyUsersForHDFSValidationItems(self, properties, services):
+ validationItems = []
+ servicesList = self.get_services_list(services)
+
+ if "HDFS" in servicesList:
+ ambari_user = self.getAmbariUser(services)
+ props = (
+ "hadoop.proxyuser.{0}.hosts".format(ambari_user),
+ "hadoop.proxyuser.{0}.groups".format(ambari_user)
+ )
+ for prop in props:
+ validationItems.append({"config-name": prop, "item": self.validatorNotEmpty(properties, prop)})
+
+ return validationItems
+
+ def _getHadoopProxyUsersForService(self, serviceName, serviceUserComponents, services, hosts):
+ Logger.info("Calculating Hadoop Proxy User recommendations for {0} service.".format(serviceName))
+ servicesList = self.get_services_list(services)
+ resultUsers = {}
+
+ if serviceName in servicesList:
+ usersComponents = {}
+ for values in serviceUserComponents:
+
+ # Filter, if 4th argument is present in the tuple
+ filterFunction = values[3:]
+ if filterFunction and not filterFunction[0](services, hosts):
+ continue
+
+ userNameConfig, userNameProperty, hostSelectorMap = values[:3]
+ user = get_from_dict(services, ("configurations", userNameConfig, "properties", userNameProperty), None)
+ if user:
+ usersComponents[user] = (userNameConfig, userNameProperty, hostSelectorMap)
+
+ for user, (userNameConfig, userNameProperty, hostSelectorMap) in usersComponents.iteritems():
+ proxyUsers = {"config": userNameConfig, "propertyName": userNameProperty}
+ for proxyPropertyName, hostSelector in hostSelectorMap.iteritems():
+ componentHostNamesString = hostSelector if isinstance(hostSelector, basestring) else '*'
+ if isinstance(hostSelector, (list, tuple)):
+ componentHostNames = set()
+ for component in hostSelector:
+ componentHosts = self.getHostsWithComponent(serviceName, component, services, hosts)
+ if componentHosts is not None:
+ for componentHost in componentHosts:
+ componentHostName = componentHost["Hosts"]["host_name"]
+ componentHostNames.add(componentHostName)
+
+ componentHostNamesString = ",".join(sorted(componentHostNames))
+ Logger.info("Host List for [service='{0}'; user='{1}'; components='{2}']: {3}".format(serviceName, user, ','.join(hostSelector), componentHostNamesString))
+
+ if not proxyPropertyName in proxyUsers:
+ proxyUsers[proxyPropertyName] = componentHostNamesString
+
+ if not user in resultUsers:
+ resultUsers[user] = proxyUsers
+
+ return resultUsers
+
+ def getHadoopProxyUsers(self, services, hosts):
+ """
+ Gets Hadoop Proxy User recommendations based on the configuration that is provided by
+ getServiceHadoopProxyUsersConfigurationDict.
+
+ See getServiceHadoopProxyUsersConfigurationDict
+ """
+ servicesList = self.get_services_list(services)
users = {}
+ for serviceName, serviceUserComponents in self.getServiceHadoopProxyUsersConfigurationDict().iteritems():
+ users.update(self._getHadoopProxyUsersForService(serviceName, serviceUserComponents, services, hosts))
+
+ return users
+
+ def getServiceHadoopProxyUsersConfigurationDict(self):
+ """
+ Returns a map that is used by 'getHadoopProxyUsers' to determine service
+ user properties and related components and get proxyuser recommendations.
+ This method can be overridden in stackadvisors for the further stacks to
+ add additional services or change the previous logic.
+
+ Example of the map format:
+ {
+ "serviceName": [
+ ("configTypeName1", "userPropertyName1", {"propertyHosts": "*", "propertyGroups": "exact string value"})
+ ("configTypeName2", "userPropertyName2", {"propertyHosts": ["COMPONENT1", "COMPONENT2", "COMPONENT3"], "propertyGroups": "*"}),
+ ("configTypeName3", "userPropertyName3", {"propertyHosts": ["COMPONENT1", "COMPONENT2", "COMPONENT3"]}, filterFunction)
+ ],
+ "serviceName2": [
+ ...
+ }
+
+ If the third element of a tuple is map that maps proxy property to it's value.
+ The key could be either 'propertyHosts' or 'propertyGroups'. (Both are optional)
+ If the map value is a string, then this string will be used for the proxyuser
+ value (e.g. 'hadoop.proxyuser.{user}.hosts' = '*').
+ Otherwise map value should be alist or a tuple with component names.
+ All hosts with the provided components will be added
+ to the property (e.g. 'hadoop.proxyuser.{user}.hosts' = 'host1,host2,host3')
+
+ The forth element of the tuple is optional and if it's provided,
+ it should be a function that takes two arguments: services and hosts.
+ If it returns False, proxyusers for the tuple will not be added.
+ """
+ ALL_WILDCARD = "*"
+ HOSTS_PROPERTY = "propertyHosts"
+ GROUPS_PROPERTY = "propertyGroups"
+
+ return {
+ "HDFS": [("hadoop-env", "hdfs_user", {HOSTS_PROPERTY: ALL_WILDCARD, GROUPS_PROPERTY: ALL_WILDCARD})],
+ "OOZIE": [("oozie-env", "oozie_user", {HOSTS_PROPERTY: ["OOZIE_SERVER"], GROUPS_PROPERTY: ALL_WILDCARD})],
+ "HIVE": [("hive-env", "hive_user", {HOSTS_PROPERTY: ["HIVE_SERVER", "HIVE_SERVER_INTERACTIVE"], GROUPS_PROPERTY: ALL_WILDCARD}),
+ ("hive-env", "webhcat_user", {HOSTS_PROPERTY: ["WEBHCAT_SERVER"], GROUPS_PROPERTY: ALL_WILDCARD})],
+ "YARN": [("yarn-env", "yarn_user", {HOSTS_PROPERTY: ["RESOURCEMANAGER"]}, lambda services, hosts: len(self.getHostsWithComponent("YARN", "RESOURCEMANAGER", services, hosts)) > 1)],
+ "FALCON": [("falcon-env", "falcon_user", {HOSTS_PROPERTY: ALL_WILDCARD, GROUPS_PROPERTY: ALL_WILDCARD})],
+ "SPARK": [("livy-env", "livy_user", {HOSTS_PROPERTY: ALL_WILDCARD, GROUPS_PROPERTY: ALL_WILDCARD})]
+ }
+
+ def recommendHadoopProxyUsers(self, configurations, services, hosts):
+ servicesList = self.get_services_list(services)
+
if 'forced-configurations' not in services:
services["forced-configurations"] = []
- if "HDFS" in servicesList:
- hdfs_user = None
- if "hadoop-env" in services["configurations"] and "hdfs_user" in services["configurations"]["hadoop-env"]["properties"]:
- hdfs_user = services["configurations"]["hadoop-env"]["properties"]["hdfs_user"]
- if not hdfs_user in users and hdfs_user is not None:
- users[hdfs_user] = {"propertyHosts" : "*","propertyGroups" : "*", "config" : "hadoop-env", "propertyName" : "hdfs_user"}
-
- if "OOZIE" in servicesList:
- oozie_user = None
- if "oozie-env" in services["configurations"] and "oozie_user" in services["configurations"]["oozie-env"]["properties"]:
- oozie_user = services["configurations"]["oozie-env"]["properties"]["oozie_user"]
- oozieServerrHosts = self.getHostsWithComponent("OOZIE", "OOZIE_SERVER", services, hosts)
- if oozieServerrHosts is not None:
- oozieServerHostsNameList = []
- for oozieServerHost in oozieServerrHosts:
- oozieServerHostsNameList.append(oozieServerHost["Hosts"]["host_name"])
- oozieServerHostsNames = ",".join(oozieServerHostsNameList)
- if not oozie_user in users and oozie_user is not None:
- users[oozie_user] = {"propertyHosts" : oozieServerHostsNames,"propertyGroups" : "*", "config" : "oozie-env", "propertyName" : "oozie_user"}
-
- hive_user = None
- if "HIVE" in servicesList:
- webhcat_user = None
- if "hive-env" in services["configurations"] and "hive_user" in services["configurations"]["hive-env"]["properties"] \
- and "webhcat_user" in services["configurations"]["hive-env"]["properties"]:
- hive_user = services["configurations"]["hive-env"]["properties"]["hive_user"]
- webhcat_user = services["configurations"]["hive-env"]["properties"]["webhcat_user"]
- hiveServerHosts = self.getHostsWithComponent("HIVE", "HIVE_SERVER", services, hosts)
- hiveServerInteractiveHosts = self.getHostsWithComponent("HIVE", "HIVE_SERVER_INTERACTIVE", services, hosts)
- webHcatServerHosts = self.getHostsWithComponent("HIVE", "WEBHCAT_SERVER", services, hosts)
-
- if hiveServerHosts is not None:
- hiveServerHostsNameList = []
- for hiveServerHost in hiveServerHosts:
- hiveServerHostsNameList.append(hiveServerHost["Hosts"]["host_name"])
- # Append Hive Server Interactive host as well, as it is Hive2/HiveServer2 component.
- if hiveServerInteractiveHosts:
- for hiveServerInteractiveHost in hiveServerInteractiveHosts:
- hiveServerInteractiveHostName = hiveServerInteractiveHost["Hosts"]["host_name"]
- if hiveServerInteractiveHostName not in hiveServerHostsNameList:
- hiveServerHostsNameList.append(hiveServerInteractiveHostName)
- Logger.info("Appended (if not exiting), Hive Server Interactive Host : '{0}', to Hive Server Host List : '{1}'".format(hiveServerInteractiveHostName, hiveServerHostsNameList))
-
- hiveServerHostsNames = ",".join(hiveServerHostsNameList) # includes Hive Server interactive host also.
- Logger.info("Hive Server and Hive Server Interactive (if enabled) Host List : {0}".format(hiveServerHostsNameList))
- if not hive_user in users and hive_user is not None:
- users[hive_user] = {"propertyHosts" : hiveServerHostsNames,"propertyGroups" : "*", "config" : "hive-env", "propertyName" : "hive_user"}
-
- if webHcatServerHosts is not None:
- webHcatServerHostsNameList = []
- for webHcatServerHost in webHcatServerHosts:
- webHcatServerHostsNameList.append(webHcatServerHost["Hosts"]["host_name"])
- webHcatServerHostsNames = ",".join(webHcatServerHostsNameList)
- if not webhcat_user in users and webhcat_user is not None:
- users[webhcat_user] = {"propertyHosts" : webHcatServerHostsNames,"propertyGroups" : "*", "config" : "hive-env", "propertyName" : "webhcat_user"}
-
- if "YARN" in servicesList:
- yarn_user = None
- if "yarn-env" in services["configurations"] and "yarn_user" in services["configurations"]["yarn-env"]["properties"]:
- yarn_user = services["configurations"]["yarn-env"]["properties"]["yarn_user"]
- rmHosts = self.getHostsWithComponent("YARN", "RESOURCEMANAGER", services, hosts)
-
- if len(rmHosts) > 1:
- rmHostsNameList = []
- for rmHost in rmHosts:
- rmHostsNameList.append(rmHost["Hosts"]["host_name"])
- rmHostsNames = ",".join(rmHostsNameList)
- if not yarn_user in users and yarn_user is not None:
- users[yarn_user] = {"propertyHosts" : rmHostsNames, "config" : "yarn-env", "propertyName" : "yarn_user"}
-
-
- if "FALCON" in servicesList:
- falconUser = None
- if "falcon-env" in services["configurations"] and "falcon_user" in services["configurations"]["falcon-env"]["properties"]:
- falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"]
- if not falconUser in users and falconUser is not None:
- users[falconUser] = {"propertyHosts" : "*","propertyGroups" : "*", "config" : "falcon-env", "propertyName" : "falcon_user"}
-
- if "SPARK" in servicesList:
- livyUser = None
- if "livy-env" in services["configurations"] and "livy_user" in services["configurations"]["livy-env"]["properties"]:
- livyUser = services["configurations"]["livy-env"]["properties"]["livy_user"]
- if not livyUser in users and livyUser is not None:
- users[livyUser] = {"propertyHosts" : "*","propertyGroups" : "*", "config" : "livy-env", "propertyName" : "livy_user"}
-
putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site")
+ users = self.getHadoopProxyUsers(services, hosts)
+
+ # Force dependencies for HIVE
+ if "HIVE" in servicesList:
+ hive_user = get_from_dict(services, ("configurations", "hive-env", "properties", "hive_user"), default_value=None)
+ if hive_user and get_from_dict(users, (hive_user, "propertyHosts"), default_value=None):
+ services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.hosts".format(hive_user)})
+
for user_name, user_properties in users.iteritems():
- if hive_user and hive_user == user_name:
- if "propertyHosts" in user_properties:
- services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.hosts".format(hive_user)})
# Add properties "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" to core-site for all users
putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(user_name) , user_properties["propertyHosts"])
- Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(hive_user, user_properties["propertyHosts"]))
+ Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(user_name, user_properties["propertyHosts"]))
if "propertyGroups" in user_properties:
putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(user_name) , user_properties["propertyGroups"])
@@ -300,6 +337,19 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
self.recommendAmbariProxyUsersForHDFS(services, servicesList, putCoreSiteProperty, putCoreSitePropertyAttribute)
+ def getHadoopProxyUsersValidationItems(self, properties, services, hosts):
+ validationItems = []
+ users = self.getHadoopProxyUsers(services, hosts)
+ for user_name, user_properties in users.iteritems():
+ props = ["hadoop.proxyuser.{0}.hosts".format(user_name)]
+ if "propertyGroups" in user_properties:
+ props.append("hadoop.proxyuser.{0}.groups".format(user_name))
+
+ for prop in props:
+ validationItems.append({"config-name": prop, "item": self.validatorNotEmpty(properties, prop)})
+
+ return validationItems
+
def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
putHDFSProperty = self.putProperty(configurations, "hadoop-env", services)
putHDFSSiteProperty = self.putProperty(configurations, "hdfs-site", services)
@@ -938,7 +988,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
def getServiceConfigurationValidators(self):
return {
"HDFS": { "hdfs-site": self.validateHDFSConfigurations,
- "hadoop-env": self.validateHDFSConfigurationsEnv},
+ "hadoop-env": self.validateHDFSConfigurationsEnv,
+ "core-site": self.validateHDFSConfigurationsCoreSite},
"MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
"YARN": {"yarn-site": self.validateYARNConfigurations,
"yarn-env": self.validateYARNEnvConfigurations},
@@ -1396,6 +1447,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"for property {1}".format(recommendedValue, propertyName))
return None
+ def validatorNotEmpty(self, properties, propertyName):
+ if not propertyName in properties:
+ return self.getErrorItem("Value should be set for {0}".format(propertyName))
+ value = properties.get(propertyName)
+ if not value:
+ return self.getWarnItem("Empty value for {0}".format(propertyName))
+ return None
+
def validateMinMemorySetting(self, properties, defaultValue, propertyName):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
@@ -1511,6 +1570,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
{"config-name": 'namenode_opt_maxnewsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_maxnewsize')}]
return self.toConfigurationValidationProblems(validationItems, "hadoop-env")
+ def validateHDFSConfigurationsCoreSite(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+ validationItems.extend(self.getHadoopProxyUsersValidationItems(properties, services, hosts))
+ validationItems.extend(self.getAmbariProxyUsersForHDFSValidationItems(properties, services))
+ return self.toConfigurationValidationProblems(validationItems, "core-site")
+
def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index ee96cf8..517a6f3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -805,7 +805,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
return configurationValidationProblems
def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
- super(HDP23StackAdvisor, self).validateHiveServer2Configurations(properties, recommendedDefaults, configurations, services, hosts)
+ parentValidationProblems = super(HDP23StackAdvisor, self).validateHiveServer2Configurations(properties, recommendedDefaults, configurations, services, hosts)
hive_server2 = properties
validationItems = []
#Adding Ranger Plugin logic here
@@ -870,10 +870,13 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
"item": self.getWarnItem(
"If Ranger Hive Plugin is disabled."\
" {0} needs to be set to {1}".format(prop_name,prop_val))})
- return self.toConfigurationValidationProblems(validationItems, "hiveserver2-site")
+
+ validationProblems = self.toConfigurationValidationProblems(validationItems, "hiveserver2-site")
+ validationProblems.extend(parentValidationProblems)
+ return validationProblems
def validateHBASEConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- super(HDP23StackAdvisor, self).validateHBASEConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ parentValidationProblems = super(HDP23StackAdvisor, self).validateHBASEConfigurations(properties, recommendedDefaults, configurations, services, hosts)
hbase_site = properties
validationItems = []
@@ -909,7 +912,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
"If Ranger HBase Plugin is enabled."\
" {0} needs to contain {1} instead of {2}".format(prop_name,prop_val,exclude_val))})
- return self.toConfigurationValidationProblems(validationItems, "hbase-site")
+ validationProblems = self.toConfigurationValidationProblems(validationItems, "hbase-site")
+ validationProblems.extend(parentValidationProblems)
+ return validationProblems
def validateKAFKAConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
kafka_broker = properties
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index cd69dc3..ec9ade5 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -1620,6 +1620,221 @@ class TestHDP206StackAdvisor(TestCase):
res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
self.assertEquals(res, res_expected)
+ @patch("socket.getfqdn", new=lambda: 'test-mock-ambari-server-hostname1')
+ def test_recommendHadoopProxyUsers(self):
+ # input data stub
+ configurations = {'hadoop-env': {'properties': {'hdfs_user': 'hdfs-user'}},
+ 'yarn-env': {'properties': {'yarn_user': 'yarn-user'}},
+ 'oozie-env': {'properties': {'oozie_user': 'oozie-user'}},
+ 'hive-env': {'properties': {'hive_user': 'hive-user',
+ 'webhcat_user': 'webhcat-user'}},
+ 'falcon-env': {'properties': {'falcon_user': 'falcon-user'}},
+ 'livy-env': {'properties': {'livy_user': 'livy-user'}}
+ }
+ services = {
+ 'services': [
+ { 'StackServices': {'service_name': 'HDFS'}},
+ { 'StackServices': {'service_name': 'FALCON'}},
+ { 'StackServices': {'service_name': 'SPARK'}},
+ { 'StackServices': {'service_name': 'YARN'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'RESOURCEMANAGER',
+ 'hostnames': ['host1', 'host2']
+ }
+ }
+ ]
+ },
+ { 'StackServices': {'service_name': 'OOZIE'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'OOZIE_SERVER',
+ 'hostnames': ['host2']
+ }
+ }
+ ]
+ },
+ { 'StackServices': {'service_name': 'HIVE'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'HIVE_SERVER',
+ 'hostnames': ['host1']
+ }
+ },
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'HIVE_SERVER_INTERACTIVE',
+ 'hostnames': ['host3']
+ }
+ },
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'WEBHCAT_SERVER',
+ 'hostnames': ['host4']
+ }
+ }
+ ]
+ },
+ ],
+ 'ambari-server-properties': {'ambari-server.user': 'ambari-user'},
+ 'configurations': configurations
+ }
+ hosts = {
+ 'items' : [
+ {'Hosts' : {'host_name' : 'host1'}},
+ {'Hosts' : {'host_name' : 'host2'}},
+ {'Hosts' : {'host_name' : 'host3'}},
+ {'Hosts' : {'host_name' : 'host4'}}
+ ]
+ }
+
+ # 1) ok: check recommendations
+ expected = {
+ 'hadoop.proxyuser.ambari-user.groups': '*',
+ 'hadoop.proxyuser.ambari-user.hosts': 'test-mock-ambari-server-hostname1',
+ 'hadoop.proxyuser.falcon-user.groups': '*',
+ 'hadoop.proxyuser.falcon-user.hosts': '*',
+ 'hadoop.proxyuser.hdfs-user.groups': '*',
+ 'hadoop.proxyuser.hdfs-user.hosts': '*',
+ 'hadoop.proxyuser.hive-user.groups': '*',
+ 'hadoop.proxyuser.hive-user.hosts': 'host1,host3',
+ 'hadoop.proxyuser.livy-user.groups': '*',
+ 'hadoop.proxyuser.livy-user.hosts': '*',
+ 'hadoop.proxyuser.oozie-user.groups': '*',
+ 'hadoop.proxyuser.oozie-user.hosts': 'host2',
+ 'hadoop.proxyuser.webhcat-user.groups': '*',
+ 'hadoop.proxyuser.webhcat-user.hosts': 'host4',
+ 'hadoop.proxyuser.yarn-user.hosts': 'host1,host2'
+ }
+
+ self.stackAdvisor.recommendHadoopProxyUsers(configurations, services, hosts)
+ self.assertEquals(expected, configurations['core-site']['properties'])
+
+ @patch("socket.getfqdn", new=lambda: 'test-mock-ambari-server-hostname1')
+ def test_validateHDFSConfigurationsCoreSite(self):
+ # input data stub
+ configurations = {'hadoop-env': {'properties': {'hdfs_user': 'hdfs-user'}},
+ 'yarn-env': {'properties': {'yarn_user': 'yarn-user'}}}
+ recommendedDefaults = {'hadoop.proxyuser.ambari-user.hosts': '*',
+ 'hadoop.proxyuser.ambari-user.groups': '*',
+ 'hadoop.proxyuser.hdfs-user.hosts': '*',
+ 'hadoop.proxyuser.hdfs-user.groups': '*',
+ 'hadoop.proxyuser.yarn-user.hosts': 'host1,host2',
+ 'hadoop.proxyuser.yarn-user.groups': '*'}
+ properties = {'hadoop.proxyuser.ambari-user.hosts': '*',
+ 'hadoop.proxyuser.ambari-user.groups': '*',
+ 'hadoop.proxyuser.hdfs-user.hosts': '*',
+ 'hadoop.proxyuser.hdfs-user.groups': '*',
+ 'hadoop.proxyuser.yarn-user.hosts': 'host1,host2',
+ 'hadoop.proxyuser.yarn-user.groups': '*'}
+ services = {
+ 'services': [
+ { 'StackServices': {'service_name': 'HDFS'}},
+ { 'StackServices': {'service_name': 'YARN'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'RESOURCEMANAGER',
+ 'hostnames': ['host1', 'host2']
+ }
+ }
+ ]
+ }
+ ],
+ 'ambari-server-properties': {'ambari-server.user': 'ambari-user'},
+ 'configurations': configurations
+ }
+ hosts = {
+ 'items' : [
+ {'Hosts' : {'host_name' : 'host1'}},
+ {'Hosts' : {'host_name' : 'host2'}}
+ ]
+ }
+
+ # 1) ok: HDFS and Ambari proxyusers are present
+ res_expected = []
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEquals(res, res_expected)
+
+ # 2) fail: test filter function: two RESOURCE_MANAGERs, hadoop.proxyuser.yarn-user.hosts is expected to be set
+ del properties['hadoop.proxyuser.yarn-user.hosts']
+ res_expected = [{'config-name': 'hadoop.proxyuser.yarn-user.hosts',
+ 'config-type': 'core-site',
+ 'level': 'ERROR',
+ 'message': 'Value should be set for hadoop.proxyuser.yarn-user.hosts',
+ 'type': 'configuration'}]
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEquals(res, res_expected)
+
+ # 3) ok: test filter function: only one RESOURCE_MANAGER
+ services['services'][1]['components'][0]['StackServiceComponents']['hostnames'] = ["host1"]
+ res_expected = []
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEquals(res, res_expected)
+
+ # 4) fail: some proxyusers are empty or absent:
+ del properties['hadoop.proxyuser.ambari-user.hosts']
+ properties['hadoop.proxyuser.hdfs-user.groups'] = ''
+ res_expected = [{'config-name': 'hadoop.proxyuser.hdfs-user.groups',
+ 'config-type': 'core-site',
+ 'level': 'WARN',
+ 'message': 'Empty value for hadoop.proxyuser.hdfs-user.groups',
+ 'type': 'configuration'},
+ {'config-type': 'core-site',
+ 'message': 'Value should be set for hadoop.proxyuser.ambari-user.hosts',
+ 'type': 'configuration',
+ 'config-name': 'hadoop.proxyuser.ambari-user.hosts',
+ 'level': 'ERROR'}]
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEquals(res, res_expected)
+
+ def test_getHadoopProxyUsers(self):
+ # input data stub
+ configurations = {'hadoop-env': {'properties': {'hdfs_user': 'hdfs-user'}},
+ 'yarn-env': {'properties': {'yarn_user': 'yarn-user'}}}
+ services = {
+ 'services': [
+ { 'StackServices': {'service_name': 'HDFS'}},
+ { 'StackServices': {'service_name': 'YARN'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'RESOURCEMANAGER',
+ 'hostnames': ['host1', 'host2']
+ }
+ }
+ ]
+ }
+ ],
+ 'ambari-server-properties': {'ambari-server.user': 'ambari-user'},
+ 'configurations': configurations
+ }
+ hosts = {
+ 'items' : [
+ {'Hosts' : {'host_name' : 'host1'}},
+ {'Hosts' : {'host_name' : 'host2'}}
+ ]
+ }
+
+ # 1) HDFS + YARN:
+ res_expected = {
+ 'hdfs-user': {'propertyName': 'hdfs_user', 'config': 'hadoop-env', 'propertyHosts': '*', 'propertyGroups': '*'},
+ 'yarn-user': {'propertyName': 'yarn_user', 'config': 'yarn-env', 'propertyHosts': 'host1,host2'}
+ }
+ res = self.stackAdvisor.getHadoopProxyUsers(services, hosts)
+ self.assertEquals(res, res_expected)
+
+ # 2) test filter function: only one RESOURCE_MANAGER
+ services['services'][1]['components'][0]['StackServiceComponents']['hostnames'] = ["host1"]
+ res_expected = {
+ 'hdfs-user': {'propertyName': 'hdfs_user', 'config': 'hadoop-env', 'propertyHosts': '*', 'propertyGroups': '*'}
+ }
+ res = self.stackAdvisor.getHadoopProxyUsers(services, hosts)
+ self.assertEquals(res, res_expected)
+
def test_validateOneDataDirPerPartition(self):
recommendedDefaults = {
'dfs.datanode.du.reserved': '1024'