You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/05/22 21:02:22 UTC
[1/2] ambari git commit: AMBARI-11274. Update Authorizer classnames -
from XaSecure to Ranger. BE patch. (alexantonenko)
Repository: ambari
Updated Branches:
refs/heads/trunk 52aaaa0df -> 11ef5ca16
AMBARI-11274. Update Authorizer classnames - from XaSecure to Ranger. BE patch. (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/11ef5ca1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/11ef5ca1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/11ef5ca1
Branch: refs/heads/trunk
Commit: 11ef5ca16ec87fe29e6b20cf671bd7925c7c8ab0
Parents: 6a572bb
Author: Alex Antonenko <hi...@gmail.com>
Authored: Fri May 22 19:47:08 2015 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Fri May 22 21:55:16 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.3/services/stack_advisor.py | 128 ++++++++++++++++++-
1 file changed, 123 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/11ef5ca1/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 1304e7c..a892c8d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -24,6 +24,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
childRecommendConfDict = {
"TEZ": self.recommendTezConfigurations,
"HDFS": self.recommendHDFSConfigurations,
+ "HIVE": self.recommendHIVEConfigurations,
+ "HBASE": self.recommendHBASEConfigurations
}
parentRecommendConfDict.update(childRecommendConfDict)
return parentRecommendConfDict
@@ -43,6 +45,37 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
putTezAttribute = self.putPropertyAttribute(configurations, "tez-site")
putTezAttribute("tez.runtime.io.sort.mb", "maximum", 2047)
+ def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
+ super(HDP23StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts)
+ putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
+ putHbaseSitePropertyAttributes = self.putPropertyAttribute(configurations, "hbase-site")
+
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if 'ranger-hbase-plugin-properties' in services['configurations'] and ('ranger-hbase-plugin-enabled' in services['configurations']['ranger-hbase-plugin-properties']['properties']):
+ rangerPluginEnabled = services['configurations']['ranger-hbase-plugin-properties']['properties']['ranger-hbase-plugin-enabled']
+ if ("RANGER" in servicesList) and (rangerPluginEnabled.lower() == "Yes".lower()):
+ putHbaseSiteProperty("hbase.security.authorization", 'true')
+ putHbaseSiteProperty("hbase.coprocessor.master.classes", 'org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor')
+ putHbaseSiteProperty("hbase.coprocessor.region.classes", 'org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor')
+
+
+ def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts):
+ super(HDP23StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+
+ putHiveServerProperty = self.putProperty(configurations, "hiveserver2-site", services)
+
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if 'ranger-hive-plugin-properties' in services['configurations'] and ('ranger-hive-plugin-enabled' in services['configurations']['ranger-hive-plugin-properties']['properties']):
+ rangerPluginEnabled = services['configurations']['ranger-hive-plugin-properties']['properties']['ranger-hive-plugin-enabled']
+ if ("RANGER" in servicesList) :
+ if (rangerPluginEnabled.lower() == "Yes".lower()):
+ putHiveServerProperty("hive.security.authorization.manager", 'org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory')
+ putHiveServerProperty("hive.security.authenticator.manager", 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator')
+ elif (rangerPluginEnabled.lower() == "No".lower()):
+ putHiveServerProperty("hive.security.authorization.manager", 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory')
+ putHiveServerProperty("hive.security.authenticator.manager", 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator')
+
+
def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
super(HDP23StackAdvisor, self).recommendHDFSConfigurations(configurations, clusterData, services, hosts)
@@ -56,7 +89,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
def getServiceConfigurationValidators(self):
parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
childValidators = {
- "HDFS": {"hdfs-site": self.validateHDFSConfigurations}
+ "HDFS": {"hdfs-site": self.validateHDFSConfigurations},
+ "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations},
+ "HBASE": {"hbase-site": self.validateHBASEConfigurations}
}
parentValidators.update(childValidators)
return parentValidators
@@ -67,8 +102,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
# We can not access property hadoop.security.authentication from the
# other config (core-site). That's why we are using another heuristics here
hdfs_site = properties
- validationItems = []
- #Adding Ranger Plugin logic here
+ validationItems = [] #Adding Ranger Plugin logic here
ranger_plugin_properties = getSiteProperties(configurations, "ranger-hdfs-plugin-properties")
ranger_plugin_enabled = ranger_plugin_properties['ranger-hdfs-plugin-enabled'] if ranger_plugin_properties else 'No'
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
@@ -76,9 +110,93 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
if hdfs_site['dfs.namenode.inode.attributes.provider.class'].lower() != 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer'.lower():
validationItems.append({"config-name": 'dfs.namenode.inode.attributes.provider.class',
"item": self.getWarnItem(
- "dfs.namenode.inode.attributes.provider.class needs to be set to 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer' if Ranger HDFS Plugin is enabled.")})
+ "dfs.namenode.inode.attributes.provider.class needs to be set to 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer' if Ranger HDFS Plugin is enabled.")})
return self.toConfigurationValidationProblems(validationItems, "hdfs-site")
+ def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ super(HDP23StackAdvisor, self).validateHiveServer2Configurations(properties, recommendedDefaults, configurations, services, hosts)
+ hive_server2 = properties
+ validationItems = []
+ #Adding Ranger Plugin logic here
+ ranger_plugin_properties = getSiteProperties(configurations, "ranger-hive-plugin-properties")
+ ranger_plugin_enabled = ranger_plugin_properties['ranger-hive-plugin-enabled']
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ ##Add stack validations only if Ranger is enabled.
+ if ("RANGER" in servicesList):
+ ##Add stack validations for Ranger plugin enabled.
+ if (ranger_plugin_enabled.lower() == 'Yes'.lower()):
+ prop_name = 'hive.security.authorization.manager'
+ prop_val = "org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory"
+ if hive_server2[prop_name] != prop_val:
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "If Ranger Hive Plugin is enabled."\
+ " {0} needs to be set to {1}".format(prop_name,prop_val))})
+ prop_name = 'hive.security.authenticator.manager'
+ prop_val = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator"
+ if hive_server2[prop_name] != prop_val:
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "If Ranger Hive Plugin is enabled."\
+ " {0} needs to be set to {1}".format(prop_name,prop_val))})
+ ##Add stack validations for Ranger plugin disabled.
+ elif (ranger_plugin_enabled.lower() == 'No'.lower()):
+ prop_name = 'hive.security.authorization.manager'
+ prop_val = "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"
+ if hive_server2[prop_name] != prop_val:
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "If Ranger Hive Plugin is disabled."\
+ " {0} needs to be set to {1}".format(prop_name,prop_val))})
+ prop_name = 'hive.security.authenticator.manager'
+ prop_val = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator"
+ if hive_server2[prop_name] != prop_val:
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "If Ranger Hive Plugin is disabled."\
+ " {0} needs to be set to {1}".format(prop_name,prop_val))})
+ return self.toConfigurationValidationProblems(validationItems, "hiveserver2-site")
+
+ def validateHBASEConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ super(HDP23StackAdvisor, self).validateHBASEConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ hbase_site = properties
+ validationItems = []
+
+ #Adding Ranger Plugin logic here
+ ranger_plugin_properties = getSiteProperties(configurations, "ranger-hbase-plugin-properties")
+ ranger_plugin_enabled = ranger_plugin_properties['ranger-hbase-plugin-enabled']
+ prop_name = 'hbase.security.authorization'
+ prop_val = "true"
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()):
+ if hbase_site[prop_name] != prop_val:
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "If Ranger HBase Plugin is enabled."\
+ "{0} needs to be set to {1}".format(prop_name,prop_val))})
+ prop_name = "hbase.coprocessor.master.classes"
+ prop_val = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor"
+ exclude_val = "org.apache.hadoop.hbase.security.access.AccessController"
+ if (prop_val in hbase_site[prop_name] and exclude_val not in hbase_site[prop_name]):
+ pass
+ else:
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "If Ranger HBase Plugin is enabled."\
+ " {0} needs to contain {1} instead of {2}".format(prop_name,prop_val,exclude_val))})
+ prop_name = "hbase.coprocessor.region.classes"
+ prop_val = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor"
+ if (prop_val in hbase_site[prop_name] and exclude_val not in hbase_site[prop_name]):
+ pass
+ else:
+ validationItems.append({"config-name": prop_name,
+ "item": self.getWarnItem(
+ "If Ranger HBase Plugin is enabled."\
+ " {0} needs to contain {1} instead of {2}".format(prop_name,prop_val,exclude_val))})
+
+ return self.toConfigurationValidationProblems(validationItems, "hbase-site")
+
+
def isComponentUsingCardinalityForLayout(self, componentName):
- return componentName == 'NFS_GATEWAY'
\ No newline at end of file
+ return componentName == 'NFS_GATEWAY'
[2/2] ambari git commit: AMBARI-11344. Categorize and appropriately
label/describe HiveServer2, Hive Metastore,
and Hive CLI heap size settings (alexantonenko)
Posted by al...@apache.org.
AMBARI-11344. Categorize and appropriately label/describe HiveServer2, Hive Metastore, and Hive CLI heap size settings (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6a572bb2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6a572bb2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6a572bb2
Branch: refs/heads/trunk
Commit: 6a572bb2053545800221bba2e8af0a2a6144066a
Parents: 52aaaa0
Author: Alex Antonenko <hi...@gmail.com>
Authored: Fri May 22 19:12:22 2015 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Fri May 22 21:55:16 2015 +0300
----------------------------------------------------------------------
.../app/controllers/wizard/step7_controller.js | 1 -
ambari-web/app/data/HDP2/site_properties.js | 28 ++++++++++++++++++--
ambari-web/app/models/stack_service.js | 6 +++--
3 files changed, 30 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a572bb2/ambari-web/app/controllers/wizard/step7_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/wizard/step7_controller.js b/ambari-web/app/controllers/wizard/step7_controller.js
index e1faf39..59bcc63 100644
--- a/ambari-web/app/controllers/wizard/step7_controller.js
+++ b/ambari-web/app/controllers/wizard/step7_controller.js
@@ -610,7 +610,6 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, App.E
App.config.addAdvancedConfigs(configs, advancedConfigs);
this.set('groupsToDelete', this.get('wizardController').getDBProperty('groupsToDelete') || []);
-
if (this.get('wizardController.name') === 'addServiceController') {
App.router.get('configurationController').getConfigsByTags(this.get('serviceConfigTags')).done(function (loadedConfigs) {
self.setInstalledServiceConfigs(self.get('serviceConfigTags'), configs, loadedConfigs, self.get('installedServiceNames'));
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a572bb2/ambari-web/app/data/HDP2/site_properties.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js
index 1ea26a5..1b36ac5 100644
--- a/ambari-web/app/data/HDP2/site_properties.js
+++ b/ambari-web/app/data/HDP2/site_properties.js
@@ -454,18 +454,42 @@ var hdp2properties = [
},
{
"id": "site property",
+ "name": "hive.metastore.heapsize", // the default value is overwritten in code
+ "displayName": "Hive Metastore heapsize",
+ "isOverridable": false,
+ "category": "HIVE_METASTORE",
+ "serviceName": "HIVE",
+ "filename": "hive-env.xml",
+ "displayType": "int",
+ "unit": "MB",
+ "index": 11
+ },
+ {
+ "id": "site property",
"name": "hive.heapsize",
- "displayName": "Hive heap size",
+ "displayName": "HiveServer2 heap size",
"displayType": "int",
"unit": "MB",
"isOverridable": false,
"serviceName": "HIVE",
"filename": "hive-site.xml",
- "category": "General",
+ "category": "HIVE_SERVER2",
"index": 9
},
{
"id": "site property",
+ "name": "hive.client.heapsize",
+ "displayName": "Hive Client heapsize",
+ "isOverridable": false,
+ "category": "HIVE_CLIENT",
+ "serviceName": "HIVE",
+ "filename": "hive-env.xml",
+ "displayType": "int",
+ "unit": "MB",
+ "index": 10
+ },
+ {
+ "id": "site property",
"name": "javax.jdo.option.ConnectionUserName",
"displayName": "Database Username",
"displayType": "user",
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a572bb2/ambari-web/app/models/stack_service.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/stack_service.js b/ambari-web/app/models/stack_service.js
index dcdb60b..e86a312 100644
--- a/ambari-web/app/models/stack_service.js
+++ b/ambari-web/app/models/stack_service.js
@@ -168,7 +168,7 @@ App.StackService = DS.Model.extend({
var configTypes = this.get('configTypes');
var serviceComponents = this.get('serviceComponents');
if (configTypes && Object.keys(configTypes).length) {
- var pattern = ["General", "CapacityScheduler", "FaultTolerance", "Isolation", "Performance", "KDC", "Kadmin","^Advanced", "Env$", "^Custom", "Falcon - Oozie integration", "FalconStartupSite", "FalconRuntimeSite", "MetricCollector", "Settings$"];
+ var pattern = ["General", "CapacityScheduler", "FaultTolerance", "Isolation", "Performance", "HIVE_SERVER2", "KDC", "Kadmin","^Advanced", "Env$", "^Custom", "Falcon - Oozie integration", "FalconStartupSite", "FalconRuntimeSite", "MetricCollector", "Settings$"];
configCategories = App.StackService.configCategories.call(this).filter(function (_configCategory) {
var serviceComponentName = _configCategory.get('name');
var isServiceComponent = serviceComponents.someProperty('componentName', serviceComponentName);
@@ -259,7 +259,9 @@ App.StackService.configCategories = function () {
App.ServiceConfigCategory.create({ name: 'HIVE_METASTORE', displayName: 'Hive Metastore'}),
App.ServiceConfigCategory.create({ name: 'WEBHCAT_SERVER', displayName: 'WebHCat Server'}),
App.ServiceConfigCategory.create({ name: 'General', displayName: 'General'}),
- App.ServiceConfigCategory.create({ name: 'Performance', displayName: 'Performance'})
+ App.ServiceConfigCategory.create({ name: 'Performance', displayName: 'Performance'}),
+ App.ServiceConfigCategory.create({ name: 'HIVE_SERVER2', displayName: 'Hive Server2'}),
+ App.ServiceConfigCategory.create({ name: 'HIVE_CLIENT', displayName: 'Hive Client'})
]);
break;
case 'HBASE':