You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/03/04 20:27:08 UTC
git commit: AMBARI-4943. Verify Tez configurations are according to
recommended settings. (srimanth)
Repository: ambari
Updated Branches:
refs/heads/trunk 20fa2346b -> 01e59101c
AMBARI-4943. Verify Tez configurations are according to recommended settings. (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/01e59101
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/01e59101
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/01e59101
Branch: refs/heads/trunk
Commit: 01e59101c2a70bc0eca94a5a5b2dfd1e31fa359d
Parents: 20fa234
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Tue Mar 4 11:16:07 2014 -0800
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Tue Mar 4 11:26:56 2014 -0800
----------------------------------------------------------------------
.../services/HIVE/configuration/hive-site.xml | 2 +-
.../services/TEZ/configuration/tez-site.xml | 2 +-
ambari-web/app/data/service_configs.js | 10 +++--
.../storm_defaults_provider.js | 2 +-
.../defaults_providers/tez_defaults_provider.js | 40 ++++++++++++++++++++
.../yarn_defaults_provider.js | 2 +-
.../validators/service_configs_validator.js | 2 +-
.../configs/validators/tez_configs_validator.js | 38 +++++++++++++++++++
8 files changed, 90 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
index f0be795..88c8e88 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
@@ -91,7 +91,7 @@ limitations under the License.
<property>
<name>hive.semantic.analyzer.factory.impl</name>
<value>org.apache.hivealog.cli.HCatSemanticAnalyzerFactory</value>
- <description>controls which SemanticAnalyzerFactory implemenation class is used by CLI</description>
+ <description>controls which SemanticAnalyzerFactory implementation class is used by CLI</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
index 333a8ac..5a750bf 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
@@ -47,7 +47,7 @@
<property>
<name>tez.am.java.opts</name>
<value>-server -Xmx1024m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC</value>
- <description>Java options for the Tez AppMaster process</description>
+ <description>Java options for the Tez AppMaster process. The -Xmx parameter value is generally 0.8 times tez.am.resource.memory.mb config.</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/data/service_configs.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/service_configs.js b/ambari-web/app/data/service_configs.js
index b3266fb..0473134 100644
--- a/ambari-web/app/data/service_configs.js
+++ b/ambari-web/app/data/service_configs.js
@@ -19,8 +19,10 @@
var App = require('app');
require('models/service_config');
require('utils/configs/defaults_providers/yarn_defaults_provider');
+require('utils/configs/defaults_providers/tez_defaults_provider');
require('utils/configs/defaults_providers/storm_defaults_provider');
require('utils/configs/validators/yarn_configs_validator');
+require('utils/configs/validators/tez_configs_validator');
require('utils/configs/validators/mapreduce2_configs_validator');
require('utils/configs/validators/storm_configs_validator');
@@ -74,7 +76,7 @@ module.exports = [
serviceName: 'YARN',
displayName: 'YARN',
configsValidator: App.YARNConfigsValidator,
- defaultsProviders: [App.YARNDefaultsProvider],
+ defaultsProviders: [App.YARNDefaultsProvider.create()],
filename: 'yarn-site',
configCategories: [
App.ServiceConfigCategory.create({ name: 'ResourceManager', displayName : 'Resource Manager', hostComponentNames : ['RESOURCEMANAGER']}),
@@ -95,7 +97,7 @@ module.exports = [
displayName: 'MapReduce 2',
filename: 'mapred-site',
configsValidator: App.MapReduce2ConfigsValidator,
- defaultsProviders: [App.YARNDefaultsProvider],
+ defaultsProviders: [App.YARNDefaultsProvider.create()],
configCategories: [
App.ServiceConfigCategory.create({ name: 'HistoryServer', displayName : 'History Server', hostComponentNames : ['HISTORYSERVER']}),
App.ServiceConfigCategory.create({ name: 'General', displayName : 'General'}),
@@ -240,7 +242,7 @@ module.exports = [
serviceName: 'STORM',
displayName: 'Storm',
configsValidator: App.STORMConfigsValidator,
- defaultsProviders: [App.STORMDefaultsProvider],
+ defaultsProviders: [App.STORMDefaultsProvider.create()],
filename: 'storm-site',
configCategories: [
App.ServiceConfigCategory.create({ name: 'Nimbus', displayName : 'Nimbus'}),
@@ -260,6 +262,8 @@ module.exports = [
serviceName: 'TEZ',
displayName: 'Tez',
filename: 'tez-site',
+ configsValidator: App.TezConfigsValidator,
+ defaultsProviders: [App.TezDefaultsProvider.create()],
configCategories: [
App.ServiceConfigCategory.create({ name: 'General', displayName : 'General'}),
App.ServiceConfigCategory.create({ name: 'Advanced', displayName : 'Advanced'}),
http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js b/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js
index cdd8ab9..ce8704d 100644
--- a/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js
+++ b/ambari-web/app/utils/configs/defaults_providers/storm_defaults_provider.js
@@ -18,7 +18,7 @@
var App = require('app');
require('utils/configs/defaults_providers/defaultsProvider');
-App.STORMDefaultsProvider = App.DefaultsProvider.create({
+App.STORMDefaultsProvider = App.DefaultsProvider.extend({
clusterData: null,
http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/defaults_providers/tez_defaults_provider.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/defaults_providers/tez_defaults_provider.js b/ambari-web/app/utils/configs/defaults_providers/tez_defaults_provider.js
new file mode 100644
index 0000000..18a3109
--- /dev/null
+++ b/ambari-web/app/utils/configs/defaults_providers/tez_defaults_provider.js
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/defaults_providers/yarn_defaults_provider');
+
+App.TezDefaultsProvider = App.YARNDefaultsProvider.extend({
+
+ configsTemplate: {
+ 'tez.am.resource.memory.mb': null,
+ 'tez.am.java.opts': null
+ },
+
+ getDefaults : function(localDB) {
+ var configs = this._super(localDB);
+ if (configs['yarn.app.mapreduce.am.resource.mb'] != null) {
+ configs['tez.am.resource.memory.mb'] = configs['yarn.app.mapreduce.am.resource.mb'];
+ configs['tez.am.java.opts'] = '-server -Xmx' + Math.round(0.8 * configs['tez.am.resource.memory.mb'])
+ + 'm -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC';
+ } else {
+ jQuery.extend(configs, this.get('configsTemplate'));
+ }
+ return configs;
+ }
+
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js b/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js
index 0e6e312..da395e0 100644
--- a/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js
+++ b/ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js
@@ -18,7 +18,7 @@
var App = require('app');
require('utils/configs/defaults_providers/defaultsProvider');
-App.YARNDefaultsProvider = App.DefaultsProvider.create({
+App.YARNDefaultsProvider = App.DefaultsProvider.extend({
/**
* List of the configs that should be calculated
http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/validators/service_configs_validator.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/validators/service_configs_validator.js b/ambari-web/app/utils/configs/validators/service_configs_validator.js
index d4dbd34..5507641 100644
--- a/ambari-web/app/utils/configs/validators/service_configs_validator.js
+++ b/ambari-web/app/utils/configs/validators/service_configs_validator.js
@@ -93,7 +93,7 @@ App.ServiceConfigsValidator = Em.Object.extend({
* @private
*/
_checkXmxValueFormat: function(value) {
- var regex = /^\-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)?(\s+)?(\s.+)?$/;
+ var regex = /(^|\s)\-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)(\s|$)/;
if (!regex.test(value)) {
return false;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/01e59101/ambari-web/app/utils/configs/validators/tez_configs_validator.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/validators/tez_configs_validator.js b/ambari-web/app/utils/configs/validators/tez_configs_validator.js
new file mode 100644
index 0000000..14042fb
--- /dev/null
+++ b/ambari-web/app/utils/configs/validators/tez_configs_validator.js
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/validators/service_configs_validator');
+
+App.TezConfigsValidator = App.ServiceConfigsValidator.create({
+ /**
+ * List of the configs that should be validated
+ */
+ configValidators: {
+ 'tez.am.resource.memory.mb': 'tezAMResourceMb',
+ 'tez.am.java.opts': 'tezAMJavaOpts',
+ },
+
+ tezAMResourceMb: function(config) {
+ return this.validatorLessThenDefaultValue(config);
+ },
+
+ tezAMJavaOpts: function(config) {
+ return this.validateXmxValue(config);
+ }
+
+});