You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/09/03 17:21:30 UTC
[1/2] ambari git commit: AMBARI-13000. Disable Ranger Hive Plugin is
not reverting all the required properties (alexantonenko)
Repository: ambari
Updated Branches:
refs/heads/trunk c3a81ca00 -> 2771d477f
AMBARI-13000. Disable Ranger Hive Plugin is not reverting all the required properties (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2771d477
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2771d477
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2771d477
Branch: refs/heads/trunk
Commit: 2771d477fd2f2dca3f7fac350758857af7a59bd0
Parents: 2ae7e8f
Author: Alex Antonenko <hi...@gmail.com>
Authored: Thu Sep 3 18:16:12 2015 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Thu Sep 3 18:21:24 2015 +0300
----------------------------------------------------------------------
ambari-web/app/controllers/wizard.js | 5 +++--
.../app/mixins/common/configs/enhanced_configs.js | 12 ++++++++++++
.../common/configs/service_configs_by_category_view.js | 13 ++++++++++++-
ambari-web/test/controllers/wizard_test.js | 2 ++
4 files changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2771d477/ambari-web/app/controllers/wizard.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/wizard.js b/ambari-web/app/controllers/wizard.js
index 5c4ef30..04a4dc2 100644
--- a/ambari-web/app/controllers/wizard.js
+++ b/ambari-web/app/controllers/wizard.js
@@ -43,7 +43,8 @@ App.WizardController = Em.Controller.extend(App.LocalStorage, App.ThemesMappingM
'allHostNames',
'installOptions',
'allHostNamesPattern',
- 'serviceComponents'
+ 'serviceComponents',
+ 'fileNamesToUpdate'
],
sensibleConfigs: [
@@ -883,7 +884,7 @@ App.WizardController = Em.Controller.extend(App.LocalStorage, App.ThemesMappingM
*/
saveServiceConfigProperties: function (stepController) {
var serviceConfigProperties = [];
- var fileNamesToUpdate = [];
+ var fileNamesToUpdate = this.getDBProperty('fileNamesToUpdate') || [];
var installedServiceNames = stepController.get('installedServiceNames') || [];
var installedServiceNamesMap = {};
var notAllowed = ['masterHost', 'masterHosts', 'slaveHosts', 'slaveHost'];
http://git-wip-us.apache.org/repos/asf/ambari/blob/2771d477/ambari-web/app/mixins/common/configs/enhanced_configs.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/configs/enhanced_configs.js b/ambari-web/app/mixins/common/configs/enhanced_configs.js
index 18f6da6..c7fc3a7 100644
--- a/ambari-web/app/mixins/common/configs/enhanced_configs.js
+++ b/ambari-web/app/mixins/common/configs/enhanced_configs.js
@@ -562,6 +562,9 @@ App.EnhancedConfigsMixin = Em.Mixin.create({
Em.keys(attributes).forEach(function (attributeName) {
if (attributeName == 'delete') {
if (!updateOnlyBoundaries) {
+ var fileName = App.config.getOriginalFileName(siteName);
+ var modifiedFileNames = self.get('modifiedFileNames');
+ var wizardController = self.get('wizardController');
var dependentProperty = self.get('_dependentConfigValues').filterProperty('propertyName', propertyName).filterProperty('fileName', siteName).findProperty('configGroup', group && Em.get(group,'name'));
if (dependentProperty) {
Em.set(dependentProperty, 'toDelete', true);
@@ -585,6 +588,15 @@ App.EnhancedConfigsMixin = Em.Mixin.create({
recommendedValue: null
});
}
+ if (modifiedFileNames && !modifiedFileNames.contains(fileName)) {
+ modifiedFileNames.push(fileName);
+ } else if (wizardController) {
+ var fileNamesToUpdate = wizardController.getDBProperty('fileNamesToUpdate') || [];
+ if (!fileNamesToUpdate.contains(fileName)) {
+ fileNamesToUpdate.push(fileName);
+ wizardController.setDBProperty('fileNamesToUpdate', fileNamesToUpdate);
+ }
+ }
}
} else if (stackProperty) {
if (selectedConfigGroup) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/2771d477/ambari-web/app/views/common/configs/service_configs_by_category_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/configs/service_configs_by_category_view.js b/ambari-web/app/views/common/configs/service_configs_by_category_view.js
index c564359..db82636 100644
--- a/ambari-web/app/views/common/configs/service_configs_by_category_view.js
+++ b/ambari-web/app/views/common/configs/service_configs_by_category_view.js
@@ -580,7 +580,18 @@ App.ServiceConfigsByCategoryView = Em.View.extend(App.UserPref, App.ConfigOverri
}
// push config's file name if this config was stored on server
if (!serviceConfigProperty.get('isNotSaved')) {
- this.get('controller').get('modifiedFileNames').push(serviceConfigProperty.get('filename'));
+ var modifiedFileNames = this.get('controller.modifiedFileNames'),
+ wizardController = this.get('controller.wizardController'),
+ filename = serviceConfigProperty.get('filename');
+ if (modifiedFileNames && !modifiedFileNames.contains(filename)) {
+ modifiedFileNames.push(serviceConfigProperty.get('filename'));
+ } else if (wizardController) {
+ var fileNamesToUpdate = wizardController.getDBProperty('fileNamesToUpdate') || [];
+ if (!fileNamesToUpdate.contains(filename)) {
+ fileNamesToUpdate.push(filename);
+ wizardController.setDBProperty('fileNamesToUpdate', fileNamesToUpdate);
+ }
+ }
}
Em.$('body>.tooltip').remove(); //some tooltips get frozen when their owner's DOM element is removed
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/2771d477/ambari-web/test/controllers/wizard_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/wizard_test.js b/ambari-web/test/controllers/wizard_test.js
index b0750e1..f88101a 100644
--- a/ambari-web/test/controllers/wizard_test.js
+++ b/ambari-web/test/controllers/wizard_test.js
@@ -914,11 +914,13 @@ describe('App.WizardController', function () {
beforeEach(function () {
c.set('content', {});
sinon.stub(c, 'setDBProperties', Em.K);
+ sinon.stub(c, 'getDBProperty').withArgs('fileNamesToUpdate').returns([]);
sinon.stub(App.config, 'shouldSupportFinal').returns(true);
});
afterEach(function () {
c.setDBProperties.restore();
+ c.getDBProperty.restore();
App.config.shouldSupportFinal.restore();
});
[2/2] ambari git commit: AMBARI-12998. Add HiveServer2, Metastore,
and Client Heap Sizes to Smart Configs (alexantonenko)
Posted by al...@apache.org.
AMBARI-12998. Add HiveServer2, Metastore, and Client Heap Sizes to Smart Configs (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2ae7e8f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2ae7e8f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2ae7e8f5
Branch: refs/heads/trunk
Commit: 2ae7e8f505bf75071af555d9b05222a4f616db5d
Parents: c3a81ca
Author: Alex Antonenko <hi...@gmail.com>
Authored: Thu Sep 3 18:09:28 2015 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Thu Sep 3 18:21:24 2015 +0300
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/configuration/hive-env.xml | 16 ++++
.../services/HIVE/configuration/hive-env.xml | 10 ++-
.../HDP/2.2/services/HIVE/themes/theme.json | 45 ++++++++++
.../stacks/HDP/2.2/services/stack_advisor.py | 20 ++++-
.../stacks/2.2/common/test_stack_advisor.py | 87 ++++++++++++++++++++
5 files changed, 176 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ae7e8f5/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
index 784f06b..2d0a182 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
@@ -25,12 +25,28 @@
<name>hive.client.heapsize</name>
<value>512</value>
<description>Hive Client Java heap size</description>
+ <display-name>Client Heap Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <maximum>2048</maximum>
+ <unit>MB</unit>
+ <increment-step>512</increment-step>
+ </value-attributes>
</property>
<property>
<name>hive.metastore.heapsize</name>
<value>1024</value>
<description>Hive Metastore Java heap size</description>
+ <display-name>Metastore Heap Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <maximum>2048</maximum>
+ <unit>MB</unit>
+ <increment-step>512</increment-step>
+ </value-attributes>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ae7e8f5/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
index ec0381a..443648f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
@@ -23,8 +23,16 @@
<configuration>
<property>
<name>hive.heapsize</name>
- <value>1024</value>
+ <value>512</value>
<description>Hive Java heap size</description>
+ <display-name>HiveServer2 Heap Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <maximum>2048</maximum>
+ <unit>MB</unit>
+ <increment-step>512</increment-step>
+ </value-attributes>
</property>
<!-- hive-env.sh -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ae7e8f5/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
index 1a3b9fc..42bc889 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
@@ -149,6 +149,18 @@
},
{
"config": "hive-env/hive_exec_orc_storage_strategy",
+ "subsection-name": "optimization-row2-col1"
+ },
+ {
+ "config": "hive-env/hive.heapsize",
+ "subsection-name": "optimization-row2-col2"
+ },
+ {
+ "config": "hive-env/hive.metastore.heapsize",
+ "subsection-name": "optimization-row2-col2"
+ },
+ {
+ "config": "hive-env/hive.client.heapsize",
"subsection-name": "optimization-row2-col2"
},
{
@@ -250,6 +262,39 @@
}
},
{
+ "config": "hive-env/hive.heapsize",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "hive-env/hive.metastore.heapsize",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "hive-env/hive.client.heapsize",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ },
+ {
"config": "hive-site/hive.auto.convert.join.noconditionaltask.size",
"widget": {
"type": "slider",
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ae7e8f5/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 529df83..9a7d7a0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -242,7 +242,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveEnvProperty = self.putProperty(configurations, "hive-env", services)
putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site")
-
+ putHiveEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hive-env")
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
# Storage
@@ -475,6 +475,24 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
("hive-site" in services["configurations"] and "hive.server2.custom.authentication.class" in services["configurations"]["hive-site"]["properties"]):
putHiveSitePropertyAttribute("hive.server2.custom.authentication.class", "delete", "true")
+ # HiveServer, Client, Metastore heapsize
+ hs_heapsize_multiplier = 3.0/8
+ hm_heapsize_multiplier = 1.0/8
+ # HiveServer2 and HiveMetastore located on the same host
+ hive_server_hosts = self.getHostsWithComponent("HIVE", "HIVE_SERVER", services, hosts)
+ hive_client_hosts = self.getHostsWithComponent("HIVE", "HIVE_CLIENT", services, hosts)
+
+ if hive_server_hosts is not None and len(hive_server_hosts):
+ hs_host_ram = hive_server_hosts[0]["Hosts"]["total_mem"]/1024
+ putHiveEnvProperty("hive.metastore.heapsize", max(512, int(hs_host_ram*hm_heapsize_multiplier)))
+ putHiveEnvProperty("hive.heapsize", max(512, int(hs_host_ram*hs_heapsize_multiplier)))
+ putHiveEnvPropertyAttributes("hive.metastore.heapsize", "maximum", max(1024, hs_host_ram))
+ putHiveEnvPropertyAttributes("hive.heapsize", "maximum", max(1024, hs_host_ram))
+
+ if hive_client_hosts is not None and len(hive_client_hosts):
+ putHiveEnvProperty("hive.client.heapsize", 1024)
+ putHiveEnvPropertyAttributes("hive.client.heapsize", "maximum", max(1024, int(hive_client_hosts[0]["Hosts"]["total_mem"]/1024)))
+
def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP22StackAdvisor, self).recommendHbaseConfigurations(configurations, clusterData, services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ae7e8f5/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 851c8b5..d0260f9 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1149,6 +1149,80 @@ class TestHDP22StackAdvisor(TestCase):
"changed-configurations": [ ]
}
+ hiveService = {
+ "services": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/HIVE",
+ "StackServices": {
+ "service_name": "HIVE",
+ "service_version": "2.6.0.2.2",
+ "stack_name": "HDP",
+ "stack_version": "2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1",
+ "component_category": "MASTER",
+ "component_name": "HIVE_SERVER",
+ "display_name": "HiveServer2",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": [
+ "c6402.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "HIVE_CLIENT",
+ "display_name": "Hive Client",
+ "is_client": "true",
+ "is_master": "false",
+ "hostnames": [
+ "c6402.ambari.apache.org",
+ "c6403.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ }
+ ]
+ },
+ ],
+ "configurations": {
+ "hive-env": {
+ "properties": {
+ "hive.heapsize": "200",
+ "hive.metastore.heapsize": "200",
+ "hive.client.heapsize": "200"
+ }
+ },
+ "hive-site": {
+ "properties": {
+ "hive.server2.authentication": "none",
+ "hive.server2.authentication.ldap.url": "",
+ "hive.server2.authentication.ldap.baseDN": "",
+ "hive.server2.authentication.kerberos.keytab": "",
+ "hive.server2.authentication.kerberos.principal": "",
+ "hive.server2.authentication.pam.services": "",
+ "hive.server2.custom.authentication.class": ""
+ }
+ },
+ "hiveserver2-site": {
+ "properties": {
+ "hive.security.authorization.manager": "",
+ "hive.security.authenticator.manager": ""
+ }
+ }
+ },
+ "changed-configurations": [ ]
+ }
+
hosts = {
"items" : [
{
@@ -1286,6 +1360,19 @@ class TestHDP22StackAdvisor(TestCase):
self.assertEquals(configurations['hive-site']['property_attributes']['hive.server2.tez.default.queues'], expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'])
self.assertEquals(configurations['hive-site']['properties']['hive.server2.tez.default.queues'], expected['hive-site']['properties']['hive.server2.tez.default.queues'])
+ # Hive heapsize properties
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, hiveService, hosts)
+
+ # Recommended default values
+ self.assertEquals(configurations["hive-env"]["properties"]["hive.metastore.heapsize"], "512")
+ self.assertEquals(configurations["hive-env"]["properties"]["hive.heapsize"], "703")
+ self.assertEquals(configurations["hive-env"]["properties"]["hive.client.heapsize"], "1024")
+
+ # Recommended attributes for maximum values, minimum values defined in stack definition
+ self.assertEquals(configurations["hive-env"]["property_attributes"]["hive.heapsize"]["maximum"], "1877")
+ self.assertEquals(configurations["hive-env"]["property_attributes"]["hive.metastore.heapsize"]["maximum"], "1877")
+ self.assertEquals(configurations["hive-env"]["property_attributes"]["hive.client.heapsize"]["maximum"], "1877")
+
def test_recommendMapredConfigurationAttributesWithPigService(self):
configurations = {
"mapred-site": {