You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/01/05 13:38:47 UTC
[14/15] ambari git commit: AMBARI-19383 Configs changes after
adding/deleting host component: value shouldn't be changed if it differs in
host names order only. (ababiichuk)
AMBARI-19383 Configs changes after adding/deleting host component: value shouldn't be changed if it differs in host names order only. (ababiichuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1527bd15
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1527bd15
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1527bd15
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 1527bd15c6226fbb8e690c64f68b593ac5088c2b
Parents: 6b198cc
Author: ababiichuk <ab...@hortonworks.com>
Authored: Thu Jan 5 13:57:21 2017 +0200
Committer: ababiichuk <ab...@hortonworks.com>
Committed: Thu Jan 5 14:34:46 2017 +0200
----------------------------------------------------------------------
ambari-web/app/controllers/main/host/details.js | 94 ++++++------
ambari-web/app/utils/config.js | 42 +++++-
.../configs/add_component_config_initializer.js | 5 +-
.../configs/hosts_based_initializer_mixin.js | 2 +-
.../test/controllers/main/host/details_test.js | 12 +-
ambari-web/test/utils/config_test.js | 147 ++++++++++++++++++-
6 files changed, 244 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/controllers/main/host/details.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/host/details.js b/ambari-web/app/controllers/main/host/details.js
index 4a7cce4..7df2c4e 100644
--- a/ambari-web/app/controllers/main/host/details.js
+++ b/ambari-web/app/controllers/main/host/details.js
@@ -923,10 +923,10 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow
fileName: fileName,
name: propertyName,
value: currentValue
- };
- var configProperty = initializer.initialValue(propertyDef, hostComponentsTopology, dependencies);
+ },
+ configProperty = initializer.initialValue(propertyDef, hostComponentsTopology, dependencies);
initializer.updateSiteObj(configs[fileName], configProperty);
- if (this.get('isReconfigureRequired') && currentValue !== propertyDef.value) {
+ if (this.get('isReconfigureRequired') && currentValue !== configs[fileName][propertyName]) {
var service = App.config.get('serviceByConfigTypeMap')[fileName];
propertiesToChange.pushObject({
propertyFileName: fileName,
@@ -1101,9 +1101,10 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow
fileName: fileName,
name: propertyName,
value: currentValue
- };
- configs[fileName][propertyName] = Em.get(initializer.initialValue(propertyDef, localDB, dependencies), 'value');
- if (this.get('isReconfigureRequired') && propertyDef.value !== currentValue) {
+ },
+ configProperty = initializer.initialValue(propertyDef, localDB, dependencies);
+ initializer.updateSiteObj(configs[fileName], configProperty);
+ if (this.get('isReconfigureRequired') && currentValue !== configs[fileName][propertyName]) {
var service = App.config.get('serviceByConfigTypeMap')[fileName];
propertiesToChange.pushObject({
propertyFileName: fileName,
@@ -1286,50 +1287,55 @@ App.MainHostDetailsController = Em.Controller.extend(App.SupportClientConfigsDow
*/
onLoadRangerConfigs: function (data) {
var properties = [
- {
- type: 'core-site',
- name: 'hadoop.security.key.provider.path'
- },
- {
- type: 'hdfs-site',
- name: 'dfs.encryption.key.provider.uri'
- }
- ];
- var hostToInstall = this.get('rangerKMSServerHost');
- var rkmsHosts = this.getRangerKMSServerHosts();
- var rkmsPort = data.items.findProperty('type', 'kms-env').properties['kms_port'];
- var newValue = 'kms://http@' + rkmsHosts.join(';') + ':' + rkmsPort + '/kms';
- var coreSiteConfigs = data.items.findProperty('type', 'core-site');
- var hdfsSiteConfigs = data.items.findProperty('type', 'hdfs-site');
- var groups = [
- {
- properties: {
- 'core-site': coreSiteConfigs.properties,
- 'hdfs-site': hdfsSiteConfigs.properties
+ {
+ type: 'core-site',
+ name: 'hadoop.security.key.provider.path'
},
- properties_attributes: {
- 'core-site': coreSiteConfigs.properties_attributes,
- 'hdfs-site': hdfsSiteConfigs.properties_attributes
+ {
+ type: 'hdfs-site',
+ name: 'dfs.encryption.key.provider.uri'
}
- }
- ];
- var propertiesToChange = this.get('allPropertiesToChange');
+ ],
+ hostToInstall = this.get('rangerKMSServerHost'),
+ rkmsHosts = this.getRangerKMSServerHosts().join(';'),
+ rkmsPort = data.items.findProperty('type', 'kms-env').properties['kms_port'],
+ newValue = 'kms://http@' + rkmsHosts + ':' + rkmsPort + '/kms',
+ coreSiteConfigs = data.items.findProperty('type', 'core-site'),
+ hdfsSiteConfigs = data.items.findProperty('type', 'hdfs-site'),
+ groups = [
+ {
+ properties: {
+ 'core-site': coreSiteConfigs.properties,
+ 'hdfs-site': hdfsSiteConfigs.properties
+ },
+ properties_attributes: {
+ 'core-site': coreSiteConfigs.properties_attributes,
+ 'hdfs-site': hdfsSiteConfigs.properties_attributes
+ }
+ }
+ ],
+ propertiesToChange = this.get('allPropertiesToChange');
properties.forEach(function (property) {
var typeConfigs = data.items.findProperty('type', property.type).properties,
- currentValue = typeConfigs[property.name];
- if (this.get('isReconfigureRequired') && currentValue !== newValue) {
- var service = App.config.get('serviceByConfigTypeMap')[property.type];
- propertiesToChange.pushObject({
- propertyFileName: property.type,
- propertyName: property.name,
- serviceDisplayName: service && service.get('displayName'),
- initialValue: currentValue,
- recommendedValue: newValue,
- saveRecommended: true
- });
+ currentValue = typeConfigs[property.name],
+ pattern = new RegExp('^kms:\\/\\/http@(.+):' + rkmsPort + '\\/kms$'),
+ patternMatch = currentValue && currentValue.match(pattern),
+ currentHostsList = patternMatch && patternMatch[1].split(';').sort().join(';');
+ if (currentHostsList !== rkmsHosts) {
+ typeConfigs[property.name] = newValue;
+ if (this.get('isReconfigureRequired')) {
+ var service = App.config.get('serviceByConfigTypeMap')[property.type];
+ propertiesToChange.pushObject({
+ propertyFileName: property.type,
+ propertyName: property.name,
+ serviceDisplayName: service && service.get('displayName'),
+ initialValue: currentValue,
+ recommendedValue: newValue,
+ saveRecommended: true
+ });
+ }
}
- typeConfigs[property.name] = newValue;
}, this);
if (this.get('isReconfigureRequired')) {
this.setConfigsChanges(groups);
http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/utils/config.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/config.js b/ambari-web/app/utils/config.js
index d0dbb69..8eeca5f 100644
--- a/ambari-web/app/utils/config.js
+++ b/ambari-web/app/utils/config.js
@@ -37,6 +37,17 @@ App.config = Em.Object.create({
preDefinedServiceConfigs: [],
/**
+ * Map for methods used to parse hosts lists from certain config properties
+ */
+ uniqueHostsListParsers: [
+ {
+ propertyName: 'templeton.hive.properties',
+ type: 'webhcat-site',
+ method: 'getTempletonHiveHosts'
+ }
+ ],
+
+ /**
*
* Returns file name version that stored on server.
*
@@ -1234,6 +1245,12 @@ App.config = Em.Object.create({
return false;
},
+ getTempletonHiveHosts: function (value) {
+ var pattern = /thrift:\/\/.+:\d+/,
+ patternMatch = value.match(pattern);
+ return patternMatch ? patternMatch[0].split('\\,') : value;
+ },
+
/**
* Update config property value based on its current value and list of zookeeper server hosts.
* Used to prevent sort order issues.
@@ -1251,14 +1268,31 @@ App.config = Em.Object.create({
*
* @method updateHostsListValue
* @param {Object} siteConfigs - prepared site config object to store
+ * @param {String} propertyType - type of the property to update
* @param {String} propertyName - name of the property to update
* @param {String} hostsList - list of ZooKeeper Server names to set as config property value
+ * @param {Boolean} isArray - determines whether value string is formatted as array
* @return {String} - result value
*/
- updateHostsListValue: function(siteConfigs, propertyName, hostsList) {
- var value = hostsList;
- var propertyHosts = (siteConfigs[propertyName] || '').split(',');
- var hostsToSet = hostsList.split(',');
+ updateHostsListValue: function(siteConfigs, propertyType, propertyName, hostsList, isArray) {
+ var value = hostsList,
+ propertyHosts = (siteConfigs[propertyName] || ''),
+ hostsToSet = hostsList,
+ parser = this.get('uniqueHostsListParsers').find(function (property) {
+ return property.type === propertyType && property.propertyName === propertyName;
+ });
+ if (parser) {
+ propertyHosts = this.get(parser.method)(propertyHosts);
+ hostsToSet = this.get(parser.method)(hostsToSet);
+ } else {
+ if (isArray) {
+ var pattern = /(^\[|]$)/g;
+ propertyHosts = propertyHosts.replace(pattern, '');
+ hostsToSet = hostsToSet.replace(pattern, '');
+ }
+ propertyHosts = propertyHosts.split(',');
+ hostsToSet = hostsToSet.split(',');
+ }
if (!Em.isEmpty(siteConfigs[propertyName])) {
var diffLength = propertyHosts.filter(function(hostName) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/utils/configs/add_component_config_initializer.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/add_component_config_initializer.js b/ambari-web/app/utils/configs/add_component_config_initializer.js
index 7e3fedf..21fb6b4 100644
--- a/ambari-web/app/utils/configs/add_component_config_initializer.js
+++ b/ambari-web/app/utils/configs/add_component_config_initializer.js
@@ -146,7 +146,10 @@ App.AddComponentConfigInitializer = App.HaConfigInitializerClass.extend(App.Host
*/
updateSiteObj: function(siteConfigs, configProperty) {
if (!siteConfigs || !configProperty) return false;
- App.config.updateHostsListValue(siteConfigs, configProperty.name, configProperty.value);
+ var initializer = this.get('initializers')[configProperty.name],
+ isArray = !!(initializer && (initializer.type === 'json_stringified_value'
+ || Em.isArray(initializer) && initializer.someProperty('type', 'json_stringified_value')));
+ App.config.updateHostsListValue(siteConfigs, configProperty.fileName, configProperty.name, configProperty.value, isArray);
return true;
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js b/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js
index d0d2503..086ad4a 100644
--- a/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js
+++ b/ambari-web/app/utils/configs/hosts_based_initializer_mixin.js
@@ -371,7 +371,7 @@ App.HostsBasedInitializerMixin = Em.Mixin.create({
component: component,
componentExists: componentExists,
modifier: {
- delmitier: Em.isNone(delimiter) ? ',' : delimiter
+ delimiter: Em.isNone(delimiter) ? ',' : delimiter
}
};
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/test/controllers/main/host/details_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/main/host/details_test.js b/ambari-web/test/controllers/main/host/details_test.js
index a4cb177..03dadc6 100644
--- a/ambari-web/test/controllers/main/host/details_test.js
+++ b/ambari-web/test/controllers/main/host/details_test.js
@@ -3247,7 +3247,7 @@ describe('App.MainHostDetailsController', function () {
'result': [
{
properties: {
- 'core-site': {'hadoop.security.key.provider.path': 'kms://http@host1;host2:port/kms'},
+ 'core-site': {'hadoop.security.key.provider.path': 'kms://http@host2;host1:port/kms'},
'hdfs-site': {'dfs.encryption.key.provider.uri': 'kms://http@host1;host2:port/kms'}
},
properties_attributes: {
@@ -3278,11 +3278,15 @@ describe('App.MainHostDetailsController', function () {
},
{
type: 'core-site',
- properties: {}
+ properties: {
+ 'hadoop.security.key.provider.path': 'kms://http@host2;host1:port/kms'
+ }
},
{
type: 'hdfs-site',
- properties: {}
+ properties: {
+ 'dfs.encryption.key.provider.uri': 'kms://http@host2:port/kms'
+ }
}
]
};
@@ -3290,7 +3294,7 @@ describe('App.MainHostDetailsController', function () {
beforeEach(function () {
controller.set('rangerKMSServerHost', item.hostToInstall);
sinon.stub(controller, 'getRangerKMSServerHosts').returns(item.kmsHosts);
- controller.onLoadRangerConfigs(data, null, {});
+ controller.onLoadRangerConfigs(data);
});
it('saveConfigsBatch is called with valid arguments', function () {
http://git-wip-us.apache.org/repos/asf/ambari/blob/1527bd15/ambari-web/test/utils/config_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/utils/config_test.js b/ambari-web/test/utils/config_test.js
index 8c450ed..7c2515a 100644
--- a/ambari-web/test/utils/config_test.js
+++ b/ambari-web/test/utils/config_test.js
@@ -1060,6 +1060,7 @@ describe('App.config', function() {
'hadoop.registry.zk.quorum': 'host1,host2'
},
propertyName: 'hadoop.registry.zk.quorum',
+ propertyType: 'yarn-site',
hostsList: 'host1',
e: 'host1'
},
@@ -1068,6 +1069,7 @@ describe('App.config', function() {
'hadoop.registry.zk.quorum': 'host1:10,host2:10'
},
propertyName: 'hadoop.registry.zk.quorum',
+ propertyType: 'yarn-site',
hostsList: 'host2:10,host1:10',
e: 'host1:10,host2:10'
},
@@ -1076,6 +1078,7 @@ describe('App.config', function() {
'hadoop.registry.zk.quorum': 'host1:10,host2:10,host3:10'
},
propertyName: 'hadoop.registry.zk.quorum',
+ propertyType: 'yarn-site',
hostsList: 'host2:10,host1:10',
e: 'host2:10,host1:10'
},
@@ -1084,6 +1087,7 @@ describe('App.config', function() {
'hadoop.registry.zk.quorum': 'host1:10,host2:10,host3:10'
},
propertyName: 'hadoop.registry.zk.quorum',
+ propertyType: 'yarn-site',
hostsList: 'host2:10,host1:10,host3:10,host4:11',
e: 'host2:10,host1:10,host3:10,host4:11'
},
@@ -1092,15 +1096,114 @@ describe('App.config', function() {
'hive.zookeeper.quorum': 'host1'
},
propertyName: 'some.new.property',
+ propertyType: 'hive-site',
hostsList: 'host2,host1:10',
e: 'host2,host1:10'
+ },
+ {
+ siteConfigs: {
+ 'some.new.property': '[\'host1\',\'host2\']'
+ },
+ propertyName: 'some.new.property',
+ propertyType: 'property-type',
+ hostsList: '[\'host1\',\'host2\']',
+ isArray: true,
+ e: '[\'host1\',\'host2\']',
+ message: 'array-formatted property value with no changes'
+ },
+ {
+ siteConfigs: {
+ 'some.new.property': '[\'host2\',\'host1\']'
+ },
+ propertyName: 'some.new.property',
+ propertyType: 'property-type',
+ hostsList: '[\'host1\',\'host2\']',
+ isArray: true,
+ e: '[\'host2\',\'host1\']',
+ message: 'array-formatted property value with different hosts order'
+ },
+ {
+ siteConfigs: {
+ 'some.new.property': '[\'host1\',\'host2\']'
+ },
+ propertyName: 'some.new.property',
+ propertyType: 'property-type',
+ hostsList: '[\'host3\',\'host4\']',
+ isArray: true,
+ e: '[\'host3\',\'host4\']',
+ message: 'array-formatted property value with changes'
+ },
+ {
+ siteConfigs: {
+ 'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true'
+ },
+ propertyName: 'templeton.hive.properties',
+ propertyType: 'webhcat-site',
+ hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ message: 'templeton.hive.properties, no changes'
+ },
+ {
+ siteConfigs: {
+ 'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host2:9083\\,thrift://host1:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true'
+ },
+ propertyName: 'templeton.hive.properties',
+ propertyType: 'webhcat-site',
+ hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host2:9083\\,thrift://host1:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ message: 'templeton.hive.properties, different hosts order'
+ },
+ {
+ siteConfigs: {
+ 'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9082\\,thrift://host2:9082,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true'
+ },
+ propertyName: 'templeton.hive.properties',
+ propertyType: 'webhcat-site',
+ hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ message: 'templeton.hive.properties, different ports'
+ },
+ {
+ siteConfigs: {
+ 'templeton.hive.properties': 'hive.metastore.local=false,hive.metastore.uris=thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true'
+ },
+ propertyName: 'templeton.hive.properties',
+ propertyType: 'webhcat-site',
+ hostsList: 'hive.metastore.local=false,hive.metastore.uris=thrift://host3:9083\\,thrift://host4:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ e: 'hive.metastore.local=false,hive.metastore.uris=thrift://host3:9083\\,thrift://host4:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ message: 'templeton.hive.properties, different hosts'
+ },
+ {
+ siteConfigs: {
+ 'templeton.hive.properties': 'hive.metastore.local=false'
+ },
+ propertyName: 'templeton.hive.properties',
+ propertyType: 'hive-site',
+ hostsList: 'hive.metastore.local=true',
+ e: 'hive.metastore.local=true',
+ message: 'custom templeton.hive.properties'
}
];
+
tests.forEach(function(test) {
- it('ZK located on {0}, current prop value is "{1}" "{2}" value should be "{3}"'.format(test.hostsList, ''+test.siteConfigs[test.propertyName], test.propertyName, test.e), function() {
- var result = App.config.updateHostsListValue(test.siteConfigs, test.propertyName, test.hostsList);
- expect(result).to.be.eql(test.e);
- expect(test.siteConfigs[test.propertyName]).to.be.eql(test.e);
+ var message = test.message
+ || 'ZK located on {0}, current prop value is "{1}" "{2}" value should be "{3}"'
+ .format(test.hostsList, ''+test.siteConfigs[test.propertyName], test.propertyName, test.e);
+
+ describe(message, function () {
+ var result;
+
+ beforeEach(function () {
+ result = App.config.updateHostsListValue(test.siteConfigs, test.propertyType, test.propertyName, test.hostsList, test.isArray);
+ });
+
+ it('returned value', function() {
+ expect(result).to.be.eql(test.e);
+ });
+
+ it('value in configs object', function() {
+ expect(test.siteConfigs[test.propertyName]).to.be.eql(test.e);
+ });
});
});
});
@@ -1312,6 +1415,42 @@ describe('App.config', function() {
});
});
+ describe('#getTempletonHiveHosts', function () {
+ var testCases = [
+ {
+ value: 'hive.metastore.local=false,hive.metastore.uris=thrift://host0:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ result: ['thrift://host0:9083'],
+ message: 'one host'
+ },
+ {
+ value: 'hive.metastore.local=false,hive.metastore.uris=thrift://host0:9083\\,thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ result: ['thrift://host0:9083', 'thrift://host1:9083', 'thrift://host2:9083'],
+ message: 'several hosts'
+ },
+ {
+ value: 'thrift://host0:9083\\,thrift://host1:9083\\,thrift://host2:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true',
+ result: ['thrift://host0:9083', 'thrift://host1:9083', 'thrift://host2:9083'],
+ message: 'no leading text'
+ },
+ {
+ value: 'hive.metastore.local=false,hive.metastore.uris=thrift://host0:9083\\,thrift://host1:9083\\,thrift://host2:9083',
+ result: ['thrift://host0:9083', 'thrift://host1:9083', 'thrift://host2:9083'],
+ message: 'no trailing text'
+ },
+ {
+ value: 'hive.metastore.local=false',
+ result: 'hive.metastore.local=false',
+ message: 'no hosts list'
+ }
+ ];
+
+ testCases.forEach(function (test) {
+ it(test.message, function () {
+ expect(App.config.getTempletonHiveHosts(test.value)).to.eql(test.result);
+ });
+ });
+ });
+
describe('#isDirHeterogeneous', function () {
it ('retruns true for dfs.datanode.data.dir', function () {
expect(App.config.isDirHeterogeneous('dfs.datanode.data.dir')).to.be.true;