You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2015/12/04 19:44:37 UTC
ambari git commit: AMBARI-14177. Multiple Ranger KMS server cause
exception with Blueprint provisioningn (Oliver Szabo via rlevas)
Repository: ambari
Updated Branches:
refs/heads/trunk 8b9d23ba9 -> cf976c922
AMBARI-14177. Multiple Ranger KMS server cause exception with Blueprint provisioningn (Oliver Szabo via rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cf976c92
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cf976c92
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cf976c92
Branch: refs/heads/trunk
Commit: cf976c922ac52d0273187a6cbb9e806a6491da7f
Parents: 8b9d23b
Author: Oliver Szabo <os...@hortonworks.com>
Authored: Fri Dec 4 13:44:24 2015 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Fri Dec 4 13:44:29 2015 -0500
----------------------------------------------------------------------
.../BlueprintConfigurationProcessor.java | 24 +++++++++-------
.../stacks/HDP/2.2/services/stack_advisor.py | 2 +-
.../BlueprintConfigurationProcessorTest.java | 29 ++++++++++++--------
.../stacks/2.2/common/test_stack_advisor.py | 8 +++---
4 files changed, 36 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cf976c92/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 462bddc..467889a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -1591,6 +1591,10 @@ public class BlueprintConfigurationProcessor {
return origValue;
}
+ if (origValue.contains("localhost") && topology.getHostGroupsForComponent(component).size() == 1) {
+ return origValue.replace("localhost", topology.getHostAssignmentsForComponent(component).iterator().next());
+ }
+
String prefix = null;
Collection<String> hostStrings = getHostStrings(origValue, topology);
if (hostStrings.isEmpty()) {
@@ -1631,12 +1635,12 @@ public class BlueprintConfigurationProcessor {
}
// parse out suffix if one exists
- int indexOfEnd = -1;
- while (matcher.find()) {
+ int indexOfEnd;
+ do {
indexOfEnd = matcher.end();
- }
+ } while (matcher.find());
- if ((indexOfEnd > -1) && (indexOfEnd < (origValue.length() - 1))) {
+ if (indexOfEnd < (origValue.length() - 1)) {
suffix = origValue.substring(indexOfEnd);
}
}
@@ -2039,9 +2043,9 @@ public class BlueprintConfigurationProcessor {
Map<String, PropertyUpdater> multiYarnSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> multiOozieSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> multiAccumuloSiteMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> multiRangerKmsSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> dbHiveSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> rangerAdminPropsMap = new HashMap<String, PropertyUpdater>();
- Map<String, PropertyUpdater> rangerKmsSitePropsMap = new HashMap<String, PropertyUpdater>();
@@ -2060,7 +2064,6 @@ public class BlueprintConfigurationProcessor {
singleHostTopologyUpdaters.put("kafka-broker", kafkaBrokerMap);
singleHostTopologyUpdaters.put("application-properties", atlasPropsMap);
singleHostTopologyUpdaters.put("admin-properties", rangerAdminPropsMap);
- singleHostTopologyUpdaters.put("kms-site", rangerKmsSitePropsMap);
mPropertyUpdaters.put("hadoop-env", hadoopEnvMap);
@@ -2079,6 +2082,7 @@ public class BlueprintConfigurationProcessor {
multiHostTopologyUpdaters.put("yarn-site", multiYarnSiteMap);
multiHostTopologyUpdaters.put("oozie-site", multiOozieSiteMap);
multiHostTopologyUpdaters.put("accumulo-site", multiAccumuloSiteMap);
+ multiHostTopologyUpdaters.put("kms-site", multiRangerKmsSiteMap);
dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap);
@@ -2096,13 +2100,12 @@ public class BlueprintConfigurationProcessor {
hdfsSiteMap.put("dfs.namenode.http-address", new SingleHostTopologyUpdater("NAMENODE"));
hdfsSiteMap.put("dfs.namenode.https-address", new SingleHostTopologyUpdater("NAMENODE"));
hdfsSiteMap.put("dfs.namenode.rpc-address", new SingleHostTopologyUpdater("NAMENODE"));
- hdfsSiteMap.put("dfs.encryption.key.provider.uri", new OptionalSingleHostTopologyUpdater("RANGER_KMS_SERVER"));
coreSiteMap.put("fs.defaultFS", new SingleHostTopologyUpdater("NAMENODE"));
- coreSiteMap.put("hadoop.security.key.provider.path", new OptionalSingleHostTopologyUpdater("RANGER_KMS_SERVER"));
hbaseSiteMap.put("hbase.rootdir", new SingleHostTopologyUpdater("NAMENODE"));
accumuloSiteMap.put("instance.volumes", new SingleHostTopologyUpdater("NAMENODE"));
// HDFS shared.edits JournalNode Quorum URL uses semi-colons as separators
multiHdfsSiteMap.put("dfs.namenode.shared.edits.dir", new MultipleHostTopologyUpdater("JOURNALNODE", ';', false));
+ multiHdfsSiteMap.put("dfs.encryption.key.provider.uri", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER", ';', false));
// SECONDARY_NAMENODE
hdfsSiteMap.put("dfs.secondary.http.address", new SingleHostTopologyUpdater("SECONDARY_NAMENODE"));
@@ -2141,6 +2144,7 @@ public class BlueprintConfigurationProcessor {
multiCoreSiteMap.put("hadoop.proxyuser.HTTP.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
multiCoreSiteMap.put("hadoop.proxyuser.hcat.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
multiCoreSiteMap.put("hadoop.proxyuser.yarn.hosts", new MultipleHostTopologyUpdater("RESOURCEMANAGER"));
+ multiCoreSiteMap.put("hadoop.security.key.provider.path", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER", ';', false));
multiWebhcatSiteMap.put("templeton.hive.properties", new TempletonHivePropertyUpdater());
multiWebhcatSiteMap.put("templeton.kerberos.principal", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
hiveEnvMap.put("hive_hostname", new SingleHostTopologyUpdater("HIVE_SERVER"));
@@ -2306,8 +2310,8 @@ public class BlueprintConfigurationProcessor {
rangerAdminPropsMap.put("policymgr_external_url", new SingleHostTopologyUpdater("RANGER_ADMIN"));
// RANGER KMS
- rangerKmsSitePropsMap.put("hadoop.kms.key.provider.uri", new SingleHostTopologyUpdater("RANGER_KMS_SERVER"));
-
+ multiRangerKmsSiteMap.put("hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string",
+ new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
// Required due to AMBARI-4933. These no longer seem to be required as the default values in the stack
// are now correct but are left here in case an existing blueprint still contains an old value.
hadoopEnvMap.put("namenode_heapsize", new MPropertyUpdater());
http://git-wip-us.apache.org/repos/asf/ambari/blob/cf976c92/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index a33be1e..12f85b8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -227,7 +227,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
rangerKMSServerHostsArray = []
for rangeKMSServerHost in rangerKMSServerHosts:
rangerKMSServerHostsArray.append(rangeKMSServerHost["Hosts"]["host_name"])
- keyserverHostsString = ",".join(rangerKMSServerHostsArray)
+ keyserverHostsString = ";".join(rangerKMSServerHostsArray)
if "kms-env" in services["configurations"] and "kms_port" in services["configurations"]["kms-env"]["properties"]:
keyserverPortString = services["configurations"]["kms-env"]["properties"]["kms_port"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/cf976c92/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 648dabc..b4b50cf 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -5721,8 +5721,9 @@ public class BlueprintConfigurationProcessorTest {
Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
Map<String, String> kmsSiteProperties = new HashMap<String, String>();
properties.put(kmsSiteConfigType, kmsSiteProperties);
- kmsSiteProperties.put("hadoop.kms.key.provider.uri", "dbks://http@%HOSTGROUP::group1%:9292/kms");
-
+ kmsSiteProperties.put("hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string",
+ createHostAddress("%HOSTGROUP::group1%", "2181") + "," + createHostAddress("%HOSTGROUP::group2%", "2181"));
+ kmsSiteProperties.put("hadoop.kms.key.provider.uri", "dbks://http@localhost:9292/kms");
Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
@@ -5736,9 +5737,11 @@ public class BlueprintConfigurationProcessorTest {
kmsServerComponents.add("RANGER_KMS_SERVER");
TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1"));
+ TestHostGroup group2 = new TestHostGroup("group2", kmsServerComponents, Collections.singleton("host2"));
-
- Collection<TestHostGroup> hostGroups = Collections.singleton(group1);
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
@@ -5747,7 +5750,8 @@ public class BlueprintConfigurationProcessorTest {
configProcessor.doUpdateForClusterCreate();
// Then
- assertEquals("dbks://http@host1:9292/kms", clusterConfig.getPropertyValue(kmsSiteConfigType, "hadoop.kms.key.provider.uri"));
+ assertEquals("host1:2181,host2:2181", clusterConfig.getPropertyValue(kmsSiteConfigType, "hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string"));
+ assertEquals("dbks://http@localhost:9292/kms", clusterConfig.getPropertyValue(kmsSiteConfigType, "hadoop.kms.key.provider.uri"));
}
@@ -5760,8 +5764,8 @@ public class BlueprintConfigurationProcessorTest {
Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
Map<String, String> kmsSiteProperties = new HashMap<String, String>();
properties.put(kmsSiteConfigType, kmsSiteProperties);
- kmsSiteProperties.put("hadoop.kms.key.provider.uri", "dbks://http@localhost:9292/kms");
-
+ kmsSiteProperties.put("hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string",
+ createHostAddress("%HOSTGROUP::group1%", "2181"));
Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
@@ -5786,7 +5790,8 @@ public class BlueprintConfigurationProcessorTest {
configProcessor.doUpdateForClusterCreate();
// Then
- assertEquals("dbks://http@host1:9292/kms", clusterConfig.getPropertyValue(kmsSiteConfigType, "hadoop.kms.key.provider.uri"));
+ assertEquals("host1:2181", clusterConfig.getPropertyValue(kmsSiteConfigType,
+ "hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string"));
}
@@ -5798,7 +5803,7 @@ public class BlueprintConfigurationProcessorTest {
Map<String, String> configProperties = new HashMap<String, String>();
properties.put(configType, configProperties);
- configProperties.put("dfs.encryption.key.provider.uri", "kms://http@%HOSTGROUP::group1%:9292/kms");
+ configProperties.put("dfs.encryption.key.provider.uri", "kms://http@%HOSTGROUP::group1%;%HOSTGROUP::group2%:9292/kms");
Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
@@ -5830,7 +5835,7 @@ public class BlueprintConfigurationProcessorTest {
configProcessor.doUpdateForClusterCreate();
// Then
- assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"));
+ assertEquals("kms://http@host1;host2:9292/kms", clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"));
}
@@ -5933,7 +5938,7 @@ public class BlueprintConfigurationProcessorTest {
Map<String, String> configProperties = new HashMap<String, String>();
properties.put(configType, configProperties);
- configProperties.put("hadoop.security.key.provider.path", "kms://http@%HOSTGROUP::group1%:9292/kms");
+ configProperties.put("hadoop.security.key.provider.path", "kms://http@%HOSTGROUP::group1%;%HOSTGROUP::group2%:9292/kms");
Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
@@ -5965,7 +5970,7 @@ public class BlueprintConfigurationProcessorTest {
configProcessor.doUpdateForClusterCreate();
// Then
- assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType, "hadoop.security.key.provider.path"));
+ assertEquals("kms://http@host1;host2:9292/kms", clusterConfig.getPropertyValue(configType, "hadoop.security.key.provider.path"));
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cf976c92/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index f5c0436..1ee3e7a 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -3013,22 +3013,22 @@ class TestHDP22StackAdvisor(TestCase):
# Test 6 - Multiple RANGER_KMS_SERVERs
services["services"][len(services["services"])-1]["components"][0]["StackServiceComponents"]["hostnames"].append("host2")
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
- self.assertEqual("kms://http@host1,host2:9292/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+ self.assertEqual("kms://http@host1;host2:9292/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
# Test 6 - Multiple RANGER_KMS_SERVERs and custom port
configurations["kms-env"] = {"properties": {"kms_port": "1111"}}
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
- self.assertEqual("kms://http@host1,host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+ self.assertEqual("kms://http@host1;host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
# Test 7 - Override by API caller
configurations["hadoop-env"] = {"properties": {"keyserver_host": "myhost1", "keyserver_port": "2222"}}
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
- self.assertEqual("kms://http@host1,host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+ self.assertEqual("kms://http@host1;host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
# Test - 'https' in KMS URL
configurations["ranger-kms-site"] = {"properties": {"ranger.service.https.attrib.ssl.enabled": "true"}}
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
- self.assertEqual("kms://https@host1,host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
+ self.assertEqual("kms://https@host1;host2:1111/kms", configurations["hdfs-site"]["properties"]["dfs.encryption.key.provider.uri"])
# Test 8 - Dynamic maximum for 'dfs.namenode.handler.count'
hosts['items'][1]['Hosts']['cpu_count'] = 9