You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rn...@apache.org on 2016/02/04 20:07:13 UTC
ambari git commit: AMBARI-14786. Localhost configuration value for
multiple host properties does not work properly. (Daniel Gergely via
rnettleton)
Repository: ambari
Updated Branches:
refs/heads/trunk 8ff0b5ed8 -> 7616e0f5d
AMBARI-14786. Localhost configuration value for multiple host properties does not work properly. (Daniel Gergely via rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7616e0f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7616e0f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7616e0f5
Branch: refs/heads/trunk
Commit: 7616e0f5d6939509d1914857dc03150eacc9e3b9
Parents: 8ff0b5e
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Thu Feb 4 14:06:34 2016 -0500
Committer: Bob Nettleton <rn...@hortonworks.com>
Committed: Thu Feb 4 14:07:02 2016 -0500
----------------------------------------------------------------------
.../BlueprintConfigurationProcessor.java | 206 ++++++++++------
.../BlueprintConfigurationProcessorTest.java | 243 ++++++++++++++++++-
2 files changed, 370 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7616e0f5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 9f42901..de31a0d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -126,6 +126,11 @@ public class BlueprintConfigurationProcessor {
private static Pattern HOSTGROUP_PORT_REGEX = Pattern.compile("%HOSTGROUP::(\\S+?)%:?(\\d+)?");
/**
+ * Compiled regex for hostgroup token with port information.
+ */
+ private static Pattern LOCALHOST_PORT_REGEX = Pattern.compile("localhost:?(\\d+)?");
+
+ /**
* Statically-defined set of properties that can support HA using a nameservice name
* in the configuration, rather than just a host name.
* This set also contains other HA properties that will be exported if the
@@ -1583,7 +1588,9 @@ public class BlueprintConfigurationProcessor {
*/
private final boolean usePrefixForEachHost;
- private final Set<String> setOfKnownURLSchemes = Collections.singleton("thrift://");
+ private final boolean useSuffixForEachHost;
+
+ private final boolean usePortForEachHost;
/**
* Constructor.
@@ -1591,7 +1598,7 @@ public class BlueprintConfigurationProcessor {
* @param component component name associated with the property
*/
public MultipleHostTopologyUpdater(String component) {
- this(component, DEFAULT_SEPARATOR, false);
+ this(component, DEFAULT_SEPARATOR, false, false, true);
}
/**
@@ -1601,10 +1608,12 @@ public class BlueprintConfigurationProcessor {
* @param separator the separator character to use when multiple hosts
* are specified in a property or URL
*/
- public MultipleHostTopologyUpdater(String component, Character separator, boolean userPrefixForEachHost) {
+ public MultipleHostTopologyUpdater(String component, Character separator, boolean usePrefixForEachHost, boolean useSuffixForEachHost, boolean usePortForEachHost) {
this.component = component;
this.separator = separator;
- this.usePrefixForEachHost = userPrefixForEachHost;
+ this.usePrefixForEachHost = usePrefixForEachHost;
+ this.useSuffixForEachHost = useSuffixForEachHost;
+ this.usePortForEachHost = usePortForEachHost;
}
/**
@@ -1625,36 +1634,101 @@ public class BlueprintConfigurationProcessor {
StringBuilder sb = new StringBuilder();
- if (!origValue.contains("%HOSTGROUP") &&
- (!origValue.contains("localhost"))) {
+ if (!origValue.contains("%HOSTGROUP") && (!origValue.contains("localhost"))) {
// this property must contain FQDNs specified directly by the user
// of the Blueprint, so the processor should not attempt to update them
return origValue;
}
- if (origValue.contains("localhost") && topology.getHostGroupsForComponent(component).size() == 1) {
- return origValue.replace("localhost", topology.getHostAssignmentsForComponent(component).iterator().next());
+ Collection<String> hostStrings = getHostStrings(origValue, topology);
+ hostStrings.addAll(getHostStringsFromLocalhost(origValue, topology));
+
+ return resolveHostGroupPlaceholder(origValue, hostStrings);
+ }
+
+ /**
+ * Gets the prefix for hosts
+ * @param value property value
+ * @return prefix
+ */
+ private String getPrefix(String value) {
+ Matcher localhostMatcher = LOCALHOST_PORT_REGEX.matcher(value);
+ Matcher hostGroupMatcher = HOSTGROUP_PORT_REGEX.matcher(value);
+ String prefixCandidate = null;
+
+ if(localhostMatcher.find()) {
+ prefixCandidate = value.substring(0,localhostMatcher.start());
+ } else if(hostGroupMatcher.find()) {
+ prefixCandidate = value.substring(0,hostGroupMatcher.start());
+ } else {
+ return prefixCandidate;
+ }
+
+ // remove YAML array notation
+ if(prefixCandidate.startsWith("[")) {
+ prefixCandidate = prefixCandidate.substring(1);
+ }
+ // remove YAML string notation
+ if(prefixCandidate.startsWith("'")) {
+ prefixCandidate = prefixCandidate.substring(1);
}
- String prefix = null;
- Collection<String> hostStrings = getHostStrings(origValue, topology);
- if (hostStrings.isEmpty()) {
- //default non-exported original value
- String port;
- for (String urlScheme : setOfKnownURLSchemes) {
- if (origValue.startsWith(urlScheme)) {
- prefix = urlScheme;
- }
- }
+ return prefixCandidate;
+ }
- if (prefix != null) {
- String valueWithoutPrefix = origValue.substring(prefix.length());
- port = calculatePort(valueWithoutPrefix);
- sb.append(prefix);
- } else {
- port = calculatePort(origValue);
- }
+ /**
+ * Gets the suffix for hosts
+ * @param value property value
+ * @return suffix
+ */
+ private String getSuffix(String value) {
+ Matcher localhostMatcher = LOCALHOST_PORT_REGEX.matcher(value);
+ Matcher hostGroupMatcher = HOSTGROUP_PORT_REGEX.matcher(value);
+
+
+ Matcher activeMatcher = null;
+ if(localhostMatcher.find()) {
+ activeMatcher = localhostMatcher;
+ } else if(hostGroupMatcher.find()) {
+ activeMatcher = hostGroupMatcher;
+ } else {
+ return null;
+ }
+
+ String suffixCandidate = null;
+ int indexOfEnd;
+ do {
+ indexOfEnd = activeMatcher.end();
+ } while (activeMatcher.find());
+ suffixCandidate = value.substring(indexOfEnd);
+
+ // remove YAML array notation
+ if(suffixCandidate.endsWith("]")) {
+ suffixCandidate = suffixCandidate.substring(0, suffixCandidate.length()-1);
+ }
+ // remove YAML string notation
+ if(suffixCandidate.endsWith("'")) {
+ suffixCandidate = suffixCandidate.substring(0, suffixCandidate.length()-1);
+ }
+
+ return suffixCandidate;
+ }
+
+ /**
+ * Resolves localhost value to "host:port" elements (port is optional)
+ * @param origValue property value
+ * @param topology cluster topology
+ * @return list of hosts that have the given components
+ */
+ private Collection<String> getHostStringsFromLocalhost(String origValue, ClusterTopology topology) {
+ Set<String> hostStrings = new HashSet<String>();
+ if(origValue.contains("localhost")) {
+ Matcher localhostMatcher = LOCALHOST_PORT_REGEX.matcher(origValue);
+ String port = null;
+ if(localhostMatcher.find()) {
+ port = calculatePort(localhostMatcher.group());
+ }
for (String host : topology.getHostAssignmentsForComponent(component)) {
if (port != null) {
host += ":" + port;
@@ -1662,62 +1736,52 @@ public class BlueprintConfigurationProcessor {
hostStrings.add(host);
}
}
-
- return sb.append(resolveHostGroupPlaceholder(origValue, prefix, hostStrings)).toString();
+ return hostStrings;
}
/**
* Resolves the host group place holders in the passed in original value.
* @param originalValue The original value containing the place holders to be resolved.
- * @param prefix The prefix to be added to the returned value.
* @param hostStrings The collection of host names that are mapped to the host groups to be resolved
* @return The new value with place holders resolved.
*/
- protected String resolveHostGroupPlaceholder(String originalValue, String prefix, Collection<String> hostStrings) {
- String suffix = null;
- StringBuilder sb = new StringBuilder();
+ protected String resolveHostGroupPlaceholder(String originalValue, Collection<String> hostStrings) {
+ String prefix = getPrefix(originalValue);
+ String suffix = getSuffix(originalValue);
+ String port = removePorts(hostStrings);
- // parse out prefix if one exists
- Matcher matcher = HOSTGROUP_PORT_REGEX.matcher(originalValue);
- if (matcher.find()) {
- int indexOfStart = matcher.start();
- // handle the case of a YAML config property
- if ((indexOfStart > 0) && (!originalValue.substring(0, indexOfStart).equals("['")) && (!originalValue.substring(0, indexOfStart).equals("[")) ) {
- // append prefix before adding host names
- prefix = originalValue.substring(0, indexOfStart);
- sb.append(prefix);
- }
+ String sep = (useSuffixForEachHost ? suffix : "") + separator + (usePrefixForEachHost ? prefix : "");
+ String combinedHosts = (usePrefixForEachHost ? prefix : "") + StringUtils.join(hostStrings, sep);
- // parse out suffix if one exists
- int indexOfEnd;
- do {
- indexOfEnd = matcher.end();
- } while (matcher.find());
+ return (usePrefixForEachHost ? "" : prefix) + combinedHosts + (usePortForEachHost || port == null ? "" : ":" + port) + suffix;
+ }
- if (indexOfEnd < (originalValue.length())) {
- suffix = originalValue.substring(indexOfEnd);
- }
- }
+ /**
+ * Removes "port" part of the hosts and returns it
+ * @param hostStrings list of "host:port" strings (port is optional)
+ * @return the port
+ */
+ private String removePorts(Collection<String> hostStrings) {
+ String port = null;
+ if(!usePortForEachHost && !hostStrings.isEmpty()) {
+ Set<String> temp = new HashSet<String>();
- // add hosts to property, using the specified separator
- boolean firstHost = true;
- for (String host : hostStrings) {
- if (!firstHost) {
- sb.append(separator);
- // support config properties that use a list of full URIs
- if (usePrefixForEachHost && (prefix != null)) {
- sb.append(prefix);
+ // extract port
+ Iterator<String> i = hostStrings.iterator();
+ do {
+ port = calculatePort(i.next());
+ } while (i.hasNext() && port == null);
+
+ // update hosts
+ if(port != null) {
+ for(String host : hostStrings) {
+ temp.add(host.replace(":"+port,""));
}
- } else {
- firstHost = false;
}
- sb.append(host);
+ hostStrings.clear();
+ hostStrings.addAll(temp);
}
-
- if ((suffix != null) && (!suffix.equals("']")) && (!suffix.equals("]")) ) {
- sb.append(suffix);
- }
- return sb.toString();
+ return port;
}
private static String calculatePort(String origValue) {
@@ -1988,7 +2052,7 @@ public class BlueprintConfigurationProcessor {
TempletonHivePropertyUpdater() {
// the only known property that requires hostname substitution is hive.metastore.uris,
// but this updater should be flexible enough for other properties in the future.
- mapOfKeysToUpdaters.put("hive.metastore.uris", new MultipleHostTopologyUpdater("HIVE_METASTORE", ',', true));
+ mapOfKeysToUpdaters.put("hive.metastore.uris", new MultipleHostTopologyUpdater("HIVE_METASTORE", ',', true, false, true));
}
@Override
@@ -2180,8 +2244,8 @@ public class BlueprintConfigurationProcessor {
hbaseSiteMap.put("hbase.rootdir", new SingleHostTopologyUpdater("NAMENODE"));
accumuloSiteMap.put("instance.volumes", new SingleHostTopologyUpdater("NAMENODE"));
// HDFS shared.edits JournalNode Quorum URL uses semi-colons as separators
- multiHdfsSiteMap.put("dfs.namenode.shared.edits.dir", new MultipleHostTopologyUpdater("JOURNALNODE", ';', false));
- multiHdfsSiteMap.put("dfs.encryption.key.provider.uri", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER", ';', false));
+ multiHdfsSiteMap.put("dfs.namenode.shared.edits.dir", new MultipleHostTopologyUpdater("JOURNALNODE", ';', false, false, true));
+ multiHdfsSiteMap.put("dfs.encryption.key.provider.uri", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER", ';', false, false, false));
// SECONDARY_NAMENODE
hdfsSiteMap.put("dfs.secondary.http.address", new SingleHostTopologyUpdater("SECONDARY_NAMENODE"));
@@ -2213,14 +2277,14 @@ public class BlueprintConfigurationProcessor {
yarnSiteMap.put("yarn.timeline-service.webapp.https.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER"));
// HIVE_SERVER
- multiHiveSiteMap.put("hive.metastore.uris", new MultipleHostTopologyUpdater("HIVE_METASTORE", ',', true));
+ multiHiveSiteMap.put("hive.metastore.uris", new MultipleHostTopologyUpdater("HIVE_METASTORE", ',', true, true, true));
dbHiveSiteMap.put("javax.jdo.option.ConnectionURL",
new DBTopologyUpdater("MYSQL_SERVER", "hive-env", "hive_database"));
multiCoreSiteMap.put("hadoop.proxyuser.hive.hosts", new MultipleHostTopologyUpdater("HIVE_SERVER"));
multiCoreSiteMap.put("hadoop.proxyuser.HTTP.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
multiCoreSiteMap.put("hadoop.proxyuser.hcat.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
multiCoreSiteMap.put("hadoop.proxyuser.yarn.hosts", new MultipleHostTopologyUpdater("RESOURCEMANAGER"));
- multiCoreSiteMap.put("hadoop.security.key.provider.path", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER", ';', false));
+ multiCoreSiteMap.put("hadoop.security.key.provider.path", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER", ';', false, false, true));
multiWebhcatSiteMap.put("templeton.hive.properties", new TempletonHivePropertyUpdater());
multiWebhcatSiteMap.put("templeton.kerberos.principal", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
hiveEnvMap.put("hive_hostname", new SingleHostTopologyUpdater("HIVE_SERVER"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/7616e0f5/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 1afd6a8..7a77a25 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -41,6 +41,7 @@ import org.apache.ambari.server.topology.HostGroupInfo;
import org.apache.ambari.server.topology.InvalidTopologyException;
import org.apache.ambari.server.utils.CollectionPresentationUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -2354,6 +2355,116 @@ public class BlueprintConfigurationProcessorTest {
}
@Test
+ public void testMultipleHostTopologyUpdater__localhost__singleHost() throws Exception {
+
+ final String typeName = "hbase-site";
+ final String propertyName = "hbase.zookeeper.quorum";
+ final String originalValue = "localhost";
+ final String component1 = "ZOOKEEPER_SERVER";
+ final String component2 = "ZOOKEEPER_CLIENT";
+
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProps = new HashMap<String, String>();
+ typeProps.put(propertyName, originalValue);
+ properties.put(typeName, typeProps);
+
+ Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Collection<String> hgComponents = new HashSet<String>();
+ hgComponents.add(component1);
+ Set<String> hosts1 = new HashSet<String>();
+ hosts1.add("testhost1a");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents, hosts1);
+
+ Collection<String> hgComponents2 = new HashSet<String>();
+ hgComponents2.add(component2);
+ Set<String> hosts2 = new HashSet<String>();
+ hosts2.add("testhost2");
+ TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2);
+
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+
+ BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1);
+ String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology);
+
+ assertEquals("testhost1a", newValue);
+ }
+
+ @Test
+ public void testMultipleHostTopologyUpdater__localhost__singleHostGroup() throws Exception {
+
+ final String typeName = "hbase-site";
+ final String propertyName = "hbase.zookeeper.quorum";
+ final String originalValue = "localhost";
+ final String component1 = "ZOOKEEPER_SERVER";
+
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProps = new HashMap<String, String>();
+ typeProps.put(propertyName, originalValue);
+ properties.put(typeName, typeProps);
+
+ Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Collection<String> hgComponents = new HashSet<String>();
+ hgComponents.add(component1);
+ Set<String> hosts1 = new HashSet<String>();
+ hosts1.add("testhost1a");
+ hosts1.add("testhost1b");
+ hosts1.add("testhost1c");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents, hosts1);
+
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+
+ BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1);
+ String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology);
+
+ List<String> hostArray = Arrays.asList(newValue.split(","));
+ Assert.assertTrue(hostArray.containsAll(hosts1) && hosts1.containsAll(hostArray));
+ }
+
+ @Test
+ public void testMultipleHostTopologyUpdater__hostgroup__singleHostGroup() throws Exception {
+
+ final String typeName = "hbase-site";
+ final String propertyName = "hbase.zookeeper.quorum";
+ final String originalValue = "%HOSTGROUP::group1%";
+ final String component1 = "ZOOKEEPER_SERVER";
+
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProps = new HashMap<String, String>();
+ typeProps.put(propertyName, originalValue);
+ properties.put(typeName, typeProps);
+
+ Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Collection<String> hgComponents = new HashSet<String>();
+ hgComponents.add(component1);
+ Set<String> hosts1 = new HashSet<String>();
+ hosts1.add("testhost1a");
+ hosts1.add("testhost1b");
+ hosts1.add("testhost1c");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents, hosts1);
+
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+
+ BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1);
+ String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology);
+
+ List<String> hostArray = Arrays.asList(newValue.split(","));
+ Assert.assertTrue(hostArray.containsAll(hosts1) && hosts1.containsAll(hostArray));
+ }
+
+ @Test
public void testDoUpdateForClusterVerifyRetrySettingsDefault() throws Exception {
Map<String, Map<String, String>> configProperties =
new HashMap<String, Map<String, String>>();
@@ -5887,8 +5998,16 @@ public class BlueprintConfigurationProcessorTest {
// When
configProcessor.doUpdateForClusterCreate();
+ String updatedVal = clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri");
+ Assert.assertTrue(updatedVal.startsWith("kms://http@"));
+ Assert.assertTrue(updatedVal.endsWith(":9292/kms"));
+ String hostsString = updatedVal.substring(11,updatedVal.length()-9);
+
+ List<String> hostArray = Arrays.asList(hostsString.split(";"));
+ List<String> expected = Arrays.asList("host1","host2");
+
// Then
- assertEquals("kms://http@host1;host2:9292/kms", clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"));
+ Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray));
}
@@ -5982,6 +6101,114 @@ public class BlueprintConfigurationProcessorTest {
assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"));
}
+ @Test
+ public void testHdfsWithRangerKmsServer__multiple_hosts__localhost() throws Exception {
+ // Given
+ final String configType = "hdfs-site";
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> configProperties = new HashMap<String, String>();
+
+ properties.put(configType, configProperties);
+ configProperties.put("dfs.encryption.key.provider.uri", "kms://http@localhost:9292/kms");
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> kmsServerComponents = new HashSet<String>();
+ kmsServerComponents.add("RANGER_KMS_SERVER");
+
+ Collection<String> hdfsComponents = new HashSet<String>();
+ hdfsComponents.add("NAMENODE");
+ hdfsComponents.add("DATANODE");
+
+ Collection<String> hosts = new HashSet<String>();
+ hosts.add("host1");
+ hosts.add("host2");
+
+ TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, hosts);
+ group1.components.add("DATANODE");
+
+ TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host3"));
+
+ Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ String updatedVal = clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri");
+ Assert.assertTrue(updatedVal.startsWith("kms://http@"));
+ Assert.assertTrue(updatedVal.endsWith(":9292/kms"));
+ String hostsString = updatedVal.substring(11,updatedVal.length()-9);
+
+ List<String> hostArray = Arrays.asList(hostsString.split(";"));
+ List<String> expected = Arrays.asList("host1","host2");
+
+ // Then
+ Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray));
+ }
+
+ @Test
+ public void testHdfsWithRangerKmsServer__multiple_hosts__hostgroup() throws Exception {
+ // Given
+ final String configType = "hdfs-site";
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> configProperties = new HashMap<String, String>();
+
+ properties.put(configType, configProperties);
+ configProperties.put("dfs.encryption.key.provider.uri", "kms://http@%HOSTGROUP::group1%:9292/kms");
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> kmsServerComponents = new HashSet<String>();
+ kmsServerComponents.add("RANGER_KMS_SERVER");
+
+ Collection<String> hdfsComponents = new HashSet<String>();
+ hdfsComponents.add("NAMENODE");
+ hdfsComponents.add("DATANODE");
+
+ Collection<String> hosts = new HashSet<String>();
+ hosts.add("host1");
+ hosts.add("host2");
+
+ TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, hosts);
+ group1.components.add("DATANODE");
+
+ TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host3"));
+
+ Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ String updatedVal = clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri");
+ Assert.assertTrue(updatedVal.startsWith("kms://http@"));
+ Assert.assertTrue(updatedVal.endsWith(":9292/kms"));
+ String hostsString = updatedVal.substring(11,updatedVal.length()-9);
+
+ List<String> hostArray = Arrays.asList(hostsString.split(";"));
+ List<String> expected = Arrays.asList("host1","host2");
+
+ // Then
+ Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray));
+ }
+
@Test
public void testHadoopWithRangerKmsServer() throws Exception {
@@ -6248,8 +6475,8 @@ public class BlueprintConfigurationProcessorTest {
String propertyOriginalValue2 = "[%HOSTGROUP::group_1%]";
// When
- String updatedValue1 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue1, null, ImmutableList.<String>of("host1:100"));
- String updatedValue2 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue2, null, ImmutableList.<String>of("host1:100"));
+ String updatedValue1 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue1, ImmutableList.<String>of("host1:100"));
+ String updatedValue2 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue2, ImmutableList.<String>of("host1:100"));
// Then
assertEquals("host1:100", updatedValue1);
@@ -6269,8 +6496,8 @@ public class BlueprintConfigurationProcessorTest {
String propertyOriginalValue2 = "[%HOSTGROUP::group_1%, %HOSTGROUP::group_2%]";
// When
- String updatedValue1 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue1, null, ImmutableList.<String>of("host1:100", "host2:200"));
- String updatedValue2 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue2, null, ImmutableList.<String>of("host1:100", "host2:200"));
+ String updatedValue1 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue1, ImmutableList.<String>of("host1:100", "host2:200"));
+ String updatedValue2 = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue2, ImmutableList.<String>of("host1:100", "host2:200"));
// Then
assertEquals("host1:100,host2:200", updatedValue1);
@@ -6288,7 +6515,7 @@ public class BlueprintConfigurationProcessorTest {
String propertyOriginalValue = "http://%HOSTGROUP::group_1%#";
// When
- String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, null, ImmutableList.<String>of("host1:100"));
+ String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, ImmutableList.<String>of("host1:100"));
// Then
assertEquals("http://host1:100#", updatedValue);
@@ -6303,7 +6530,7 @@ public class BlueprintConfigurationProcessorTest {
String propertyOriginalValue = "http://%HOSTGROUP::group_1,HOSTGROUP::group_2%/resource";
// When
- String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, null, ImmutableList.<String>of("host1:100", "host2:200"));
+ String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, ImmutableList.<String>of("host1:100", "host2:200"));
// Then
assertEquals("http://host1:100,host2:200/resource", updatedValue);
@@ -6318,7 +6545,7 @@ public class BlueprintConfigurationProcessorTest {
String propertyOriginalValue = "%HOSTGROUP::group_1%:11,%HOSTGROUP::group_2%:11";
// When
- String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, null, ImmutableList.<String>of("host1:100", "host2:200"));
+ String updatedValue = mhtu.resolveHostGroupPlaceholder(propertyOriginalValue, ImmutableList.<String>of("host1:100", "host2:200"));
// Then
assertEquals("host1:100,host2:200", updatedValue);