You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rn...@apache.org on 2015/11/11 22:58:15 UTC
ambari git commit: AMBARI-13815. Blueprints Not able to specify
HOST_GROUP syntax for ranger configs. (Sebastian Toader via rnettleton)
Repository: ambari
Updated Branches:
refs/heads/trunk 1c9307f9f -> 34db83ae9
AMBARI-13815. Blueprints Not able to specify HOST_GROUP syntax for ranger configs. (Sebastian Toader via rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/34db83ae
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/34db83ae
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/34db83ae
Branch: refs/heads/trunk
Commit: 34db83ae90c187a477bccd604daf6ec2b21e3b5c
Parents: 1c9307f
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Wed Nov 11 16:57:50 2015 -0500
Committer: Bob Nettleton <rn...@hortonworks.com>
Committed: Wed Nov 11 16:58:08 2015 -0500
----------------------------------------------------------------------
.../BlueprintConfigurationProcessor.java | 32 ++
.../BlueprintConfigurationProcessorTest.java | 501 ++++++++++++++++++-
2 files changed, 513 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/34db83ae/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 0ed2eef..9e2bf85 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -21,6 +21,7 @@ package org.apache.ambari.server.controller.internal;
import com.google.common.base.Predicates;
import com.google.common.collect.Maps;
+import org.apache.ambari.server.Role;
import org.apache.ambari.server.state.PropertyDependencyInfo;
import org.apache.ambari.server.state.ValueAttributesInfo;
import org.apache.ambari.server.topology.AdvisedConfiguration;
@@ -1254,6 +1255,13 @@ public class BlueprintConfigurationProcessor {
}
}
+ if (isRangerAdmin() && matchingGroupCount > 1) {
+ if (origValue != null && !origValue.contains("localhost")) {
+ // if this Ranger admin property is a FQDN then simply return it
+ return origValue;
+ }
+ }
+
throw new IllegalArgumentException(
String.format("Unable to update configuration property '%s' with topology information. " +
"Component '%s' is mapped to an invalid number of hosts '%s'.", propertyName, component, matchingGroupCount));
@@ -1361,6 +1369,17 @@ public class BlueprintConfigurationProcessor {
}
/**
+ * Utility method to determine if the component associated with this updater
+ * instance is Ranger Admin
+ *
+ * @return true if the component associated is Ranger Admin
+ * false if the component is not Ranger Admin
+ */
+ private boolean isRangerAdmin() {
+ return component.equals("RANGER_ADMIN");
+ }
+
+ /**
* Provides access to the name of the component associated
* with this updater instance.
*
@@ -2019,6 +2038,9 @@ public class BlueprintConfigurationProcessor {
Map<String, PropertyUpdater> multiOozieSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> multiAccumuloSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> dbHiveSiteMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> rangerAdminPropsMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> rangerKmsSitePropsMap = new HashMap<String, PropertyUpdater>();
+
singleHostTopologyUpdaters.put("hdfs-site", hdfsSiteMap);
@@ -2035,6 +2057,9 @@ public class BlueprintConfigurationProcessor {
singleHostTopologyUpdaters.put("oozie-env", oozieEnvMap);
singleHostTopologyUpdaters.put("kafka-broker", kafkaBrokerMap);
singleHostTopologyUpdaters.put("application-properties", atlasPropsMap);
+ singleHostTopologyUpdaters.put("admin-properties", rangerAdminPropsMap);
+ singleHostTopologyUpdaters.put("kms-site", rangerKmsSitePropsMap);
+
mPropertyUpdaters.put("hadoop-env", hadoopEnvMap);
mPropertyUpdaters.put("hbase-env", hbaseEnvMap);
@@ -2069,7 +2094,9 @@ public class BlueprintConfigurationProcessor {
hdfsSiteMap.put("dfs.namenode.http-address", new SingleHostTopologyUpdater("NAMENODE"));
hdfsSiteMap.put("dfs.namenode.https-address", new SingleHostTopologyUpdater("NAMENODE"));
hdfsSiteMap.put("dfs.namenode.rpc-address", new SingleHostTopologyUpdater("NAMENODE"));
+ hdfsSiteMap.put("dfs.encryption.key.provider.uri", new OptionalSingleHostTopologyUpdater("RANGER_KMS_SERVER"));
coreSiteMap.put("fs.defaultFS", new SingleHostTopologyUpdater("NAMENODE"));
+ coreSiteMap.put("hadoop.security.key.provider.path", new OptionalSingleHostTopologyUpdater("RANGER_KMS_SERVER"));
hbaseSiteMap.put("hbase.rootdir", new SingleHostTopologyUpdater("NAMENODE"));
accumuloSiteMap.put("instance.volumes", new SingleHostTopologyUpdater("NAMENODE"));
// HDFS shared.edits JournalNode Quorum URL uses semi-colons as separators
@@ -2236,6 +2263,11 @@ public class BlueprintConfigurationProcessor {
// ATLAS
atlasPropsMap.put("atlas.server.bind.address", new SingleHostTopologyUpdater("ATLAS_SERVER"));
+ // RANGER_ADMIN
+ rangerAdminPropsMap.put("policymgr_external_url", new SingleHostTopologyUpdater("RANGER_ADMIN"));
+
+ // RANGER KMS
+ rangerKmsSitePropsMap.put("hadoop.kms.key.provider.uri", new SingleHostTopologyUpdater("RANGER_KMS_SERVER"));
// Required due to AMBARI-4933. These no longer seem to be required as the default values in the stack
// are now correct but are left here in case an existing blueprint still contains an old value.
http://git-wip-us.apache.org/repos/asf/ambari/blob/34db83ae/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 424baee..1e75dbf 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.controller.internal;
+import com.google.common.collect.Lists;
import org.apache.ambari.server.state.PropertyDependencyInfo;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.ValueAttributesInfo;
@@ -671,19 +672,19 @@ public class BlueprintConfigurationProcessorTest {
assertEquals("Exported properties map was not of the expected size", 1,
- properties.get("custom-test-properties").size());
+ properties.get("custom-test-properties").size());
assertEquals("ranger-yarn-plugin-properties config type was not properly exported", 1,
- properties.get("ranger-yarn-plugin-properties").size());
+ properties.get("ranger-yarn-plugin-properties").size());
// verify that the following password properties matching the "*_PASSWORD" rule have been excluded
assertFalse("Password property should have been excluded",
- properties.get("ranger-yarn-plugin-properties").containsKey("REPOSITORY_CONFIG_PASSWORD"));
+ properties.get("ranger-yarn-plugin-properties").containsKey("REPOSITORY_CONFIG_PASSWORD"));
assertFalse("Password property should have been excluded",
properties.get("ranger-yarn-plugin-properties").containsKey("SSL_KEYSTORE_PASSWORD"));
assertFalse("Password property should have been excluded",
- properties.get("ranger-yarn-plugin-properties").containsKey("SSL_TRUSTSTORE_PASSWORD"));
+ properties.get("ranger-yarn-plugin-properties").containsKey("SSL_TRUSTSTORE_PASSWORD"));
assertFalse("Password property should have been excluded",
- properties.get("ranger-yarn-plugin-properties").containsKey("XAAUDIT.DB.PASSWORD"));
+ properties.get("ranger-yarn-plugin-properties").containsKey("XAAUDIT.DB.PASSWORD"));
assertFalse("Password property should have been excluded",
properties.get("ranger-yarn-plugin-properties").containsKey("test.ssl.password"));
@@ -748,7 +749,7 @@ public class BlueprintConfigurationProcessorTest {
createExportedAddress(expectedPortNum, expectedHostGroupName), falconStartupProperties.get("*.broker.url"));
assertEquals("Falcon Kerberos Principal property not properly exported",
- "falcon/" + "%HOSTGROUP::" + expectedHostGroupName + "%" + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal"));
+ "falcon/" + "%HOSTGROUP::" + expectedHostGroupName + "%" + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal"));
assertEquals("Falcon Kerberos HTTP Principal property not properly exported",
"HTTP/" + "%HOSTGROUP::" + expectedHostGroupName + "%" + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal"));
@@ -1126,19 +1127,19 @@ public class BlueprintConfigurationProcessorTest {
configProcessor.doUpdateForBlueprintExport();
assertEquals("Yarn Log Server URL was incorrectly exported",
- "http://" + "%HOSTGROUP::" + expectedHostGroupName + "%" +":19888/jobhistory/logs", yarnSiteProperties.get("yarn.log.server.url"));
+ "http://" + "%HOSTGROUP::" + expectedHostGroupName + "%" + ":19888/jobhistory/logs", yarnSiteProperties.get("yarn.log.server.url"));
assertEquals("Yarn ResourceManager hostname was incorrectly exported",
- createExportedHostName(expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.hostname"));
+ createExportedHostName(expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.hostname"));
assertEquals("Yarn ResourceManager tracker address was incorrectly exported",
- createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.resource-tracker.address"));
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.resource-tracker.address"));
assertEquals("Yarn ResourceManager webapp address was incorrectly exported",
createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.webapp.address"));
assertEquals("Yarn ResourceManager scheduler address was incorrectly exported",
- createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.scheduler.address"));
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.scheduler.address"));
assertEquals("Yarn ResourceManager address was incorrectly exported",
createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.address"));
assertEquals("Yarn ResourceManager admin address was incorrectly exported",
- createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.admin.address"));
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.admin.address"));
assertEquals("Yarn ResourceManager timeline-service address was incorrectly exported",
createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.address"));
assertEquals("Yarn ResourceManager timeline webapp address was incorrectly exported",
@@ -1456,7 +1457,7 @@ public class BlueprintConfigurationProcessorTest {
System.out.println("RWN: exported value of hive.metastore.uris = " + hiveSiteProperties.get("hive.metastore.uris"));
assertEquals("hive property not properly exported",
- "thrift://" + createExportedAddress(expectedPortNum, expectedHostGroupName) + "," + "thrift://" + createExportedAddress(expectedPortNum, expectedHostGroupNameTwo), hiveSiteProperties.get("hive.metastore.uris"));
+ "thrift://" + createExportedAddress(expectedPortNum, expectedHostGroupName) + "," + "thrift://" + createExportedAddress(expectedPortNum, expectedHostGroupNameTwo), hiveSiteProperties.get("hive.metastore.uris"));
assertEquals("hive property not properly exported",
createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("javax.jdo.option.ConnectionURL"));
assertEquals("hive property not properly exported",
@@ -1472,10 +1473,10 @@ public class BlueprintConfigurationProcessorTest {
createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hive.hosts"));
assertEquals("hive property not properly exported",
- createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.HTTP.hosts"));
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.HTTP.hosts"));
assertEquals("hive property not properly exported",
- createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hcat.hosts"));
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hcat.hosts"));
assertEquals("hive zookeeper quorum property not properly exported",
createExportedAddress(expectedPortNum, expectedHostGroupName) + "," + createExportedAddress(expectedPortNum, expectedHostGroupNameTwo),
@@ -1549,7 +1550,7 @@ public class BlueprintConfigurationProcessorTest {
assertEquals("oozie property not exported correctly",
createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.base.url"));
assertEquals("oozie property not exported correctly",
- createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.authentication.kerberos.principal"));
+ createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.authentication.kerberos.principal"));
assertEquals("oozie property not exported correctly",
createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal"));
assertEquals("oozie property not exported correctly",
@@ -1806,7 +1807,7 @@ public class BlueprintConfigurationProcessorTest {
assertEquals("Property with undefined host was incorrectly exported",
"undefined", properties.get("nimbus.childopts"));
assertEquals("Property with undefined host was incorrectly exported",
- "some other info, undefined, more info" , properties.get("worker.childopts"));
+ "some other info, undefined, more info", properties.get("worker.childopts"));
}
@Test
@@ -2478,8 +2479,8 @@ public class BlueprintConfigurationProcessorTest {
updater.doUpdateForClusterCreate();
assertEquals("Unexpected config update for templeton.hive.properties",
- expectedPropertyValue,
- webHCatSiteProperties.get("templeton.hive.properties"));
+ expectedPropertyValue,
+ webHCatSiteProperties.get("templeton.hive.properties"));
}
@Test
@@ -4192,9 +4193,9 @@ public class BlueprintConfigurationProcessorTest {
assertFalse("hbase.coprocessor.regionserver.classes should have been filtered out of configuration",
hbaseSiteProperties.containsKey("hbase.coprocessor.regionserver.classes"));
assertTrue("hbase.coprocessor.master.classes should not have been filtered out of configuration",
- hbaseSiteProperties.containsKey("hbase.coprocessor.master.classes"));
+ hbaseSiteProperties.containsKey("hbase.coprocessor.master.classes"));
assertTrue("hbase.coprocessor.region.classes should not have been filtered out of configuration",
- hbaseSiteProperties.containsKey("hbase.coprocessor.master.classes"));
+ hbaseSiteProperties.containsKey("hbase.coprocessor.master.classes"));
}
@Test
@@ -5419,6 +5420,466 @@ public class BlueprintConfigurationProcessorTest {
assertNotNull(clusterConfig.getPropertyValue("core-site", "fs.stackDefault.key2"));
}
+
+ @Test
+ public void testRangerAdminProperties() throws Exception {
+ // Given
+
+ final String rangerAdminConfigType = "admin-properties";
+
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> rangerAdminProperties = new HashMap<String, String>();
+ properties.put(rangerAdminConfigType, rangerAdminProperties);
+ rangerAdminProperties.put("policymgr_external_url", "http://%HOSTGROUP::group1%:100");
+
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> rangerComponents = new HashSet<String>();
+ rangerComponents.add("RANGER_ADMIN");
+ rangerComponents.add("RANGER_USERSYNC");
+ TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1"));
+
+ Collection<TestHostGroup> hostGroups = Collections.singletonList(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("http://host1:100", clusterConfig.getPropertyValue(rangerAdminConfigType, "policymgr_external_url"));
+ }
+
+ @Test
+ public void testRangerAdminProperties_defaults() throws Exception {
+ // Given
+
+ final String rangerAdminConfigType = "admin-properties";
+
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> rangerAdminProperties = new HashMap<String, String>();
+ properties.put(rangerAdminConfigType, rangerAdminProperties);
+ rangerAdminProperties.put("policymgr_external_url", "http://localhost:100");
+
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> rangerComponents = new HashSet<String>();
+ rangerComponents.add("RANGER_ADMIN");
+ rangerComponents.add("RANGER_USERSYNC");
+ TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1"));
+
+ Collection<TestHostGroup> hostGroups = Collections.singletonList(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("http://host1:100", clusterConfig.getPropertyValue(rangerAdminConfigType, "policymgr_external_url"));
+ }
+
+ @Test
+ public void testRangerAdminProperties_HA() throws Exception {
+ // Given
+
+ final String rangerAdminConfigType = "admin-properties";
+
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> rangerAdminProperties = new HashMap<String, String>();
+ properties.put(rangerAdminConfigType, rangerAdminProperties);
+ rangerAdminProperties.put("policymgr_external_url", "http://my.ranger.loadbalancer.com");
+
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> rangerComponents = new HashSet<String>();
+ rangerComponents.add("RANGER_ADMIN");
+ rangerComponents.add("RANGER_USERSYNC");
+ TestHostGroup group1 = new TestHostGroup("group1", rangerComponents, Collections.singleton("host1"));
+ TestHostGroup group2 = new TestHostGroup("group2", rangerComponents, Collections.singleton("host2"));
+
+ Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("http://my.ranger.loadbalancer.com", clusterConfig.getPropertyValue(rangerAdminConfigType, "policymgr_external_url"));
+ }
+
+ @Test
+ public void testRangerKmsServerProperties() throws Exception {
+ // Given
+
+ final String kmsSiteConfigType = "kms-site";
+
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> kmsSiteProperties = new HashMap<String, String>();
+ properties.put(kmsSiteConfigType, kmsSiteProperties);
+ kmsSiteProperties.put("hadoop.kms.key.provider.uri", "dbks://http@%HOSTGROUP::group1%:9292/kms");
+
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> kmsServerComponents = new HashSet<String>();
+ kmsServerComponents.add("RANGER_KMS_SERVER");
+
+ TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1"));
+
+
+ Collection<TestHostGroup> hostGroups = Collections.singleton(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("dbks://http@host1:9292/kms", clusterConfig.getPropertyValue(kmsSiteConfigType, "hadoop.kms.key.provider.uri"));
+ }
+
+
+ @Test
+ public void testRangerKmsServerProperties_default() throws Exception {
+ // Given
+
+ final String kmsSiteConfigType = "kms-site";
+
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> kmsSiteProperties = new HashMap<String, String>();
+ properties.put(kmsSiteConfigType, kmsSiteProperties);
+ kmsSiteProperties.put("hadoop.kms.key.provider.uri", "dbks://http@localhost:9292/kms");
+
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> kmsServerComponents = new HashSet<String>();
+ kmsServerComponents.add("RANGER_KMS_SERVER");
+
+ TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1"));
+
+
+ Collection<TestHostGroup> hostGroups = Collections.singleton(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("dbks://http@host1:9292/kms", clusterConfig.getPropertyValue(kmsSiteConfigType, "hadoop.kms.key.provider.uri"));
+ }
+
+
+ @Test
+ public void testHdfsWithRangerKmsServer() throws Exception {
+ // Given
+ final String configType = "hdfs-site";
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> configProperties = new HashMap<String, String>();
+
+ properties.put(configType, configProperties);
+ configProperties.put("dfs.encryption.key.provider.uri", "kms://http@%HOSTGROUP::group1%:9292/kms");
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> kmsServerComponents = new HashSet<String>();
+ kmsServerComponents.add("RANGER_KMS_SERVER");
+
+ Collection<String> hdfsComponents = new HashSet<String>();
+ hdfsComponents.add("NAMENODE");
+ hdfsComponents.add("DATANODE");
+
+
+ TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1"));
+ group1.components.add("DATANODE");
+
+ TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2"));
+
+ Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"));
+ }
+
+
+ @Test
+ public void testHdfsWithNoRangerKmsServer() throws Exception {
+ // Given
+
+ expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes();
+ expect(stack.getCardinality("DATANODE")).andReturn(new Cardinality("1+")).anyTimes();
+ expect(stack.getCardinality("RANGER_KMS_SERVER")).andReturn(new Cardinality("1+")).anyTimes();
+
+
+ final String configType = "hdfs-site";
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> configProperties = new HashMap<String, String>();
+
+ properties.put(configType, configProperties);
+ configProperties.put("dfs.encryption.key.provider.uri", "leave_untouched");
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> hdfsComponents = new HashSet<String>();
+ hdfsComponents.add("NAMENODE");
+ hdfsComponents.add("DATANODE");
+
+
+ TestHostGroup group1 = new TestHostGroup("group1", Collections.singletonList("DATANODE"), Collections.singleton("host1"));
+
+ TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2"));
+
+ Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("leave_untouched", clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"));
+ }
+
+
+ @Test
+ public void testHdfsWithRangerKmsServer_default() throws Exception {
+ // Given
+ final String configType = "hdfs-site";
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> configProperties = new HashMap<String, String>();
+
+ properties.put(configType, configProperties);
+ configProperties.put("dfs.encryption.key.provider.uri", "kms://http@localhost:9292/kms");
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> kmsServerComponents = new HashSet<String>();
+ kmsServerComponents.add("RANGER_KMS_SERVER");
+
+ Collection<String> hdfsComponents = new HashSet<String>();
+ hdfsComponents.add("NAMENODE");
+ hdfsComponents.add("DATANODE");
+
+
+ TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1"));
+ group1.components.add("DATANODE");
+
+ TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2"));
+
+ Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType, "dfs.encryption.key.provider.uri"));
+ }
+
+
+ @Test
+ public void testHadoopWithRangerKmsServer() throws Exception {
+ // Given
+ final String configType = "core-site";
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> configProperties = new HashMap<String, String>();
+
+ properties.put(configType, configProperties);
+ configProperties.put("hadoop.security.key.provider.path", "kms://http@%HOSTGROUP::group1%:9292/kms");
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> kmsServerComponents = new HashSet<String>();
+ kmsServerComponents.add("RANGER_KMS_SERVER");
+
+ Collection<String> hdfsComponents = new HashSet<String>();
+ hdfsComponents.add("NAMENODE");
+ hdfsComponents.add("DATANODE");
+
+
+ TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1"));
+ group1.components.add("DATANODE");
+
+ TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2"));
+
+ Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType, "hadoop.security.key.provider.path"));
+ }
+
+
+ @Test
+ public void testHadoopWithNoRangerKmsServer() throws Exception {
+ // Given
+ expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes();
+ expect(stack.getCardinality("DATANODE")).andReturn(new Cardinality("1+")).anyTimes();
+ expect(stack.getCardinality("RANGER_KMS_SERVER")).andReturn(new Cardinality("1+")).anyTimes();
+
+ final String configType = "core-site";
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> configProperties = new HashMap<String, String>();
+
+ properties.put(configType, configProperties);
+ configProperties.put("hadoop.security.key.provider.path", "leave_untouched");
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> hdfsComponents = new HashSet<String>();
+ hdfsComponents.add("NAMENODE");
+ hdfsComponents.add("DATANODE");
+
+
+ TestHostGroup group1 = new TestHostGroup("group1", Collections.singletonList("DATANODE"), Collections.singleton("host1"));
+
+ TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2"));
+
+ Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("leave_untouched", clusterConfig.getPropertyValue(configType, "hadoop.security.key.provider.path"));
+ }
+
+
+ @Test
+ public void testHadoopWithRangerKmsServer_default() throws Exception {
+ // Given
+ final String configType = "core-site";
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> configProperties = new HashMap<String, String>();
+
+ properties.put(configType, configProperties);
+ configProperties.put("hadoop.security.key.provider.path", "kms://http@localhost:9292/kms");
+
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+
+ Collection<String> kmsServerComponents = new HashSet<String>();
+ kmsServerComponents.add("RANGER_KMS_SERVER");
+
+ Collection<String> hdfsComponents = new HashSet<String>();
+ hdfsComponents.add("NAMENODE");
+ hdfsComponents.add("DATANODE");
+
+
+ TestHostGroup group1 = new TestHostGroup("group1", kmsServerComponents, Collections.singleton("host1"));
+ group1.components.add("DATANODE");
+
+ TestHostGroup group2 = new TestHostGroup("group2", hdfsComponents, Collections.singleton("host2"));
+
+ Collection<TestHostGroup> hostGroups = Lists.newArrayList(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ // When
+ configProcessor.doUpdateForClusterCreate();
+
+ // Then
+ assertEquals("kms://http@host1:9292/kms", clusterConfig.getPropertyValue(configType, "hadoop.security.key.provider.path"));
+ }
+
+
private Map<String, AdvisedConfiguration> createAdvisedConfigMap() {
Map<String, AdvisedConfiguration> advMap = new HashMap<String, AdvisedConfiguration>();
Map<String, String> confProp = new HashMap<String, String>();