You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2014/09/23 18:03:06 UTC
[11/12] git commit: Fixing blueprint export problems with undisguised
hostnames
Fixing blueprint export problems with undisguised hostnames
This patch addresses AMBARI-7368.
The Blueprint configuration processor was not properly
masking the hostnames in some properties after a
blueprint export from a running cluster.
Most of these properties were related to HA or
Kerberos, which are not supported by the Blueprints
processor directly.
This patch addresses this problem by registering
PropertyUpdater instances for each of the properties
that includes hostname information, but is not currently
masked during a blueprint export.
This patch implements support for tagging the HA Namenode configuration
properties as candidates for hostname masking. The HA properties
are dynamic with respect to naming, since the properties include
the nameserivces and name nodes defined during HA setup in the
Ambari UI. This patch implements a method to inspect the HA
configuration, if present, and register the HA properties for
each NameService and NameNode in order to remove the hostname
information during the exporting process.
This patch also implements support for the Namenode HA configuration
properties to be properly handled with hostname resolution during
cluster creation.
This patch implements several new unit test cases to verify this change, and also updates some existing test cases to verify this change as well.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/efa0fa6a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/efa0fa6a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/efa0fa6a
Branch: refs/heads/branch-alerts-dev
Commit: efa0fa6aba000ef90aca3b04fec059cfa465f38f
Parents: 4365e44
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Mon Sep 15 12:48:21 2014 -0400
Committer: John Speidel <js...@hortonworks.com>
Committed: Tue Sep 23 11:29:09 2014 -0400
----------------------------------------------------------------------
.../BlueprintConfigurationProcessor.java | 184 +++-
.../BlueprintConfigurationProcessorTest.java | 906 ++++++++++++++++++-
2 files changed, 1069 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/efa0fa6a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index c31a869..b3cc098 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -22,6 +22,8 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -96,7 +98,7 @@ public class BlueprintConfigurationProcessor {
* @return updated properties
*/
public Map<String, Map<String, String>> doUpdateForClusterCreate(Map<String, ? extends HostGroup> hostGroups) {
- for (Map<String, Map<String, PropertyUpdater>> updaterMap : allUpdaters) {
+ for (Map<String, Map<String, PropertyUpdater>> updaterMap : createCollectionOfUpdaters()) {
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaterMap.entrySet()) {
String type = entry.getKey();
for (Map.Entry<String, PropertyUpdater> updaterEntry : entry.getValue().entrySet()) {
@@ -115,6 +117,43 @@ public class BlueprintConfigurationProcessor {
}
/**
+ * Creates a Collection of PropertyUpdater maps that will handle the configuration
+ * update for this cluster. If NameNode HA is enabled, then updater
+ * instances will be added to the collection, in addition to the default list
+ * of Updaters that are statically defined.
+ *
+ * @return Collection of PropertyUpdater maps used to handle cluster config update
+ */
+ private Collection<Map<String, Map<String, PropertyUpdater>>> createCollectionOfUpdaters() {
+ return (isNameNodeHAEnabled()) ? addHAUpdaters(allUpdaters) : allUpdaters;
+ }
+
+ /**
+ * Creates a Collection of PropertyUpdater maps that include the NameNode HA properties, and
+ * adds these to the list of updaters used to process the cluster configuration. The HA
+ * properties are based on the names of the HA namservices and name nodes, and so must
+ * be registered at runtime, rather than in the static list. This new Collection includes
+ * the statically-defined updaters, in addition to the HA-related updaters.
+ *
+ * @param updaters a Collection of updater maps to be included in the list of updaters for
+ * this cluster config update
+ * @return A Collection of PropertyUpdater maps to handle the cluster config update
+ */
+ private Collection<Map<String, Map<String, PropertyUpdater>>> addHAUpdaters(Collection<Map<String, Map<String, PropertyUpdater>>> updaters) {
+ Collection<Map<String, Map<String, PropertyUpdater>>> highAvailabilityUpdaters =
+ new LinkedList<Map<String, Map<String, PropertyUpdater>>>();
+
+ // always add the statically-defined list of updaters to the list to use
+ // in processing cluster configuration
+ highAvailabilityUpdaters.addAll(updaters);
+
+ // add the updaters for the dynamic HA properties, based on the HA config in hdfs-site
+ highAvailabilityUpdaters.add(createMapOfHAUpdaters());
+
+ return highAvailabilityUpdaters;
+ }
+
+ /**
* Update properties for blueprint export.
* This involves converting concrete topology information to host groups.
*
@@ -125,12 +164,101 @@ public class BlueprintConfigurationProcessor {
public Map<String, Map<String, String>> doUpdateForBlueprintExport(Collection<? extends HostGroup> hostGroups) {
doSingleHostExportUpdate(hostGroups, singleHostTopologyUpdaters);
doSingleHostExportUpdate(hostGroups, dbHostTopologyUpdaters);
+
+ if (isNameNodeHAEnabled()) {
+ doNameNodeHAUpdate(hostGroups);
+ }
+
doMultiHostExportUpdate(hostGroups, multiHostTopologyUpdaters);
return properties;
}
/**
+ * Perform export update processing for HA configuration for NameNodes. The HA NameNode property
+ * names are based on the nameservices defined when HA is enabled via the Ambari UI, so this method
+ * dynamically determines the property names, and registers PropertyUpdaters to handle the masking of
+ * host names in these configuration items.
+ *
+ * @param hostGroups cluster host groups
+ */
+ public void doNameNodeHAUpdate(Collection<? extends HostGroup> hostGroups) {
+ Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = createMapOfHAUpdaters();
+
+ // perform a single host update on these dynamically generated property names
+ if (highAvailabilityUpdaters.get("hdfs-site").size() > 0) {
+ doSingleHostExportUpdate(hostGroups, highAvailabilityUpdaters);
+ }
+ }
+
+ /**
+ * Creates map of PropertyUpdater instances that are associated with
+ * NameNode High Availability (HA). The HA configuration property
+ * names are dynamic, and based on other HA config elements in
+ * hdfs-site. This method registers updaters for the required
+ * properties associated with each nameservice and namenode.
+ *
+ * @return a Map of registered PropertyUpdaters for handling HA properties in hdfs-site
+ */
+ private Map<String, Map<String, PropertyUpdater>> createMapOfHAUpdaters() {
+ Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = new HashMap<String, Map<String, PropertyUpdater>>();
+ Map<String, PropertyUpdater> hdfsSiteUpdatersForAvailability = new HashMap<String, PropertyUpdater>();
+ highAvailabilityUpdaters.put("hdfs-site", hdfsSiteUpdatersForAvailability);
+
+ Map<String, String> hdfsSiteConfig = properties.get("hdfs-site");
+ // generate the property names based on the current HA config for the NameNode deployments
+ for (String nameService : parseNameServices(hdfsSiteConfig)) {
+ for (String nameNode : parseNameNodes(nameService, hdfsSiteConfig)) {
+ final String httpsPropertyName = "dfs.namenode.https-address." + nameService + "." + nameNode;
+ hdfsSiteUpdatersForAvailability.put(httpsPropertyName, new SingleHostTopologyUpdater("NAMENODE"));
+ final String httpPropertyName = "dfs.namenode.http-address." + nameService + "." + nameNode;
+ hdfsSiteUpdatersForAvailability.put(httpPropertyName, new SingleHostTopologyUpdater("NAMENODE"));
+ final String rpcPropertyName = "dfs.namenode.rpc-address." + nameService + "." + nameNode;
+ hdfsSiteUpdatersForAvailability.put(rpcPropertyName, new SingleHostTopologyUpdater("NAMENODE"));
+ }
+ }
+ return highAvailabilityUpdaters;
+ }
+
+ /**
+ * Convenience function to determine if NameNode HA is enabled.
+ *
+ * @return true if NameNode HA is enabled
+ * false if NameNode HA is not enabled
+ */
+ boolean isNameNodeHAEnabled() {
+ return properties.containsKey("hdfs-site") && properties.get("hdfs-site").containsKey("dfs.nameservices");
+ }
+
+
+ /**
+ * Parses out the list of nameservices associated with this HDFS configuration.
+ *
+ * @param properties config properties for this cluster
+ *
+ * @return array of Strings that indicate the nameservices for this cluster
+ */
+ static String[] parseNameServices(Map<String, String> properties) {
+ final String nameServices = properties.get("dfs.nameservices");
+ return splitAndTrimStrings(nameServices);
+ }
+
+ /**
+ * Parses out the list of name nodes associated with a given HDFS
+ * NameService, based on a given HDFS configuration.
+ *
+ * @param nameService the nameservice used for this parsing
+ * @param properties config properties for this cluster
+ *
+ * @return array of Strings that indicate the name nodes associated
+ * with this nameservice
+ */
+ static String[] parseNameNodes(String nameService, Map<String, String> properties) {
+ final String nameNodes = properties.get("dfs.ha.namenodes." + nameService);
+ return splitAndTrimStrings(nameNodes);
+ }
+
+ /**
* Update single host topology configuration properties for blueprint export.
*
* @param hostGroups cluster export
@@ -274,15 +402,22 @@ public class BlueprintConfigurationProcessor {
return hosts;
}
-
/**
- * Provides package-level access to the map of single host topology updaters.
- * This is useful for facilitating unit-testing of this class.
+ * Convenience method for splitting out the HA-related properties, while
+ * also removing leading/trailing whitespace.
+ *
+ * @param propertyName property name to parse
*
- * @return the map of single host topology updaters
+ * @return an array of Strings that represent the comma-separated
+ * elements in this property
*/
- static Map<String, Map<String, PropertyUpdater>> getSingleHostTopologyUpdaters() {
- return singleHostTopologyUpdaters;
+ private static String[] splitAndTrimStrings(String propertyName) {
+ List<String> namesWithoutWhitespace = new LinkedList<String>();
+ for (String service : propertyName.split(",")) {
+ namesWithoutWhitespace.add(service.trim());
+ }
+
+ return namesWithoutWhitespace.toArray(new String[namesWithoutWhitespace.size()]);
}
/**
@@ -630,13 +765,19 @@ public class BlueprintConfigurationProcessor {
Map<String, PropertyUpdater> mapredEnvMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> hadoopEnvMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> hbaseEnvMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> hiveEnvMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> oozieEnvMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> multiWebhcatSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> multiHbaseSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> multiStormSiteMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> multiCoreSiteMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> multiHdfsSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> dbHiveSiteMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> nagiosEnvMap = new HashMap<String ,PropertyUpdater>();
+
singleHostTopologyUpdaters.put("hdfs-site", hdfsSiteMap);
singleHostTopologyUpdaters.put("mapred-site", mapredSiteMap);
@@ -647,6 +788,9 @@ public class BlueprintConfigurationProcessor {
singleHostTopologyUpdaters.put("oozie-site", oozieSiteMap);
singleHostTopologyUpdaters.put("storm-site", stormSiteMap);
singleHostTopologyUpdaters.put("falcon-startup.properties", falconStartupPropertiesMap);
+ singleHostTopologyUpdaters.put("nagios-env", nagiosEnvMap);
+ singleHostTopologyUpdaters.put("hive-env", hiveEnvMap);
+ singleHostTopologyUpdaters.put("oozie-env", oozieEnvMap);
mPropertyUpdaters.put("hadoop-env", hadoopEnvMap);
mPropertyUpdaters.put("hbase-env", hbaseEnvMap);
@@ -655,6 +799,8 @@ public class BlueprintConfigurationProcessor {
multiHostTopologyUpdaters.put("webhcat-site", multiWebhcatSiteMap);
multiHostTopologyUpdaters.put("hbase-site", multiHbaseSiteMap);
multiHostTopologyUpdaters.put("storm-site", multiStormSiteMap);
+ multiHostTopologyUpdaters.put("core-site", multiCoreSiteMap);
+ multiHostTopologyUpdaters.put("hdfs-site", multiHdfsSiteMap);
dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap);
@@ -666,6 +812,7 @@ public class BlueprintConfigurationProcessor {
hdfsSiteMap.put("dfs.namenode.https-address", new SingleHostTopologyUpdater("NAMENODE"));
coreSiteMap.put("fs.defaultFS", new SingleHostTopologyUpdater("NAMENODE"));
hbaseSiteMap.put("hbase.rootdir", new SingleHostTopologyUpdater("NAMENODE"));
+ multiHdfsSiteMap.put("dfs.namenode.shared.edits.dir", new MultipleHostTopologyUpdater("JOURNALNODE"));
// SECONDARY_NAMENODE
hdfsSiteMap.put("dfs.secondary.http.address", new SingleHostTopologyUpdater("SECONDARY_NAMENODE"));
@@ -690,17 +837,34 @@ public class BlueprintConfigurationProcessor {
yarnSiteMap.put("yarn.resourcemanager.address", new SingleHostTopologyUpdater("RESOURCEMANAGER"));
yarnSiteMap.put("yarn.resourcemanager.admin.address", new SingleHostTopologyUpdater("RESOURCEMANAGER"));
+ // APP_TIMELINE_SERVER
+ yarnSiteMap.put("yarn.timeline-service.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER"));
+ yarnSiteMap.put("yarn.timeline-service.webapp.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER"));
+ yarnSiteMap.put("yarn.timeline-service.webapp.https.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER"));
+
+
// HIVE_SERVER
hiveSiteMap.put("hive.metastore.uris", new SingleHostTopologyUpdater("HIVE_SERVER"));
dbHiveSiteMap.put("javax.jdo.option.ConnectionURL",
new DBTopologyUpdater("MYSQL_SERVER", "hive-env", "hive_database"));
+ multiCoreSiteMap.put("hadoop.proxyuser.hive.hosts", new MultipleHostTopologyUpdater("HIVE_SERVER"));
+ multiCoreSiteMap.put("hadoop.proxyuser.HTTP.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
+ multiCoreSiteMap.put("hadoop.proxyuser.hcat.hosts", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
+ multiWebhcatSiteMap.put("templeton.hive.properties", new MultipleHostTopologyUpdater("HIVE_SERVER"));
+ multiWebhcatSiteMap.put("templeton.kerberos.principal", new MultipleHostTopologyUpdater("WEBHCAT_SERVER"));
+ hiveEnvMap.put("hive_hostname", new SingleHostTopologyUpdater("HIVE_SERVER"));
// OOZIE_SERVER
oozieSiteMap.put("oozie.base.url", new SingleHostTopologyUpdater("OOZIE_SERVER"));
+ oozieSiteMap.put("oozie.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER"));
+ oozieSiteMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER"));
+ oozieEnvMap.put("oozie_hostname", new SingleHostTopologyUpdater("OOZIE_SERVER"));
+ multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER"));
// ZOOKEEPER_SERVER
multiHbaseSiteMap.put("hbase.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
multiWebhcatSiteMap.put("templeton.zookeeper.hosts", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
+ multiCoreSiteMap.put("ha.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
// STORM
stormSiteMap.put("nimbus.host", new SingleHostTopologyUpdater("NIMBUS"));
@@ -712,6 +876,12 @@ public class BlueprintConfigurationProcessor {
// FALCON
falconStartupPropertiesMap.put("*.broker.url", new SingleHostTopologyUpdater("FALCON_SERVER"));
+ falconStartupPropertiesMap.put("*.falcon.service.authentication.kerberos.principal", new SingleHostTopologyUpdater("FALCON_SERVER"));
+ falconStartupPropertiesMap.put("*.falcon.http.authentication.kerberos.principal", new SingleHostTopologyUpdater("FALCON_SERVER"));
+
+
+ // NAGIOS
+ nagiosEnvMap.put("nagios_principal_name", new SingleHostTopologyUpdater("NAGIOS_SERVER"));
// Required due to AMBARI-4933. These no longer seem to be required as the default values in the stack
http://git-wip-us.apache.org/repos/asf/ambari/blob/efa0fa6a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 3213b20..f906092 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -18,8 +18,10 @@
package org.apache.ambari.server.controller.internal;
+import org.easymock.EasyMockSupport;
import org.junit.Test;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -29,8 +31,9 @@ import java.util.Set;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertNotNull;
import static junit.framework.Assert.assertTrue;
-
+import static org.easymock.EasyMock.expect;
/**
* BlueprintConfigurationProcessor unit tests.
@@ -897,24 +900,899 @@ public class BlueprintConfigurationProcessorTest {
}
@Test
- public void testFalconConfigPropertyUpdaterAdded() throws Exception {
- Map<String, Map<String, BlueprintConfigurationProcessor.PropertyUpdater>> singleHostUpdaters =
- BlueprintConfigurationProcessor.getSingleHostTopologyUpdaters();
+ public void testFalconConfigExport() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedHostGroupName = "host_group_1";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> falconStartupProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("falcon-startup.properties", falconStartupProperties);
+
+ // setup properties that include host information
+ falconStartupProperties.put("*.broker.url", expectedHostName + ":" + expectedPortNum);
+ falconStartupProperties.put("*.falcon.service.authentication.kerberos.principal", "falcon/" + expectedHostName + "@EXAMPLE.COM");
+ falconStartupProperties.put("*.falcon.http.authentication.kerberos.principal", "HTTP/" + expectedHostName + "@EXAMPLE.COM");
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ // call top-level export method
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne));
+
+ assertEquals("Falcon Broker URL property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), falconStartupProperties.get("*.broker.url"));
+
+ assertEquals("Falcon Kerberos Principal property not properly exported",
+ "falcon/" + "%HOSTGROUP::" + expectedHostGroupName + "%" + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal"));
+
+ assertEquals("Falcon Kerberos HTTP Principal property not properly exported",
+ "HTTP/" + "%HOSTGROUP::" + expectedHostGroupName + "%" + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal"));
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testFalconConfigClusterUpdate() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedHostGroupName = "host_group_1";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> falconStartupProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("falcon-startup.properties", falconStartupProperties);
+
+ // setup properties that include host information
+ falconStartupProperties.put("*.broker.url", createExportedAddress(expectedPortNum, expectedHostGroupName));
+ falconStartupProperties.put("*.falcon.service.authentication.kerberos.principal", "falcon/" + createExportedHostName(expectedHostGroupName) + "@EXAMPLE.COM");
+ falconStartupProperties.put("*.falcon.http.authentication.kerberos.principal", "HTTP/" + createExportedHostName(expectedHostGroupName) + "@EXAMPLE.COM");
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ Map<String, HostGroup> mapOfHostGroups =
+ new HashMap<String, HostGroup>();
+ mapOfHostGroups.put(expectedHostGroupName, mockHostGroupOne);
+
+ // call top-level export method
+ configProcessor.doUpdateForClusterCreate(mapOfHostGroups);
+
+ assertEquals("Falcon Broker URL property not properly exported",
+ expectedHostName + ":" + expectedPortNum, falconStartupProperties.get("*.broker.url"));
+
+ assertEquals("Falcon Kerberos Principal property not properly exported",
+ "falcon/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal"));
+
+ assertEquals("Falcon Kerberos HTTP Principal property not properly exported",
+ "HTTP/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal"));
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testFalconConfigClusterUpdateDefaultConfig() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedHostGroupName = "host_group_1";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+ expect(mockHostGroupOne.getComponents()).andReturn(Arrays.asList("FALCON_SERVER")).atLeastOnce();
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> falconStartupProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("falcon-startup.properties", falconStartupProperties);
+
+ // setup properties that include host information
+ falconStartupProperties.put("*.broker.url", "localhost:" + expectedPortNum);
+ falconStartupProperties.put("*.falcon.service.authentication.kerberos.principal", "falcon/" + "localhost" + "@EXAMPLE.COM");
+ falconStartupProperties.put("*.falcon.http.authentication.kerberos.principal", "HTTP/" + "localhost" + "@EXAMPLE.COM");
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ Map<String, HostGroup> mapOfHostGroups =
+ new HashMap<String, HostGroup>();
+ mapOfHostGroups.put(expectedHostGroupName, mockHostGroupOne);
+
+ // call top-level export method
+ configProcessor.doUpdateForClusterCreate(mapOfHostGroups);
+
+ assertEquals("Falcon Broker URL property not properly exported",
+ expectedHostName + ":" + expectedPortNum, falconStartupProperties.get("*.broker.url"));
+
+ assertEquals("Falcon Kerberos Principal property not properly exported",
+ "falcon/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.service.authentication.kerberos.principal"));
+
+ assertEquals("Falcon Kerberos HTTP Principal property not properly exported",
+ "HTTP/" + expectedHostName + "@EXAMPLE.COM", falconStartupProperties.get("*.falcon.http.authentication.kerberos.principal"));
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testDoUpdateForClusterWithNameNodeHAEnabled() throws Exception {
+ final String expectedNameService = "mynameservice";
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedNodeOne = "nn1";
+ final String expectedNodeTwo = "nn2";
+ final String expectedHostGroupName = "host_group_1";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> hdfsSiteProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("hdfs-site", hdfsSiteProperties);
+
+ // setup hdfs HA config for test
+ hdfsSiteProperties.put("dfs.nameservices", expectedNameService);
+ hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne + ", " + expectedNodeTwo);
+
+
+ // setup properties that include exported host group information
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, createExportedAddress(expectedPortNum, expectedHostGroupName));
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, createExportedAddress(expectedPortNum, expectedHostGroupName));
+
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ Map<String, HostGroup> mapOfHostGroups = new HashMap<String, HostGroup>();
+ mapOfHostGroups.put(expectedHostGroupName,mockHostGroupOne);
+
+ configProcessor.doUpdateForClusterCreate(mapOfHostGroups);
+
+ // verify that the expected hostname was substitued for the host group name in the config
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ expectedHostName + ":" + expectedPortNum, hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo));
+
+ mockSupport.verifyAll();
+ }
+
+ @Test
+ public void testDoNameNodeHighAvailabilityUpdateWithHAEnabled() throws Exception {
+ final String expectedNameService = "mynameservice";
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedNodeOne = "nn1";
+ final String expectedNodeTwo = "nn2";
+ final String expectedHostGroupName = "host_group_1";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> hdfsSiteProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("hdfs-site", hdfsSiteProperties);
+
+ // setup hdfs config for test
+
+ hdfsSiteProperties.put("dfs.nameservices", expectedNameService);
+ hdfsSiteProperties.put("dfs.ha.namenodes.mynameservice", expectedNodeOne + ", " + expectedNodeTwo);
+
+
+ // setup properties that include host information
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne, expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo, expectedHostName + ":" + expectedPortNum);
+
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ // call top-level export method, which will call the HA-specific method if HA is enabled
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeTwo));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameService + "." + expectedNodeTwo));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameService + "." + expectedNodeTwo));
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testDoNameNodeHighAvailabilityUpdateWithHANotEnabled() throws Exception {
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> hdfsSiteProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("hdfs-site", hdfsSiteProperties);
+
+ // hdfs-site config for this test will not include an HA values
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ assertEquals("Incorrect initial state for hdfs-site config",
+ 0, hdfsSiteProperties.size());
+
+ // call top-level export method
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne));
+
+ assertEquals("Incorrect state for hdsf-site config after HA call in non-HA environment, should be zero",
+ 0, hdfsSiteProperties.size());
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testDoNameNodeHighAvailabilityUpdateWithHAEnabledMultipleServices() throws Exception {
+ final String expectedNameServiceOne = "mynameserviceOne";
+ final String expectedNameServiceTwo = "mynameserviceTwo";
+ final String expectedHostNameOne = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.apache.ambari.org";
+
+ final String expectedPortNum = "808080";
+ final String expectedNodeOne = "nn1";
+ final String expectedNodeTwo = "nn2";
+ final String expectedHostGroupName = "host_group_1";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostNameOne, expectedHostNameTwo, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> hdfsSiteProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("hdfs-site", hdfsSiteProperties);
+
+ // setup hdfs config for test
+
+ hdfsSiteProperties.put("dfs.nameservices", expectedNameServiceOne + "," + expectedNameServiceTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceOne, expectedNodeOne + ", " + expectedNodeTwo);
+ hdfsSiteProperties.put("dfs.ha.namenodes." + expectedNameServiceTwo, expectedNodeOne + ", " + expectedNodeTwo);
+
+
+ // setup properties that include host information for nameservice one
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeOne, expectedHostNameOne + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeTwo, expectedHostNameOne + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeOne, expectedHostNameOne + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeTwo, expectedHostNameOne + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeOne, expectedHostNameOne + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeTwo, expectedHostNameOne + ":" + expectedPortNum);
+
+ // setup properties that include host information for nameservice two
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeOne, expectedHostNameTwo + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeTwo, expectedHostNameTwo + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeOne, expectedHostNameTwo + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeTwo, expectedHostNameTwo + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeOne, expectedHostNameTwo + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeTwo, expectedHostNameTwo + ":" + expectedPortNum);
+
- assertTrue("Falcon startup.properties map was not added to the list of updater maps",
- singleHostUpdaters.containsKey("falcon-startup.properties"));
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
- Map<String, BlueprintConfigurationProcessor.PropertyUpdater> fieldsToUpdaters =
- singleHostUpdaters.get("falcon-startup.properties");
+ // call top-level export method, which will call the HA-specific method if HA is enabled
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne));
- assertTrue("Expected Falcon config property was not present in updater map",
- fieldsToUpdaters.containsKey("*.broker.url"));
+ // verify results for name service one
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceOne + "." + expectedNodeTwo));
- assertTrue("PropertyUpdater was not of the expected type for Falcon config property",
- fieldsToUpdaters.get("*.broker.url") instanceof BlueprintConfigurationProcessor.SingleHostTopologyUpdater);
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceOne + "." + expectedNodeTwo));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceOne + "." + expectedNodeTwo));
+
+
+ // verify results for name service two
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameServiceTwo + "." + expectedNodeTwo));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address." + expectedNameServiceTwo + "." + expectedNodeTwo));
+
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeOne));
+ assertEquals("HTTPS address HA property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.rpc-address." + expectedNameServiceTwo + "." + expectedNodeTwo));
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testIsNameNodeHAEnabled() throws Exception {
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ assertFalse("Incorrect HA detection, hdfs-site not available",
+ configProcessor.isNameNodeHAEnabled());
+
+ Map<String, String> hdfsSiteMap = new HashMap<String, String>();
+ configProperties.put("hdfs-site", hdfsSiteMap);
+
+ assertFalse("Incorrect HA detection, HA flag not enabled",
+ configProcessor.isNameNodeHAEnabled());
+
+ hdfsSiteMap.put("dfs.nameservices", "myTestNameService");
+
+ assertTrue("Incorrect HA detection, HA was enabled",
+ configProcessor.isNameNodeHAEnabled());
+
+ }
+
+ @Test
+ public void testParseNameServices() throws Exception {
+ Map<String, String> hdfsSiteConfigMap =
+ new HashMap<String, String>();
+ hdfsSiteConfigMap.put("dfs.nameservices", "serviceOne");
+
+ // verify that a single service is parsed correctly
+ String[] result = BlueprintConfigurationProcessor.parseNameServices(hdfsSiteConfigMap);
+
+ assertNotNull("Resulting array was null",
+ result);
+ assertEquals("Incorrect array size",
+ 1, result.length);
+ assertEquals("Incorrect value for returned name service",
+ "serviceOne", result[0]);
+
+ // verify that multiple services are parsed correctly
+ hdfsSiteConfigMap.put("dfs.nameservices", " serviceTwo, serviceThree, serviceFour");
+
+ String[] resultTwo = BlueprintConfigurationProcessor.parseNameServices(hdfsSiteConfigMap);
+
+ assertNotNull("Resulting array was null",
+ resultTwo);
+ assertEquals("Incorrect array size",
+ 3, resultTwo.length);
+ assertEquals("Incorrect value for returned name service",
+ "serviceTwo", resultTwo[0]);
+ assertEquals("Incorrect value for returned name service",
+ "serviceThree", resultTwo[1]);
+ assertEquals("Incorrect value for returned name service",
+ "serviceFour", resultTwo[2]);
+ }
+
+ @Test
+ public void testParseNameNodes() throws Exception {
+ final String expectedServiceName = "serviceOne";
+ Map<String, String> hdfsSiteConfigMap =
+ new HashMap<String, String>();
+ hdfsSiteConfigMap.put("dfs.ha.namenodes." + expectedServiceName, "node1");
+
+ // verify that a single name node is parsed correctly
+ String[] result =
+ BlueprintConfigurationProcessor.parseNameNodes(expectedServiceName, hdfsSiteConfigMap);
+
+ assertNotNull("Resulting array was null",
+ result);
+ assertEquals("Incorrect array size",
+ 1, result.length);
+ assertEquals("Incorrect value for returned name nodes",
+ "node1", result[0]);
+
+ // verify that multiple name nodes are parsed correctly
+ hdfsSiteConfigMap.put("dfs.ha.namenodes." + expectedServiceName, " nodeSeven, nodeEight, nodeNine");
+
+ String[] resultTwo =
+ BlueprintConfigurationProcessor.parseNameNodes(expectedServiceName, hdfsSiteConfigMap);
+
+ assertNotNull("Resulting array was null",
+ resultTwo);
+ assertEquals("Incorrect array size",
+ 3, resultTwo.length);
+ assertEquals("Incorrect value for returned name node",
+ "nodeSeven", resultTwo[0]);
+ assertEquals("Incorrect value for returned name node",
+ "nodeEight", resultTwo[1]);
+ assertEquals("Incorrect value for returned name node",
+ "nodeNine", resultTwo[2]);
+
+ }
+
+ @Test
+ public void testYarnConfigExported() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedHostGroupName = "host_group_1";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> yarnSiteProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("yarn-site", yarnSiteProperties);
+
+ // setup properties that include host information
+ yarnSiteProperties.put("yarn.log.server.url", "http://" + expectedHostName +":19888/jobhistory/logs");
+ yarnSiteProperties.put("yarn.resourcemanager.hostname", expectedHostName);
+ yarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", expectedHostName + ":" + expectedPortNum);
+ yarnSiteProperties.put("yarn.resourcemanager.webapp.address", expectedHostName + ":" + expectedPortNum);
+ yarnSiteProperties.put("yarn.resourcemanager.scheduler.address", expectedHostName + ":" + expectedPortNum);
+ yarnSiteProperties.put("yarn.resourcemanager.address", expectedHostName + ":" + expectedPortNum);
+ yarnSiteProperties.put("yarn.resourcemanager.admin.address", expectedHostName + ":" + expectedPortNum);
+ yarnSiteProperties.put("yarn.timeline-service.address", expectedHostName + ":" + expectedPortNum);
+ yarnSiteProperties.put("yarn.timeline-service.webapp.address", expectedHostName + ":" + expectedPortNum);
+ yarnSiteProperties.put("yarn.timeline-service.webapp.https.address", expectedHostName + ":" + expectedPortNum);
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ // call top-level export method
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne));
+
+ assertEquals("Yarn Log Server URL was incorrectly exported",
+ "http://" + "%HOSTGROUP::" + expectedHostGroupName + "%" +":19888/jobhistory/logs", yarnSiteProperties.get("yarn.log.server.url"));
+ assertEquals("Yarn ResourceManager hostname was incorrectly exported",
+ createExportedHostName(expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.hostname"));
+ assertEquals("Yarn ResourceManager tracker address was incorrectly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.resource-tracker.address"));
+ assertEquals("Yarn ResourceManager webapp address was incorrectly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.webapp.address"));
+ assertEquals("Yarn ResourceManager scheduler address was incorrectly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.scheduler.address"));
+ assertEquals("Yarn ResourceManager address was incorrectly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.address"));
+ assertEquals("Yarn ResourceManager admin address was incorrectly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.resourcemanager.admin.address"));
+ assertEquals("Yarn ResourceManager timeline-service address was incorrectly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.address"));
+ assertEquals("Yarn ResourceManager timeline webapp address was incorrectly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.webapp.address"));
+ assertEquals("Yarn ResourceManager timeline webapp HTTPS address was incorrectly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), yarnSiteProperties.get("yarn.timeline-service.webapp.https.address"));
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testHDFSConfigExported() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedPortNum = "808080";
+ final String expectedHostGroupName = "host_group_1";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> hdfsSiteProperties =
+ new HashMap<String, String>();
+
+ Map<String, String> coreSiteProperties =
+ new HashMap<String, String>();
+
+ Map<String, String> hbaseSiteProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("hdfs-site", hdfsSiteProperties);
+ configProperties.put("core-site", coreSiteProperties);
+ configProperties.put("hbase-site", hbaseSiteProperties);
+
+ // setup properties that include host information
+ hdfsSiteProperties.put("dfs.http.address", expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.https.address", expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.http-address", expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.https-address", expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.secondary.http.address", expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.secondary.http-address", expectedHostName + ":" + expectedPortNum);
+ hdfsSiteProperties.put("dfs.namenode.shared.edits.dir", expectedHostName + ":" + expectedPortNum);
+
+ coreSiteProperties.put("fs.default.name", expectedHostName + ":" + expectedPortNum);
+ coreSiteProperties.put("fs.defaultFS", "hdfs://" + expectedHostName + ":" + expectedPortNum);
+
+ hbaseSiteProperties.put("hbase.rootdir", "hdfs://" + expectedHostName + ":" + expectedPortNum + "/apps/hbase/data");
+
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ // call top-level export method
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne));
+
+ assertEquals("hdfs config property not exported properly",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.http.address"));
+ assertEquals("hdfs config property not exported properly",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.https.address"));
+ assertEquals("hdfs config property not exported properly",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.http-address"));
+ assertEquals("hdfs config property not exported properly",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address"));
+ assertEquals("hdfs config property not exported properly",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.secondary.http.address"));
+ assertEquals("hdfs config property not exported properly",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.secondary.http-address"));
+ assertEquals("hdfs config property not exported properly",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.shared.edits.dir"));
+
+ assertEquals("hdfs config in core-site not exported properly",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), coreSiteProperties.get("fs.default.name"));
+ assertEquals("hdfs config in core-site not exported properly",
+ "hdfs://" + createExportedAddress(expectedPortNum, expectedHostGroupName), coreSiteProperties.get("fs.defaultFS"));
+
+ assertEquals("hdfs config in hbase-site not exported properly",
+ "hdfs://" + createExportedAddress(expectedPortNum, expectedHostGroupName) + "/apps/hbase/data", hbaseSiteProperties.get("hbase.rootdir"));
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testHiveConfigExported() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.ambari.apache.org";
+ final String expectedPortNum = "808080";
+ final String expectedHostGroupName = "host_group_1";
+ final String expectedHostGroupNameTwo = "host_group_2";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+ HostGroup mockHostGroupTwo = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupTwo.getHostInfo()).andReturn(Arrays.asList(expectedHostNameTwo, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+ expect(mockHostGroupTwo.getName()).andReturn(expectedHostGroupNameTwo).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> hiveSiteProperties =
+ new HashMap<String, String>();
+ Map<String, String> hiveEnvProperties =
+ new HashMap<String, String>();
+ Map<String, String> webHCatSiteProperties =
+ new HashMap<String, String>();
+ Map<String, String> coreSiteProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("hive-site", hiveSiteProperties);
+ configProperties.put("hive-env", hiveEnvProperties);
+ configProperties.put("webhcat-site", webHCatSiteProperties);
+ configProperties.put("core-site", coreSiteProperties);
+
+
+ // setup properties that include host information
+ hiveSiteProperties.put("hive.metastore.uris", expectedHostName + ":" + expectedPortNum);
+ hiveSiteProperties.put("javax.jdo.option.ConnectionURL", expectedHostName + ":" + expectedPortNum);
+ hiveEnvProperties.put("hive_hostname", expectedHostName);
+
+ webHCatSiteProperties.put("templeton.hive.properties", expectedHostName + "," + expectedHostNameTwo);
+ webHCatSiteProperties.put("templeton.kerberos.principal", expectedHostName);
+
+ coreSiteProperties.put("hadoop.proxyuser.hive.hosts", expectedHostName + "," + expectedHostNameTwo);
+ coreSiteProperties.put("hadoop.proxyuser.HTTP.hosts", expectedHostName + "," + expectedHostNameTwo);
+ coreSiteProperties.put("hadoop.proxyuser.hcat.hosts", expectedHostName + "," + expectedHostNameTwo);
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ // call top-level export method
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne, mockHostGroupTwo));
+
+ assertEquals("hive property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("hive.metastore.uris"));
+ assertEquals("hive property not properly exported",
+ createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("javax.jdo.option.ConnectionURL"));
+ assertEquals("hive property not properly exported",
+ createExportedHostName(expectedHostGroupName), hiveEnvProperties.get("hive_hostname"));
+
+ assertEquals("hive property not properly exported",
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo),
+ webHCatSiteProperties.get("templeton.hive.properties"));
+ assertEquals("hive property not properly exported",
+ createExportedHostName(expectedHostGroupName), webHCatSiteProperties.get("templeton.kerberos.principal"));
+
+ assertEquals("hive property not properly exported",
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hive.hosts"));
+
+ assertEquals("hive property not properly exported",
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.HTTP.hosts"));
+
+ assertEquals("hive property not properly exported",
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.hcat.hosts"));
+
+ mockSupport.verifyAll();
+ }
+
+
+ @Test
+ public void testOozieConfigExported() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.ambari.apache.org";
+ final String expectedHostGroupName = "host_group_1";
+ final String expectedHostGroupNameTwo = "host_group_2";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+ HostGroup mockHostGroupTwo = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupTwo.getHostInfo()).andReturn(Arrays.asList(expectedHostNameTwo, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+ expect(mockHostGroupTwo.getName()).andReturn(expectedHostGroupNameTwo).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> oozieSiteProperties =
+ new HashMap<String, String>();
+ Map<String, String> oozieEnvProperties =
+ new HashMap<String, String>();
+ Map<String, String> coreSiteProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("oozie-site", oozieSiteProperties);
+ configProperties.put("oozie-env", oozieEnvProperties);
+ configProperties.put("hive-env", oozieEnvProperties);
+ configProperties.put("core-site", coreSiteProperties);
+
+ oozieSiteProperties.put("oozie.base.url", expectedHostName);
+ oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName);
+ oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName);
+
+ oozieEnvProperties.put("oozie_hostname", expectedHostName);
+
+ coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo);
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ // call top-level export method
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne, mockHostGroupTwo));
+
+ assertEquals("oozie property not exported correctly",
+ createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.base.url"));
+ assertEquals("oozie property not exported correctly",
+ createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.authentication.kerberos.principal"));
+ assertEquals("oozie property not exported correctly",
+ createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal"));
+ assertEquals("oozie property not exported correctly",
+ createExportedHostName(expectedHostGroupName), oozieEnvProperties.get("oozie_hostname"));
+ assertEquals("oozie property not exported correctly",
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.oozie.hosts"));
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testZookeeperConfigExported() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.ambari.apache.org";
+ final String expectedHostGroupName = "host_group_1";
+ final String expectedHostGroupNameTwo = "host_group_2";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+ HostGroup mockHostGroupTwo = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupTwo.getHostInfo()).andReturn(Arrays.asList(expectedHostNameTwo, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+ expect(mockHostGroupTwo.getName()).andReturn(expectedHostGroupNameTwo).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> coreSiteProperties =
+ new HashMap<String, String>();
+ Map<String, String> hbaseSiteProperties =
+ new HashMap<String, String>();
+ Map<String, String> webHCatSiteProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("core-site", coreSiteProperties);
+ configProperties.put("hbase-site", hbaseSiteProperties);
+ configProperties.put("webhcat-site", webHCatSiteProperties);
+
+ coreSiteProperties.put("ha.zookeeper.quorum", expectedHostName + "," + expectedHostNameTwo);
+ hbaseSiteProperties.put("hbase.zookeeper.quorum", expectedHostName + "," + expectedHostNameTwo);
+ webHCatSiteProperties.put("templeton.zookeeper.hosts", expectedHostName + "," + expectedHostNameTwo);
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ // call top-level export method
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne, mockHostGroupTwo));
+
+ assertEquals("zookeeper config not properly exported",
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo),
+ coreSiteProperties.get("ha.zookeeper.quorum"));
+ assertEquals("zookeeper config not properly exported",
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo),
+ hbaseSiteProperties.get("hbase.zookeeper.quorum"));
+ assertEquals("zookeeper config not properly exported",
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo),
+ webHCatSiteProperties.get("templeton.zookeeper.hosts"));
+
+ mockSupport.verifyAll();
+
+ }
+
+ @Test
+ public void testNagiosConfigExported() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostGroupName = "host_group_1";
+
+ EasyMockSupport mockSupport = new EasyMockSupport();
+
+ HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+ expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName, "serverTwo")).atLeastOnce();
+ expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+ mockSupport.replayAll();
+
+ Map<String, Map<String, String>> configProperties =
+ new HashMap<String, Map<String, String>>();
+
+ Map<String, String> nagiosEnvProperties =
+ new HashMap<String, String>();
+
+ configProperties.put("nagios-env", nagiosEnvProperties);
+
+ nagiosEnvProperties.put("nagios_principal_name", expectedHostName);
+
+ BlueprintConfigurationProcessor configProcessor =
+ new BlueprintConfigurationProcessor(configProperties);
+
+ // call top-level export method
+ configProcessor.doUpdateForBlueprintExport(Arrays.asList(mockHostGroupOne));
+
+ assertEquals("nagios config not properly exported",
+ createExportedHostName(expectedHostGroupName),
+ nagiosEnvProperties.get("nagios_principal_name"));
+
+ mockSupport.verifyAll();
+
+ }
+
+ private static String createExportedAddress(String expectedPortNum, String expectedHostGroupName) {
+ return createExportedHostName(expectedHostGroupName) + ":" + expectedPortNum;
+ }
- assertEquals("PropertyUpdater was not associated with the expected component name",
- "FALCON_SERVER", ((BlueprintConfigurationProcessor.SingleHostTopologyUpdater)fieldsToUpdaters.get("*.broker.url")).getComponentName());
+ private static String createExportedHostName(String expectedHostGroupName) {
+ return "%HOSTGROUP::" + expectedHostGroupName + "%";
}
private class TestHostGroup implements HostGroup {