You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rn...@apache.org on 2015/05/22 17:17:40 UTC
ambari git commit: AMBARI-11308. Oozie HA configuration property not
exported properly in a Blueprint. (rnettleton)
Repository: ambari
Updated Branches:
refs/heads/trunk 714d2535e -> 832f7e1ec
AMBARI-11308. Oozie HA configuration property not exported properly in a Blueprint. (rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/832f7e1e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/832f7e1e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/832f7e1e
Branch: refs/heads/trunk
Commit: 832f7e1ec7a04c8ade400f9bfa679d2d7069ae13
Parents: 714d253
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Fri May 22 11:17:10 2015 -0400
Committer: Bob Nettleton <rn...@hortonworks.com>
Committed: Fri May 22 11:17:31 2015 -0400
----------------------------------------------------------------------
.../BlueprintConfigurationProcessor.java | 70 +++++++++++++++++++
.../OOZIE/5.0.0.2.3/metainfo.xml | 2 +-
.../BlueprintConfigurationProcessorTest.java | 71 ++++++++++++++++++++
3 files changed, 142 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/832f7e1e/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 6984881..a84bf3d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -231,6 +231,10 @@ public class BlueprintConfigurationProcessor {
doYarnResourceManagerHAUpdate();
}
+ if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) {
+ doOozieServerHAUpdate();
+ }
+
Collection<Map<String, Map<String, String>>> allConfigs = new ArrayList<Map<String, Map<String, String>>>();
allConfigs.add(clusterTopology.getConfiguration().getFullProperties());
for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) {
@@ -304,6 +308,10 @@ public class BlueprintConfigurationProcessor {
updaters = addYarnResourceManagerHAUpdaters(updaters);
}
+ if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) {
+ updaters = addOozieServerHAUpdaters(updaters);
+ }
+
return updaters;
}
@@ -359,6 +367,33 @@ public class BlueprintConfigurationProcessor {
return highAvailabilityUpdaters;
}
+
+ /**
+ * Creates a Collection of PropertyUpdater maps that include the OozieServer HA properties, and
+ * adds these to the list of updaters used to process the cluster configuration.
+
+ * This new Collection includes the statically-defined updaters,
+ * in addition to the HA-related updaters.
+ *
+ * @param updaters a Collection of updater maps to be included in the list of updaters for
+ * this cluster config update
+ * @return A Collection of PropertyUpdater maps to handle the cluster config update
+ */
+ private Collection<Map<String, Map<String, PropertyUpdater>>> addOozieServerHAUpdaters(Collection<Map<String, Map<String, PropertyUpdater>>> updaters) {
+ Collection<Map<String, Map<String, PropertyUpdater>>> highAvailabilityUpdaters =
+ new LinkedList<Map<String, Map<String, PropertyUpdater>>>();
+
+ // always add the statically-defined list of updaters to the list to use
+ // in processing cluster configuration
+ highAvailabilityUpdaters.addAll(updaters);
+
+ // add the updaters for the Oozie HA properties not defined in stack, but
+ // required to be present/updated in oozie-site
+ highAvailabilityUpdaters.add(createMapOfOozieServerHAUpdaters());
+
+ return highAvailabilityUpdaters;
+ }
+
/**
* Performs export update for the set of properties that do not
* require update during cluster setup, but should be removed
@@ -418,6 +453,20 @@ public class BlueprintConfigurationProcessor {
}
}
+ /**
+ * Perform export update processing for HA configuration for Oozie servers. The properties used
+ * in Oozie HA are not defined in the stack, but need to be added at runtime during an HA
+ * deployment in order to support exporting/redeploying clusters with Oozie HA config.
+ *
+ */
+ public void doOozieServerHAUpdate() {
+ Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = createMapOfOozieServerHAUpdaters();
+
+ if (highAvailabilityUpdaters.get("oozie-site").size() > 0) {
+ doMultiHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration().getFullProperties());
+ }
+ }
+
/**
* Creates map of PropertyUpdater instances that are associated with
@@ -481,6 +530,27 @@ public class BlueprintConfigurationProcessor {
}
/**
+ * Creates map of PropertyUpdater instances that are associated with
+ * Oozie Server High Availability (HA).
+ *
+ * @return a Map of registered PropertyUpdaters for handling HA properties in oozie-site
+ */
+ private Map<String, Map<String, PropertyUpdater>> createMapOfOozieServerHAUpdaters() {
+ Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = new HashMap<String, Map<String, PropertyUpdater>>();
+ Map<String, PropertyUpdater> oozieSiteUpdatersForAvailability = new HashMap<String, PropertyUpdater>();
+ highAvailabilityUpdaters.put("oozie-site", oozieSiteUpdatersForAvailability);
+
+ // register a multi-host property updater for this Oozie property.
+ // this property is not defined in the stacks, since HA is not supported yet
+ // by the stack definition syntax. This property should only be considered in
+ // an Oozie HA cluster.
+ oozieSiteUpdatersForAvailability.put("oozie.zookeeper.connection.string", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
+
+ return highAvailabilityUpdaters;
+
+ }
+
+ /**
* Static convenience function to determine if Oozie HA is enabled
* @param configProperties configuration properties for this cluster
* @return true if Oozie HA is enabled
http://git-wip-us.apache.org/repos/asf/ambari/blob/832f7e1e/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml
index 463abc3..a78c93a 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/metainfo.xml
@@ -30,7 +30,7 @@
<name>OOZIE_SERVER</name>
<displayName>Oozie Server</displayName>
<category>MASTER</category>
- <cardinality>1</cardinality>
+ <cardinality>1+</cardinality>
<versionAdvertised>true</versionAdvertised>
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/ambari/blob/832f7e1e/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 6849b45..595cc4a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -2223,6 +2223,77 @@ public class BlueprintConfigurationProcessorTest {
}
@Test
+ public void testOozieHAEnabledExport() throws Exception {
+ final String expectedHostName = "c6401.apache.ambari.org";
+ final String expectedHostNameTwo = "c6402.ambari.apache.org";
+ final String expectedExternalHost = "c6408.ambari.apache.org";
+ final String expectedHostGroupName = "host_group_1";
+ final String expectedHostGroupNameTwo = "host_group_2";
+ final String expectedPortNum = "80000";
+
+ Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> oozieSiteProperties = new HashMap<String, String>();
+ Map<String, String> oozieEnvProperties = new HashMap<String, String>();
+ Map<String, String> coreSiteProperties = new HashMap<String, String>();
+
+ configProperties.put("oozie-site", oozieSiteProperties);
+ configProperties.put("oozie-env", oozieEnvProperties);
+ configProperties.put("hive-env", oozieEnvProperties);
+ configProperties.put("core-site", coreSiteProperties);
+
+
+ oozieSiteProperties.put("oozie.base.url", expectedHostName + ":" + expectedPortNum);
+ oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName);
+ oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName);
+ oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + expectedExternalHost + "/ooziedb");
+
+ // simulate the Oozie HA configuration
+ oozieSiteProperties.put("oozie.services.ext",
+ "org.apache.oozie.service.ZKLocksService,org.apache.oozie.service.ZKXLogStreamingService,org.apache.oozie.service.ZKJobsConcurrencyService,org.apache.oozie.service.ZKUUIDService");
+ oozieSiteProperties.put("oozie.zookeeper.connection.string", createHostAddress(expectedHostName, "2181") + "," + createHostAddress(expectedHostNameTwo, "2181"));
+
+
+ oozieEnvProperties.put("oozie_hostname", expectedHostName);
+ oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost);
+
+ coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo);
+
+ Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ Collection<String> hgComponents = new HashSet<String>();
+ hgComponents.add("OOZIE_SERVER");
+ hgComponents.add("ZOOKEEPER_SERVER");
+ TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName));
+
+ Collection<String> hgComponents2 = new HashSet<String>();
+ hgComponents2.add("OOZIE_SERVER");
+ hgComponents2.add("ZOOKEEPER_SERVER");
+ TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo));
+
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes();
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
+ updater.doUpdateForBlueprintExport();
+
+ assertEquals("oozie property not updated correctly",
+ createExportedHostName(expectedHostGroupName, expectedPortNum), oozieSiteProperties.get("oozie.base.url"));
+ assertEquals("oozie property not updated correctly",
+ createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.authentication.kerberos.principal"));
+ assertEquals("oozie property not updated correctly",
+ createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal"));
+ assertEquals("oozie property not updated correctly",
+ createExportedHostName(expectedHostGroupName), oozieEnvProperties.get("oozie_hostname"));
+ assertEquals("oozie property not updated correctly",
+ createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.oozie.hosts"));
+ assertEquals("oozie property not updated correctly",
+ createExportedAddress("2181", expectedHostGroupName) + "," + createExportedAddress("2181", expectedHostGroupNameTwo), oozieSiteProperties.get("oozie.zookeeper.connection.string"));
+ }
+
+ @Test
public void testYarnHighAvailabilityConfigClusterUpdateSpecifyingHostNamesDirectly() throws Exception {
final String expectedHostName = "c6401.apache.ambari.org";
final String expectedPortNum = "808080";