You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2014/09/26 18:48:58 UTC
[43/44] git commit: Remove Oozie external DB properties from
Blueprint Export
Remove Oozie external DB properties from Blueprint Export
This patch resolves bug AMBARI-7507.
Some Oozie properties related to using an external MySQL
Database (oozie_existing_mysql_host, oozie.service.JPAService.jdbc.url)
are currently appearing in an exported Blueprint with
hostname information included. This is incorrect, since
the hostname information should be completely removed or
masked in an exported Blueprint. Having a reference to a
given hostname in this scenario is not very useful, since the
Blueprint may be applied to a completely different set of
machines.
These Oozie properties generally will only be set to
either a default, in-memory DB, or to an external
MYSQL server instance. This means that the normal
PropertyUpdater processing won't really apply to
these two properties.
This patch fixes this problem by creating a new
PropertyUpdater type, which is only responsible
for tracking these two property types, and marking
them as properties that will not be included
in an exported Blueprint.
This patch also adds a processing method to
handle any properties that use this new
PropertyUpdater, and this processing method
will remove these registered properties
during the export process.
This patch also adds a unit test to verify this change.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/20f112ac
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/20f112ac
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/20f112ac
Branch: refs/heads/branch-alerts-dev
Commit: 20f112acd619f58aaad85a5fa65ff82c2e4cc498
Parents: 2e3082a
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Thu Sep 25 21:37:42 2014 -0400
Committer: John Speidel <js...@hortonworks.com>
Committed: Fri Sep 26 11:42:07 2014 -0400
----------------------------------------------------------------------
.../BlueprintConfigurationProcessor.java | 60 ++++++++++++++++++++
.../BlueprintConfigurationProcessorTest.java | 9 +++
2 files changed, 69 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/20f112ac/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 3e1bcf0..d33ae65 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -61,6 +61,15 @@ public class BlueprintConfigurationProcessor {
new HashMap<String, Map<String, PropertyUpdater>>();
/**
+ * Updaters that preserve the original property value, functions
+ * as a placeholder for DB-related properties that need to be
+ * removed from export, but do not require an update during
+ * cluster creation
+ */
+ private static Map<String, Map<String, PropertyUpdater>> removePropertyUpdaters =
+ new HashMap<String, Map<String, PropertyUpdater>>();
+
+ /**
* Collection of all updaters
*/
private static Collection<Map<String, Map<String, PropertyUpdater>>> allUpdaters =
@@ -181,10 +190,37 @@ public class BlueprintConfigurationProcessor {
doMultiHostExportUpdate(hostGroups, multiHostTopologyUpdaters);
+ doRemovePropertyExport(removePropertyUpdaters);
+
return properties;
}
/**
+ * Performs export update for the set of properties that do not
+ * require update during cluster setup, but should be removed
+ * during a Blueprint export.
+ *
+ * In the case of a service referring to an external DB, any
+ * properties that contain external host information should
+ * be removed from the configuration that will be available in
+ * the exported Blueprint.
+ *
+ * @param updaters set of updaters for properties that should
+ * always be removed during a Blueprint export
+ */
+ private void doRemovePropertyExport(Map<String, Map<String, PropertyUpdater>> updaters) {
+ for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
+ String type = entry.getKey();
+ for (String propertyName : entry.getValue().keySet()) {
+ Map<String, String> typeProperties = properties.get(type);
+ if ( (typeProperties != null) && (typeProperties.containsKey(propertyName)) ) {
+ typeProperties.remove(propertyName);
+ }
+ }
+ }
+ }
+
+ /**
* Perform export update processing for HA configuration for NameNodes. The HA NameNode property
* names are based on the nameservices defined when HA is enabled via the Ambari UI, so this method
* dynamically determines the property names, and registers PropertyUpdaters to handle the masking of
@@ -768,6 +804,21 @@ public class BlueprintConfigurationProcessor {
}
/**
+ * PropertyUpdater implementation that will always return the original
+ * value for the updateForClusterCreate() method.
+ * This updater type should only be used in cases where a given
+ * property requires no updates, but may need to be considered
+ * during the Blueprint export process.
+ */
+ private static class OriginalValuePropertyUpdater implements PropertyUpdater {
+ @Override
+ public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String origValue, Map<String, Map<String, String>> properties) {
+ // always return the original value, since these properties do not require update handling
+ return origValue;
+ }
+ }
+
+ /**
* Register updaters for configuration properties.
*/
static {
@@ -783,6 +834,7 @@ public class BlueprintConfigurationProcessor {
Map<String, PropertyUpdater> hbaseSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> yarnSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> hiveSiteMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> oozieSiteOriginalValueMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> oozieSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> stormSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> falconStartupPropertiesMap = new HashMap<String, PropertyUpdater>();
@@ -793,6 +845,8 @@ public class BlueprintConfigurationProcessor {
Map<String, PropertyUpdater> hbaseEnvMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> hiveEnvMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> oozieEnvMap = new HashMap<String, PropertyUpdater>();
+ Map<String, PropertyUpdater> oozieEnvOriginalValueMap = new HashMap<String, PropertyUpdater>();
+
Map<String, PropertyUpdater> multiWebhcatSiteMap = new HashMap<String, PropertyUpdater>();
Map<String, PropertyUpdater> multiHbaseSiteMap = new HashMap<String, PropertyUpdater>();
@@ -830,6 +884,9 @@ public class BlueprintConfigurationProcessor {
dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap);
+ removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap);
+ removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap);
+
// NAMENODE
hdfsSiteMap.put("dfs.http.address", new SingleHostTopologyUpdater("NAMENODE"));
hdfsSiteMap.put("dfs.https.address", new SingleHostTopologyUpdater("NAMENODE"));
@@ -886,6 +943,9 @@ public class BlueprintConfigurationProcessor {
oozieSiteMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER"));
oozieEnvMap.put("oozie_hostname", new SingleHostTopologyUpdater("OOZIE_SERVER"));
multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER"));
+ // register updaters for Oozie properties that may point to an external DB
+ oozieEnvOriginalValueMap.put("oozie_existing_mysql_host", new OriginalValuePropertyUpdater());
+ oozieSiteOriginalValueMap.put("oozie.service.JPAService.jdbc.url", new OriginalValuePropertyUpdater());
// ZOOKEEPER_SERVER
multiHbaseSiteMap.put("hbase.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/20f112ac/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 590b9c2..90f83bc 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -1685,6 +1685,7 @@ public class BlueprintConfigurationProcessorTest {
public void testOozieConfigExported() throws Exception {
final String expectedHostName = "c6401.apache.ambari.org";
final String expectedHostNameTwo = "c6402.ambari.apache.org";
+ final String expectedExternalHost = "c6408.ambari.apache.org";
final String expectedHostGroupName = "host_group_1";
final String expectedHostGroupNameTwo = "host_group_2";
@@ -1718,8 +1719,10 @@ public class BlueprintConfigurationProcessorTest {
oozieSiteProperties.put("oozie.base.url", expectedHostName);
oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName);
oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName);
+ oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + expectedExternalHost + "/ooziedb");
oozieEnvProperties.put("oozie_hostname", expectedHostName);
+ oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost);
coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo);
@@ -1740,6 +1743,12 @@ public class BlueprintConfigurationProcessorTest {
assertEquals("oozie property not exported correctly",
createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.oozie.hosts"));
+ // verify that the oozie properties that can refer to an external DB are not included in the export
+ assertFalse("oozie_existing_mysql_host should not have been present in the exported configuration",
+ oozieEnvProperties.containsKey("oozie_existing_mysql_host"));
+ assertFalse("oozie.service.JPAService.jdbc.url should not have been present in the exported configuration",
+ oozieSiteProperties.containsKey("oozie.service.JPAService.jdbc.url"));
+
mockSupport.verifyAll();
}