You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by be...@apache.org on 2018/08/08 12:04:25 UTC
[ambari] branch branch-2.7 updated: AMBARI-24199 Atlas HA blueprint
support (benyoka) (#1977)
This is an automated email from the ASF dual-hosted git repository.
benyoka pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new 2955926 AMBARI-24199 Atlas HA blueprint support (benyoka) (#1977)
2955926 is described below
commit 2955926dfbd9b3a1b493ef378b006195e673042f
Author: benyoka <be...@users.noreply.github.com>
AuthorDate: Wed Aug 8 14:04:23 2018 +0200
AMBARI-24199 Atlas HA blueprint support (benyoka) (#1977)
* AMBARI-24199 Atlas HA with blueprint w.i.p. (benyoka)
* AMBARI-24199 Atlas HA with blueprint (benyoka)
---
.../internal/BlueprintConfigurationProcessor.java | 5 +-
.../BlueprintConfigurationProcessorTest.java | 69 ++++++++++++++++++++++
2 files changed, 72 insertions(+), 2 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 4d67235..985d2f0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -2851,7 +2851,6 @@ public class BlueprintConfigurationProcessor {
singleHostTopologyUpdaters.put("hive-env", hiveEnvMap);
singleHostTopologyUpdaters.put("oozie-env", oozieEnvMap);
singleHostTopologyUpdaters.put("kafka-broker", kafkaBrokerMap);
- singleHostTopologyUpdaters.put("application-properties", atlasPropsMap);
singleHostTopologyUpdaters.put("admin-properties", rangerAdminPropsMap);
singleHostTopologyUpdaters.put("ranger-env", rangerEnvPropsMap);
singleHostTopologyUpdaters.put("ranger-yarn-audit", rangerYarnAuditPropsMap);
@@ -2886,6 +2885,7 @@ public class BlueprintConfigurationProcessor {
multiHostTopologyUpdaters.put("oozie-site", multiOozieSiteMap);
multiHostTopologyUpdaters.put("accumulo-site", multiAccumuloSiteMap);
multiHostTopologyUpdaters.put("kms-site", multiRangerKmsSiteMap);
+ multiHostTopologyUpdaters.put("application-properties", atlasPropsMap);
dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap);
@@ -3157,7 +3157,8 @@ public class BlueprintConfigurationProcessor {
multiOozieSiteMap.put("oozie.service.ProxyUserService.proxyuser.knox.hosts", new MultipleHostTopologyUpdater("KNOX_GATEWAY"));
// ATLAS
- atlasPropsMap.put("atlas.server.bind.address", new SingleHostTopologyUpdater("ATLAS_SERVER"));
+ atlasPropsMap.put("atlas.server.bind.address", new MultipleHostTopologyUpdater("ATLAS_SERVER"));
+ atlasPropsMap.put("atlas.rest.address", new MultipleHostTopologyUpdater("ATLAS_SERVER", ',', true, true, true));
atlasPropsMap.put("atlas.kafka.bootstrap.servers", new MultipleHostTopologyUpdater("KAFKA_BROKER"));
atlasPropsMap.put("atlas.kafka.zookeeper.connect", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
atlasPropsMap.put("atlas.graph.index.search.solr.zookeeper-url", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER", ',', false, true, true));
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index afef4c3..c37e659 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -22,6 +22,7 @@ import static com.google.common.collect.Iterators.peekingIterator;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
@@ -647,6 +648,37 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
}
@Test
+ public void testDoUpdateForBlueprintExport_MultiHostProperty__WithPrefixAndPorts() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<>();
+ Map<String, String> typeProps = new HashMap<>();
+ typeProps.put("atlas.server.bind.address",
+ "http://testhost:21000,http://testhost2:21000,http://testhost2a:21000,http://testhost2b:21000");
+ properties.put("application-properties", typeProps);
+
+ Configuration clusterConfig = new Configuration(properties, emptyMap());
+
+ Collection<String> hgComponents = Sets.newHashSet("NAMENODE", "SECONDARY_NAMENODE", "ZOOKEEPER_SERVER");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost"));
+
+ Collection<String> hgComponents2 = Sets.newHashSet("DATANODE", "HDFS_CLIENT", "ZOOKEEPER_SERVER");
+ Set<String> hosts2 = Sets.newHashSet("testhost2", "testhost2a", "testhost2b");
+ TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, hosts2);
+
+ Collection<String> hgComponents3 = Sets.newHashSet("HDFS_CLIENT", "ZOOKEEPER_CLIENT");
+ Set<String> hosts3 = Sets.newHashSet("testhost3", "testhost3a");
+ TestHostGroup group3 = new TestHostGroup("group3", hgComponents3, hosts3);
+
+ Collection<TestHostGroup> hostGroups = Sets.newHashSet(group1, group2, group3);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+ configProcessor.doUpdateForBlueprintExport();
+
+ String updatedVal = properties.get("application-properties").get("atlas.server.bind.address");
+ assertEquals("http://%HOSTGROUP::group1%:21000,http://%HOSTGROUP::group2%:21000", updatedVal);
+ }
+
+ @Test
public void testDoUpdateForBlueprintExport_MultiHostProperty__YAML() throws Exception {
Map<String, Map<String, String>> properties = new HashMap<>();
Map<String, String> typeProps = new HashMap<>();
@@ -2939,6 +2971,43 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
}
@Test
+ public void testMultipleHostTopologyUpdater__hostgroup__multipleHostGroups() throws Exception {
+
+ final String typeName = "application-properties";
+ final String propertyName = "atlas.rest.address";
+ final String originalValue = "http://%HOSTGROUP::group1%:21000,http://%HOSTGROUP::group2%:21000";
+ final String component = "ATLAS_SERVER";
+
+ Map<String, Map<String, String>> properties = new HashMap<>();
+ Map<String, String> typeProps = new HashMap<>();
+ typeProps.put(propertyName, originalValue);
+ properties.put(typeName, typeProps);
+
+ Configuration clusterConfig = new Configuration(properties, emptyMap());
+
+ Set<String> components = ImmutableSet.of(component);
+
+ Set<String> group1Hosts = ImmutableSet.of("testhost1a", "testhost1b", "testhost1c");
+ Set<String> group2Hosts = ImmutableSet.of("testhost2a", "testhost2b", "testhost2c");
+
+ TestHostGroup group1 = new TestHostGroup("group1", components, group1Hosts);
+ TestHostGroup group2 = new TestHostGroup("group2", components, group2Hosts);
+
+ Collection<TestHostGroup> hostGroups = ImmutableSet.of(group1, group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+
+ BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu =
+ new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component, ',', true, true, true);
+ String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology);
+
+ Set<String> expectedAddresses =
+ Sets.union(group1Hosts, group2Hosts).stream().map(host -> "http://" + host + ":21000").collect(toSet());
+ Set<String> replacedAddresses = ImmutableSet.copyOf(newValue.split(","));
+ assertEquals(expectedAddresses, replacedAddresses);
+ }
+
+ @Test
public void testDoUpdateForClusterVerifyRetrySettingsDefault() throws Exception {
Map<String, Map<String, String>> configProperties =
new HashMap<>();