You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by pz...@apache.org on 2018/01/03 20:54:59 UTC
knox git commit: KNOX-1043
Repository: knox
Updated Branches:
refs/heads/master a438bcc1e -> 348c4d7de
KNOX-1043
Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/348c4d7d
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/348c4d7d
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/348c4d7d
Branch: refs/heads/master
Commit: 348c4d7def61f6296a7cf192ed00e38a8e5c3714
Parents: a438bcc
Author: Phil Zampino <pz...@gmail.com>
Authored: Thu Dec 21 11:49:51 2017 -0500
Committer: Phil Zampino <pz...@apache.org>
Committed: Wed Jan 3 15:39:36 2018 -0500
----------------------------------------------------------------------
.../discovery/ambari/AmbariCluster.java | 6 +-
.../ambari/AmbariDynamicServiceURLCreator.java | 4 +-
.../discovery/ambari/PropertyEqualsHandler.java | 20 +++-
.../discovery/ambari/ServiceURLCreator.java | 32 +++++
.../discovery/ambari/ServiceURLFactory.java | 75 ++++++++++++
.../ambari/ServiceURLPropertyConfig.java | 7 +-
.../discovery/ambari/WebHdfsUrlCreator.java | 84 ++++++++++++++
.../ambari-service-discovery-url-mappings.xml | 24 ++--
.../AmbariDynamicServiceURLCreatorTest.java | 116 +++++++++++++------
9 files changed, 311 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/knox/blob/348c4d7d/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
index 1d308cc..2dff181 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
@@ -27,7 +27,7 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
private String name = null;
- private AmbariDynamicServiceURLCreator urlCreator;
+ private ServiceURLFactory urlFactory;
private Map<String, Map<String, ServiceConfiguration>> serviceConfigurations = new HashMap<>();
@@ -37,7 +37,7 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
AmbariCluster(String name) {
this.name = name;
components = new HashMap<>();
- urlCreator = new AmbariDynamicServiceURLCreator(this);
+ urlFactory = ServiceURLFactory.newInstance(this);
}
void addServiceConfiguration(String serviceName, String configurationType, ServiceConfiguration serviceConfig) {
@@ -87,7 +87,7 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
@Override
public List<String> getServiceURLs(String serviceName) {
List<String> urls = new ArrayList<>();
- urls.addAll(urlCreator.create(serviceName));
+ urls.addAll(urlFactory.create(serviceName));
return urls;
}
http://git-wip-us.apache.org/repos/asf/knox/blob/348c4d7d/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
index ed5d3e7..c35ed66 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
@@ -28,7 +28,7 @@ import java.util.List;
import java.util.Map;
-class AmbariDynamicServiceURLCreator {
+class AmbariDynamicServiceURLCreator implements ServiceURLCreator {
static final String MAPPING_CONFIG_OVERRIDE_PROPERTY = "org.apache.gateway.topology.discovery.ambari.config";
@@ -69,7 +69,7 @@ class AmbariDynamicServiceURLCreator {
config = new ServiceURLPropertyConfig(new ByteArrayInputStream(mappings.getBytes()));
}
- List<String> create(String serviceName) {
+ public List<String> create(String serviceName) {
List<String> urls = new ArrayList<>();
Map<String, String> placeholderValues = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/knox/blob/348c4d7d/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/PropertyEqualsHandler.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/PropertyEqualsHandler.java b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/PropertyEqualsHandler.java
index 642a676..e5f1c68 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/PropertyEqualsHandler.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/PropertyEqualsHandler.java
@@ -44,10 +44,22 @@ class PropertyEqualsHandler implements ConditionalValueHandler {
ServiceURLPropertyConfig.Property p = config.getConfigProperty(serviceName, propertyName);
if (p != null) {
String value = getActualPropertyValue(cluster, p);
- if (propertyValue.equals(value)) {
- result = affirmativeResult.evaluate(config, cluster);
- } else if (negativeResult != null) {
- result = negativeResult.evaluate(config, cluster);
+ if (propertyValue == null) {
+ // If the property value isn't specified, then we're just checking if the property is set with any value
+ if (value != null) {
+ // So, if there is a value in the config, respond with the affirmative
+ result = affirmativeResult.evaluate(config, cluster);
+ } else if (negativeResult != null) {
+ result = negativeResult.evaluate(config, cluster);
+ }
+ }
+
+ if (result == null) {
+ if (propertyValue.equals(value)) {
+ result = affirmativeResult.evaluate(config, cluster);
+ } else if (negativeResult != null) {
+ result = negativeResult.evaluate(config, cluster);
+ }
}
// Check if the result is a reference to a local derived property
http://git-wip-us.apache.org/repos/asf/knox/blob/348c4d7d/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLCreator.java
new file mode 100644
index 0000000..8295155
--- /dev/null
+++ b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLCreator.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import java.util.List;
+
+public interface ServiceURLCreator {
+
+ /**
+ * Creates one or more cluster-specific URLs for the specified service.
+ *
+ * @param service The service identifier.
+ *
+ * @return A List of created URL strings; the list may be empty.
+ */
+ List<String> create(String service);
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/348c4d7d/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLFactory.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLFactory.java b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLFactory.java
new file mode 100644
index 0000000..fa9f89a
--- /dev/null
+++ b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLFactory.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Factory for creating cluster-specific service URLs.
+ */
+public class ServiceURLFactory {
+
+ private Map<String, ServiceURLCreator> urlCreators = new HashMap<>();
+
+ private ServiceURLCreator defaultURLCreator = null;
+
+
+ private ServiceURLFactory(AmbariCluster cluster) {
+ // Default URL creator
+ defaultURLCreator = new AmbariDynamicServiceURLCreator(cluster);
+
+ // Custom (internal) URL creators
+ urlCreators.put("WEBHDFS", new WebHdfsUrlCreator(cluster));
+ }
+
+
+ /**
+ * Create a new factory for the specified cluster.
+ *
+ * @param cluster The cluster.
+ *
+ * @return A ServiceURLFactory instance.
+ */
+ public static ServiceURLFactory newInstance(AmbariCluster cluster) {
+ return new ServiceURLFactory(cluster);
+ }
+
+
+ /**
+ * Create one or more cluster-specific URLs for the specified service.
+ *
+ * @param service The service.
+ *
+ * @return A List of service URL strings; the list may be empty.
+ */
+ public List<String> create(String service) {
+ List<String> urls = new ArrayList<>();
+
+ ServiceURLCreator creator = urlCreators.get(service);
+ if (creator == null) {
+ creator = defaultURLCreator;
+ }
+
+ urls.addAll(creator.create(service));
+
+ return urls;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/348c4d7d/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLPropertyConfig.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLPropertyConfig.java b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLPropertyConfig.java
index deb5bb3..d4be904 100644
--- a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLPropertyConfig.java
+++ b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/ServiceURLPropertyConfig.java
@@ -277,7 +277,12 @@ class ServiceURLPropertyConfig {
if (ifNode != null) {
NamedNodeMap attrs = ifNode.getAttributes();
String comparisonPropName = attrs.getNamedItem(ATTR_PROPERTY).getNodeValue();
- String comparisonValue = attrs.getNamedItem(ATTR_VALUE).getNodeValue();
+
+ String comparisonValue = null;
+ Node valueNode = attrs.getNamedItem(ATTR_VALUE);
+ if (valueNode != null) {
+ comparisonValue = attrs.getNamedItem(ATTR_VALUE).getNodeValue();
+ }
ConditionalValueHandler affirmativeResult = null;
Node thenNode = (Node) THEN.evaluate(ifNode, XPathConstants.NODE);
http://git-wip-us.apache.org/repos/asf/knox/blob/348c4d7d/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java
new file mode 100644
index 0000000..1d11c66
--- /dev/null
+++ b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A ServiceURLCreator implementation for WEBHDFS.
+ */
+public class WebHdfsUrlCreator implements ServiceURLCreator {
+
+ private static final String SERVICE = "WEBHDFS";
+
+ private AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class);
+
+ private AmbariCluster cluster = null;
+
+ WebHdfsUrlCreator(AmbariCluster cluster) {
+ this.cluster = cluster;
+ }
+
+ @Override
+ public List<String> create(String service) {
+ List<String> urls = new ArrayList<>();
+
+ if (SERVICE.equals(service)) {
+ AmbariCluster.ServiceConfiguration sc = cluster.getServiceConfiguration("HDFS", "hdfs-site");
+
+ // First, check if it's HA config
+ String nameServices = null;
+ AmbariComponent nameNodeComp = cluster.getComponent("NAMENODE");
+ if (nameNodeComp != null) {
+ nameServices = nameNodeComp.getConfigProperty("dfs.nameservices");
+ }
+
+ if (nameServices != null && !nameServices.isEmpty()) {
+ // If it is an HA configuration
+ Map<String, String> props = sc.getProperties();
+
+ // Name node HTTP addresses are defined as properties of the form:
+ // dfs.namenode.http-address.<NAMESERVICES>.nn<INDEX>
+ // So, this iterates over the nn<INDEX> properties until there is no such property (since it cannot be known how
+ // many are defined by any other means).
+ int i = 1;
+ String propertyValue = getHANameNodeHttpAddress(props, nameServices, i++);
+ while (propertyValue != null) {
+ urls.add(createURL(propertyValue));
+ propertyValue = getHANameNodeHttpAddress(props, nameServices, i++);
+ }
+ } else { // If it's not an HA configuration, get the single name node HTTP address
+ urls.add(createURL(sc.getProperties().get("dfs.namenode.http-address")));
+ }
+ }
+
+ return urls;
+ }
+
+ private static String getHANameNodeHttpAddress(Map<String, String> props, String nameServices, int index) {
+ return props.get("dfs.namenode.http-address." + nameServices + ".nn" + index);
+ }
+
+ private static String createURL(String address) {
+ return "http://" + address + "/webhdfs";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/348c4d7d/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
index 8953b8d..23fbf9c 100644
--- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
+++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
@@ -24,12 +24,24 @@
<service-discovery-url-mappings>
<service name="NAMENODE">
- <url-pattern>hdfs://{DFS_NAMENODE_RPC_ADDRESS}</url-pattern>
+ <url-pattern>hdfs://{DFS_NAMENODE_ADDRESS}</url-pattern>
<properties>
<property name="DFS_NAMENODE_RPC_ADDRESS">
<component>NAMENODE</component>
<config-property>dfs.namenode.rpc-address</config-property>
</property>
+ <property name="DFS_NAMESERVICES">
+ <component>NAMENODE</component>
+ <config-property>dfs.nameservices</config-property>
+ </property>
+ <property name="DFS_NAMENODE_ADDRESS">
+ <config-property>
+ <if property="DFS_NAMESERVICES">
+ <then>DFS_NAMESERVICES</then>
+ <else>DFS_NAMENODE_RPC_ADDRESS</else>
+ </if>
+ </config-property>
+ </property>
</properties>
</service>
@@ -43,16 +55,6 @@
</properties>
</service>
- <service name="WEBHDFS">
- <url-pattern>http://{WEBHDFS_ADDRESS}/webhdfs</url-pattern>
- <properties>
- <property name="WEBHDFS_ADDRESS">
- <service-config name="HDFS">hdfs-site</service-config>
- <config-property>dfs.namenode.http-address</config-property>
- </property>
- </properties>
- </service>
-
<service name="WEBHCAT">
<url-pattern>http://{HOST}:{PORT}/templeton</url-pattern>
<properties>
http://git-wip-us.apache.org/repos/asf/knox/blob/348c4d7d/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
index dd35dbb..63043d3 100644
--- a/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
+++ b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
@@ -34,6 +34,7 @@ import java.util.Map;
import static junit.framework.TestCase.assertTrue;
import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -112,6 +113,7 @@ public class AmbariDynamicServiceURLCreatorTest {
validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, HTTP_PATH);
}
+
@Test
public void testResourceManagerURLFromInternalMapping() throws Exception {
testResourceManagerURL(null);
@@ -213,6 +215,35 @@ public class AmbariDynamicServiceURLCreatorTest {
assertEquals("hdfs://" + ADDRESS, url);
}
+
+ @Test
+ public void testNameNodeHAURLFromInternalMapping() throws Exception {
+ testNameNodeURLHA(null);
+ }
+
+ @Test
+ public void testNameNodeHAURLFromExternalMapping() throws Exception {
+ testNameNodeURLHA(TEST_MAPPING_CONFIG);
+ }
+
+ private void testNameNodeURLHA(Object mappingConfiguration) throws Exception {
+ final String NAMESERVICE = "myNSCluster";
+
+ AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
+ EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICE).anyTimes();
+ EasyMock.replay(namenode);
+
+ AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+ EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
+ EasyMock.replay(cluster);
+
+ // Run the test
+ AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration);
+ String url = builder.create("NAMENODE").get(0);
+ assertEquals("hdfs://" + NAMESERVICE, url);
+ }
+
+
@Test
public void testWebHCatURLFromInternalMapping() throws Exception {
testWebHCatURL(null);
@@ -309,29 +340,6 @@ public class AmbariDynamicServiceURLCreatorTest {
testWebHdfsURL(TEST_MAPPING_CONFIG);
}
- @Test
- public void testWebHdfsURLFromSystemPropertyOverride() throws Exception {
- // Write the test mapping configuration to a temp file
- File mappingFile = File.createTempFile("mapping-config", "xml");
- FileUtils.write(mappingFile, OVERRIDE_MAPPING_FILE_CONTENTS, "utf-8");
-
- // Set the system property to point to the temp file
- System.setProperty(AmbariDynamicServiceURLCreator.MAPPING_CONFIG_OVERRIDE_PROPERTY,
- mappingFile.getAbsolutePath());
- try {
- final String ADDRESS = "host3:1357";
- // The URL creator should apply the file contents, and create the URL accordingly
- String url = getTestWebHdfsURL(ADDRESS, null);
-
- // Verify the URL matches the pattern from the file
- assertEquals("http://" + ADDRESS + "/webhdfs/OVERRIDE", url);
- } finally {
- // Reset the system property, and delete the temp file
- System.clearProperty(AmbariDynamicServiceURLCreator.MAPPING_CONFIG_OVERRIDE_PROPERTY);
- mappingFile.delete();
- }
- }
-
private void testWebHdfsURL(Object mappingConfiguration) throws Exception {
final String ADDRESS = "host3:1357";
assertEquals("http://" + ADDRESS + "/webhdfs", getTestWebHdfsURL(ADDRESS, mappingConfiguration));
@@ -350,8 +358,42 @@ public class AmbariDynamicServiceURLCreatorTest {
EasyMock.replay(cluster);
// Create the URL
- AmbariDynamicServiceURLCreator creator = newURLCreator(cluster, mappingConfiguration);
- return creator.create("WEBHDFS").get(0);
+ List<String> urls = ServiceURLFactory.newInstance(cluster).create("WEBHDFS");
+ assertNotNull(urls);
+ assertFalse(urls.isEmpty());
+ return urls.get(0);
+ }
+
+ @Test
+ public void testWebHdfsURLHA() throws Exception {
+ final String NAMESERVICES = "myNameServicesCluster";
+ final String HTTP_ADDRESS_1 = "host1:50070";
+ final String HTTP_ADDRESS_2 = "host2:50077";
+
+ final String EXPECTED_ADDR_1 = "http://" + HTTP_ADDRESS_1 + "/webhdfs";
+ final String EXPECTED_ADDR_2 = "http://" + HTTP_ADDRESS_2 + "/webhdfs";
+
+ AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class);
+ EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
+ EasyMock.replay(namenode);
+
+ AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+ Map<String, String> hdfsProps = new HashMap<>();
+ hdfsProps.put("dfs.namenode.http-address." + NAMESERVICES + ".nn1", HTTP_ADDRESS_1);
+ hdfsProps.put("dfs.namenode.http-address." + NAMESERVICES + ".nn2", HTTP_ADDRESS_2);
+ EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
+ EasyMock.replay(hdfsSC);
+
+ AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+ EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
+ EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes();
+ EasyMock.replay(cluster);
+
+ // Create the URL
+ List<String> webhdfsURLs = ServiceURLFactory.newInstance(cluster).create("WEBHDFS");
+ assertEquals(2, webhdfsURLs.size());
+ assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1));
+ assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2));
}
@@ -731,12 +773,24 @@ public class AmbariDynamicServiceURLCreatorTest {
"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
"<service-discovery-url-mappings>\n" +
" <service name=\"NAMENODE\">\n" +
- " <url-pattern>hdfs://{DFS_NAMENODE_RPC_ADDRESS}</url-pattern>\n" +
+ " <url-pattern>hdfs://{DFS_NAMENODE_ADDRESS}</url-pattern>\n" +
" <properties>\n" +
" <property name=\"DFS_NAMENODE_RPC_ADDRESS\">\n" +
" <component>NAMENODE</component>\n" +
" <config-property>dfs.namenode.rpc-address</config-property>\n" +
" </property>\n" +
+ " <property name=\"DFS_NAMESERVICES\">\n" +
+ " <component>NAMENODE</component>\n" +
+ " <config-property>dfs.nameservices</config-property>\n" +
+ " </property>\n" +
+ " <property name=\"DFS_NAMENODE_ADDRESS\">\n" +
+ " <config-property>\n" +
+ " <if property=\"DFS_NAMESERVICES\">\n" +
+ " <then>DFS_NAMESERVICES</then>\n" +
+ " <else>DFS_NAMENODE_RPC_ADDRESS</else>\n" +
+ " </if>\n" +
+ " </config-property>\n" +
+ " </property>\n" +
" </properties>\n" +
" </service>\n" +
"\n" +
@@ -750,16 +804,6 @@ public class AmbariDynamicServiceURLCreatorTest {
" </properties>\n" +
" </service>\n" +
"\n" +
- " <service name=\"WEBHDFS\">\n" +
- " <url-pattern>http://{WEBHDFS_ADDRESS}/webhdfs</url-pattern>\n" +
- " <properties>\n" +
- " <property name=\"WEBHDFS_ADDRESS\">\n" +
- " <service-config name=\"HDFS\">hdfs-site</service-config>\n" +
- " <config-property>dfs.namenode.http-address</config-property>\n" +
- " </property>\n" +
- " </properties>\n" +
- " </service>\n" +
- "\n" +
" <service name=\"WEBHCAT\">\n" +
" <url-pattern>http://{HOST}:{PORT}/templeton</url-pattern>\n" +
" <properties>\n" +