You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/09/04 20:47:30 UTC
ambari git commit: AMBARI-12970. Fix Tez view to work with RM HA
(Dipayan Bhowmick via srimanth)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 1180d02e1 -> 5f42e387e
AMBARI-12970. Fix Tez view to work with RM HA (Dipayan Bhowmick via srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5f42e387
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5f42e387
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5f42e387
Branch: refs/heads/branch-2.1
Commit: 5f42e387ea8c1d2b9aa022b8e842ba9036c26355
Parents: 1180d02
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Fri Sep 4 11:47:11 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Fri Sep 4 11:47:11 2015 -0700
----------------------------------------------------------------------
contrib/views/tez/pom.xml | 21 ++++++
.../ambari/view/tez/ViewControllerImpl.java | 76 +++-----------------
contrib/views/tez/src/main/resources/view.xml | 2 +-
.../ambari/view/utils/ambari/Services.java | 50 +++++++++++++
.../ambari/view/utils/ambari/ServicesTest.java | 68 ++++++++++++++++++
5 files changed, 148 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f42e387/contrib/views/tez/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/views/tez/pom.xml b/contrib/views/tez/pom.xml
index c7b9776..170fd94 100644
--- a/contrib/views/tez/pom.xml
+++ b/contrib/views/tez/pom.xml
@@ -77,6 +77,16 @@
<version>2.9</version>
<executions>
<execution>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ <execution>
<id>unpack</id>
<phase>process-resources</phase>
<goals>
@@ -137,6 +147,11 @@
<include>view.xml</include>
</includes>
</resource>
+ <resource>
+ <targetPath>WEB-INF/lib</targetPath>
+ <filtering>false</filtering>
+ <directory>target/lib</directory>
+ </resource>
</resources>
</build>
@@ -192,5 +207,11 @@
<version>2.4</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.ambari.contrib.views</groupId>
+ <artifactId>ambari-views-utils</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f42e387/contrib/views/tez/src/main/java/org/apache/ambari/view/tez/ViewControllerImpl.java
----------------------------------------------------------------------
diff --git a/contrib/views/tez/src/main/java/org/apache/ambari/view/tez/ViewControllerImpl.java b/contrib/views/tez/src/main/java/org/apache/ambari/view/tez/ViewControllerImpl.java
index df6b468..ba98ade 100644
--- a/contrib/views/tez/src/main/java/org/apache/ambari/view/tez/ViewControllerImpl.java
+++ b/contrib/views/tez/src/main/java/org/apache/ambari/view/tez/ViewControllerImpl.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.cluster.Cluster;
+import org.apache.ambari.view.utils.ambari.AmbariApi;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
@@ -36,35 +37,13 @@ import com.google.inject.Singleton;
@Singleton
public class ViewControllerImpl implements ViewController {
- private static final Logger LOG = LoggerFactory.getLogger(ViewControllerImpl.class);
+ private AmbariApi ambariApi;
- private final static String TYPE_YARN_SITE = "yarn-site";
- private final String YARN_HTTP_POLICY_PROPERTY = "yarn.http.policy";
- private final static Map<String, String> HTTP_PROPERTY_MAP = new HashMap<String, String>();
- private final static Map<String, String> HTTPS_PROPERTY_MAP = new HashMap<String, String>();
- private final static String YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_PROPERTY =
- "yarn.timeline-service.webapp.address";
- private final static String YARN_RM_WEBAPP_HTTP_ADDRESS_PROPERTY =
- "yarn.resourcemanager.webapp.address";
- private final static String YARN_TIMELINE_WEBAPP_HTTPS_ADDRESS_PROPERTY =
- "yarn.timeline-service.webapp.https.address";
- private final static String YARN_RM_WEBAPP_HTTPS_ADDRESS_PROPERTY =
- "yarn.resourcemanager.webapp.https.address";
- private final static String YARN_HTTPS_ONLY = "HTTPS_ONLY";
-
- static {
- HTTP_PROPERTY_MAP.put(ViewController.PARAM_YARN_ATS_URL,
- YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_PROPERTY);
- HTTP_PROPERTY_MAP.put(ViewController.PARAM_YARN_RESOURCEMANAGER_URL,
- YARN_RM_WEBAPP_HTTP_ADDRESS_PROPERTY);
- HTTPS_PROPERTY_MAP.put(ViewController.PARAM_YARN_ATS_URL,
- YARN_TIMELINE_WEBAPP_HTTPS_ADDRESS_PROPERTY);
- HTTPS_PROPERTY_MAP.put(ViewController.PARAM_YARN_RESOURCEMANAGER_URL,
- YARN_RM_WEBAPP_HTTPS_ADDRESS_PROPERTY);
- }
@Inject
- private ViewContext viewContext;
+ public ViewControllerImpl(ViewContext viewContext) {
+ this.ambariApi = new AmbariApi(viewContext);
+ }
/**
* Because only an admin user is allowed to see the properties in
@@ -77,50 +56,11 @@ public class ViewControllerImpl implements ViewController {
public ViewStatus getViewStatus() {
ViewStatus status = new ViewStatus();
Map<String, String> parameters = new HashMap<String, String>();
- parameters.put(ViewController.PARAM_YARN_ATS_URL, getViewParameterValue(ViewController.PARAM_YARN_ATS_URL));
- parameters.put(ViewController.PARAM_YARN_RESOURCEMANAGER_URL, getViewParameterValue(ViewController.PARAM_YARN_RESOURCEMANAGER_URL));
+ parameters.put(ViewController.PARAM_YARN_ATS_URL, ambariApi.getServices().getTimelineServerUrl());
+ parameters.put(ViewController.PARAM_YARN_RESOURCEMANAGER_URL, ambariApi.getServices().getRMUrl());
status.setParameters(parameters);
return status;
}
- /**
- * @param parameterName Parameter to get the value for
- * @return Returns the value of the given parameter
- */
- private String getViewParameterValue(String parameterName) {
- String value = null;
- Cluster cluster = viewContext.getCluster();
- if (cluster == null) {
- value = viewContext.getProperties().get(parameterName);
- } else {
- if (!parameterName.equals(ViewController.PARAM_YARN_ATS_URL)
- && !parameterName.equals(ViewController.PARAM_YARN_RESOURCEMANAGER_URL)) {
- throw new RuntimeException("Requested configured value for unknown parameter: "
- + parameterName);
- }
-
- String httpPolicy = cluster.getConfigurationValue(TYPE_YARN_SITE, YARN_HTTP_POLICY_PROPERTY);
- if (httpPolicy != null && httpPolicy.equalsIgnoreCase(YARN_HTTPS_ONLY)) {
- String addr = cluster.getConfigurationValue(TYPE_YARN_SITE,
- HTTPS_PROPERTY_MAP.get(parameterName));
- if (!addr.startsWith("https")) {
- value = "https://" + addr;
- }
- } else {
- String addr = cluster.getConfigurationValue(TYPE_YARN_SITE,
- HTTP_PROPERTY_MAP.get(parameterName));
- if (!addr.startsWith("http")) {
- value = "http://" + addr;
- }
- }
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("ViewControllerImpl, paramName=" + parameterName
- + ", value=" + value);
- }
- if ("null".equals(value)) {
- return null;
- }
- return value;
- }
}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f42e387/contrib/views/tez/src/main/resources/view.xml
----------------------------------------------------------------------
diff --git a/contrib/views/tez/src/main/resources/view.xml b/contrib/views/tez/src/main/resources/view.xml
index 91196fa..7d458b3 100644
--- a/contrib/views/tez/src/main/resources/view.xml
+++ b/contrib/views/tez/src/main/resources/view.xml
@@ -30,7 +30,7 @@ limitations under the License. Kerberos, LDAP, Custom. Binary/Htt
</parameter>
<parameter>
<name>yarn.resourcemanager.url</name>
- <description>The URL to the YARN ResourceManager, used to provide YARN Application data. Typically this is the yarn.resourcemanager.webapp.address property in the yarn-site.xml configuration. Url must be accessible from Ambari server.</description>
+ <description>The URL to the YARN ResourceManager, used to provide YARN Application data. If YARN ResourceManager HA is enabled, provide a comma separated list of URLs for all the Resource Managers.</description>
<label>YARN ResourceManager URL</label>
<placeholder>yarn.resourcemanager.hostname:8088</placeholder>
<cluster-config>yarn-site/yarn.resourcemanager.webapp.address</cluster-config>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f42e387/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java
----------------------------------------------------------------------
diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java
index 120e377..421a365 100644
--- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java
+++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java
@@ -45,6 +45,10 @@ public class Services {
private static final String YARN_RESOURCEMANAGER_HA_RM_IDS_KEY = "yarn.resourcemanager.ha.rm-ids";
private static final String YARN_RESOURCEMANAGER_HTTP_HA_PARTIAL_KEY = "yarn.resourcemanager.webapp.address.";
private static final String YARN_RESOURCEMANAGER_HTTPS_HA_PARTIAL_KEY = "yarn.resourcemanager.webapp.https.address.";
+
+ private static final String YARN_ATS_URL = "yarn.timeline-server.url";
+ private final static String YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_KEY = "yarn.timeline-service.webapp.address";
+ private final static String YARN_TIMELINE_WEBAPP_HTTPS_ADDRESS_KEY = "yarn.timeline-service.webapp.https.address";
public static final String RM_INFO_API_ENDPOINT = "/ws/v1/cluster/info";
private final AmbariApi ambariApi;
@@ -238,6 +242,52 @@ public class Services {
return String.format("http://%s:%s/templeton/v1", host, port);
}
+ /**
+ * @return The timeline server url. If the view instance is cluster associated, the value is taken from the
+ * yarn-site.xml else it is retrieved from the view configuration.
+ */
+ public String getTimelineServerUrl() {
+ String url = ambariApi.isClusterAssociated() ? getATSUrlFromCluster() : getATSUrlFromCustom();
+ return removeTrailingSlash(url);
+ }
+
+ private String getATSUrlFromCustom() {
+ String atsUrl = context.getProperties().get(YARN_ATS_URL);
+ if (!StringUtils.isEmpty(atsUrl)) {
+ if (!hasProtocol(atsUrl)) {
+ throw new AmbariApiException(
+ "RA070 View is not cluster associated. Timeline Server URL should contain protocol.");
+ }
+ return atsUrl;
+ } else {
+ throw new AmbariApiException(
+ "RA070 View is not cluster associated. 'YARN Timeline Server URL' should be provided");
+ }
+ }
+
+ private String getATSUrlFromCluster() {
+ String url;
+ String protocol;
+
+ String httpPolicy = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_HTTP_POLICY);
+
+ if (!(HTTP_ONLY.equals(httpPolicy) || HTTPS_ONLY.equals(httpPolicy))) {
+ LOG.error(String.format("RA030 Unknown value %s of yarn-site/yarn.http.policy. HTTP_ONLY assumed.", httpPolicy));
+ httpPolicy = HTTP_ONLY;
+ }
+
+ if (httpPolicy.equals(HTTPS_ONLY)) {
+ protocol = "https";
+ url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_TIMELINE_WEBAPP_HTTPS_ADDRESS_KEY);
+ } else {
+ protocol = "http";
+ url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_KEY);
+ }
+ url = addProtocolIfMissing(url, protocol);
+
+ return url;
+ }
+
public static String addProtocolIfMissing(String url, String protocol) throws AmbariApiException {
if (!hasProtocol(url)) {
url = protocol + "://" + url;
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f42e387/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java
index 1950df8..5d8918d 100644
--- a/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java
+++ b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java
@@ -46,6 +46,72 @@ public class ServicesTest extends EasyMockSupport {
private static final String RM_INFO_API_ENDPOINT = Services.RM_INFO_API_ENDPOINT;
@Test(expected = AmbariApiException.class)
+ public void shouldCheckForEmptyATSUrlInCustomConfig() {
+ ViewContext viewContext = getViewContext(new HashMap<String, String>());
+ AmbariApi ambariApi = createNiceMock(AmbariApi.class);
+ expect(ambariApi.isClusterAssociated()).andReturn(false);
+
+ replay(viewContext);
+
+ Services services = new Services(ambariApi, viewContext);
+ services.getTimelineServerUrl();
+
+ }
+
+ @Test
+ public void shouldReturnATSUrlConfiguredInCustomMode() throws Exception {
+ Map<String, String> map = new HashMap<>();
+ map.put("yarn.timeline-server.url", HTTP_RM_URL1);
+ ViewContext viewContext = getViewContext(map);
+
+ AmbariApi ambariApi = createNiceMock(AmbariApi.class);
+ expect(ambariApi.isClusterAssociated()).andReturn(false);
+
+ replay(viewContext);
+
+ Services services = new Services(ambariApi, viewContext);
+ assertEquals(HTTP_RM_URL1, services.getTimelineServerUrl());
+ }
+
+ @Test(expected = AmbariApiException.class)
+ public void shouldThrowExceptionIfNoProtocolInCustomMode() {
+ Map<String, String> map = new HashMap<>();
+ map.put("yarn.timeline-server.url", RM_URL1_HOST_PORT);
+ ViewContext viewContext = getViewContext(map);
+
+ AmbariApi ambariApi = createNiceMock(AmbariApi.class);
+ expect(ambariApi.isClusterAssociated()).andReturn(false);
+
+ replay(viewContext);
+
+ Services services = new Services(ambariApi, viewContext);
+ services.getTimelineServerUrl();
+ }
+
+
+ @Test
+ public void shouldReturnATSUrlFromYarnSiteInClusteredMode() throws Exception {
+ ViewContext viewContext = getViewContext(new HashMap<String, String>());
+ AmbariApi ambariApi = createNiceMock(AmbariApi.class);
+ Cluster cluster = createNiceMock(Cluster.class);
+ Services services = new Services(ambariApi, viewContext);
+
+ expect(ambariApi.isClusterAssociated()).andReturn(true).anyTimes();
+ setClusterExpectation(cluster, "HTTP_ONLY");
+ expect(ambariApi.getCluster()).andReturn(cluster).anyTimes();
+ replayAll();
+
+ assertEquals(HTTP_RM_URL1, services.getTimelineServerUrl());
+
+ reset(cluster);
+ setClusterExpectation(cluster, "HTTPS_ONLY");
+ replay(cluster);
+
+ assertEquals(HTTPS_RM_URL2, services.getTimelineServerUrl());
+
+ }
+
+ @Test(expected = AmbariApiException.class)
public void shouldCheckForEmptyYarnRMUrlInCustomConfig() {
ViewContext viewContext = getViewContext(new HashMap<String, String>());
AmbariApi ambariApi = createNiceMock(AmbariApi.class);
@@ -261,6 +327,8 @@ public class ServicesTest extends EasyMockSupport {
expect(cluster.getConfigurationValue("yarn-site", "yarn.http.policy")).andReturn(httpPolicy);
expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.webapp.address")).andReturn(RM_URL1_HOST_PORT);
expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.webapp.https.address")).andReturn(RM_URL2_HOST_PORT);
+ expect(cluster.getConfigurationValue("yarn-site", "yarn.timeline-service.webapp.address")).andReturn(RM_URL1_HOST_PORT);
+ expect(cluster.getConfigurationValue("yarn-site", "yarn.timeline-service.webapp.https.address")).andReturn(RM_URL2_HOST_PORT);
}
private void setClusterExpectationInHA(Cluster cluster, String httpPolicy) {