You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2015/03/19 00:33:43 UTC
hadoop git commit: YARN-3351. AppMaster tracking URL is broken in HA.
(Anubhav Dhoot via kasha)
Repository: hadoop
Updated Branches:
refs/heads/trunk 8c40e88d5 -> 20b49224e
YARN-3351. AppMaster tracking URL is broken in HA. (Anubhav Dhoot via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20b49224
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20b49224
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20b49224
Branch: refs/heads/trunk
Commit: 20b49224eb90c796f042ac4251508f3979fd4787
Parents: 8c40e88
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Mar 18 16:30:33 2015 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Mar 18 16:30:33 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../hadoop/yarn/webapp/util/WebAppUtils.java | 31 +++++++-
.../hadoop/yarn/util/TestWebAppUtils.java | 81 ++++++++++++++++++++
.../server/nodemanager/webapp/NavBlock.java | 2 +-
4 files changed, 114 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20b49224/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e0ed568..7322c6b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -78,6 +78,8 @@ Release 2.8.0 - UNRELEASED
YARN-3205 FileSystemRMStateStore should disable FileSystem Cache to avoid
get a Filesystem with an old configuration. (Zhihai Xu via ozawa)
+ YARN-3351. AppMaster tracking URL is broken in HA. (Anubhav Dhoot via kasha)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20b49224/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 3aeb33e..459c110 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -130,11 +130,23 @@ public class WebAppUtils {
return addr;
}
+ public static String getResolvedRemoteRMWebAppURLWithScheme(
+ Configuration conf) {
+ return getHttpSchemePrefix(conf)
+ + getResolvedRemoteRMWebAppURLWithoutScheme(conf);
+ }
+
public static String getResolvedRMWebAppURLWithScheme(Configuration conf) {
return getHttpSchemePrefix(conf)
+ getResolvedRMWebAppURLWithoutScheme(conf);
}
+ public static String getResolvedRemoteRMWebAppURLWithoutScheme(
+ Configuration conf) {
+ return getResolvedRemoteRMWebAppURLWithoutScheme(conf,
+ YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY);
+ }
+
public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf) {
return getResolvedRMWebAppURLWithoutScheme(conf,
YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY);
@@ -143,6 +155,23 @@ public class WebAppUtils {
public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf,
Policy httpPolicy) {
InetSocketAddress address = null;
+ if (httpPolicy == Policy.HTTPS_ONLY) {
+ address =
+ conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
+ } else {
+ address =
+ conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+ }
+ return getResolvedAddress(address);
+ }
+
+ public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf,
+ Policy httpPolicy) {
+ InetSocketAddress address = null;
String rmId = null;
if (HAUtil.isHAEnabled(conf)) {
// If HA enabled, pick one of the RM-IDs and rely on redirect to go to
@@ -167,7 +196,7 @@ public class WebAppUtils {
: HAUtil.addSuffix(
YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
}
return getResolvedAddress(address);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20b49224/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
new file mode 100644
index 0000000..169787f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestWebAppUtils {
+
+ private static final String RM1_NODE_ID = "rm1";
+ private static final String RM2_NODE_ID = "rm2";
+
+ // Because WebAppUtils#getResolvedAddress tries to resolve the hostname, we add a static mapping for dummy hostnames
+ // to make this test run anywhere without having to give some resolvable hostnames
+ private static String dummyHostNames[] = {"host1", "host2", "host3"};
+ private static final String anyIpAddress = "1.2.3.4";
+ private static Map<String, String> savedStaticResolution = new HashMap<>();
+
+ @BeforeClass
+ public static void initializeDummyHostnameResolution() throws Exception {
+ String previousIpAddress;
+ for (String hostName : dummyHostNames) {
+ if (null != (previousIpAddress = NetUtils.getStaticResolution(hostName))) {
+ savedStaticResolution.put(hostName, previousIpAddress);
+ }
+ NetUtils.addStaticResolution(hostName, anyIpAddress);
+ }
+ }
+
+ @AfterClass
+ public static void restoreDummyHostnameResolution() throws Exception {
+ for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution.entrySet()) {
+ NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), hostnameToIpEntry.getValue());
+ }
+ }
+
+ @Test
+ public void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
+ Configuration configuration = new Configuration();
+ final String rmAddress = "host1:8088";
+ configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, rmAddress);
+ final String rm1Address = "host2:8088";
+ final String rm2Address = "host3:8088";
+ configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID, rm1Address);
+ configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address);
+ configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
+
+ String rmRemoteUrl = WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(configuration);
+ Assert.assertEquals("ResolvedRemoteRMWebAppUrl should resolve to the first HA RM address", rm1Address, rmRemoteUrl);
+
+ String rmLocalUrl = WebAppUtils.getResolvedRMWebAppURLWithoutScheme(configuration);
+ Assert.assertEquals("ResolvedRMWebAppUrl should resolve to the default RM webapp address", rmAddress, rmLocalUrl);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20b49224/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
index 1c3b63b..b8a10f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
@@ -38,7 +38,7 @@ public class NavBlock extends HtmlBlock implements YarnWebParams {
protected void render(Block html) {
String RMWebAppURL =
- WebAppUtils.getResolvedRMWebAppURLWithScheme(this.conf);
+ WebAppUtils.getResolvedRemoteRMWebAppURLWithScheme(this.conf);
html
.div("#nav")
.h3()._("ResourceManager")._()