You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/09/04 23:42:11 UTC
[06/50] [abbrv] hadoop git commit: YARN-4092. Fixed UI redirection to
print useful messages when both RMs are in standby mode. Contributed by Xuan
Gong
YARN-4092. Fixed UI redirection to print useful messages when both RMs are in standby mode. Contributed by Xuan Gong
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3fd2ccc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3fd2ccc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3fd2ccc
Branch: refs/heads/YARN-1197
Commit: a3fd2ccc869dfc1f04d1cf0a8678d4d90a43a80f
Parents: 826ae1c
Author: Jian He <ji...@apache.org>
Authored: Mon Aug 31 17:33:24 2015 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Aug 31 17:33:24 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/client/TestRMFailover.java | 27 ++++++
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../resourcemanager/webapp/RMWebAppFilter.java | 90 +++++++++++++++++++-
4 files changed, 117 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3fd2ccc/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4201b4f..19c1082 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -422,6 +422,9 @@ Release 2.8.0 - UNRELEASED
YARN-1556. NPE getting application report with a null appId. (Weiwei Yang via
junping_du)
+ YARN-4092. Fixed UI redirection to print useful messages when both RMs are
+ in standby mode. (Xuan Gong via jianhe)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3fd2ccc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index 0d03fd4..cbc220a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -27,6 +27,7 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
+
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
@@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -265,6 +267,7 @@ public class TestRMFailover extends ClientBaseWithFixes {
getAdminService(0).transitionToActive(req);
String rm1Url = "http://0.0.0.0:18088";
String rm2Url = "http://0.0.0.0:28088";
+
String redirectURL = getRedirectURL(rm2Url);
// if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
assertEquals(redirectURL,rm1Url+"/");
@@ -304,6 +307,17 @@ public class TestRMFailover extends ClientBaseWithFixes {
redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
assertNull(redirectURL);
+
+ // transit the active RM to standby
+ // Both of RMs are in standby mode
+ getAdminService(0).transitionToStandby(req);
+ // RM2 is expected to send the httpRequest to itself.
+ // The Header Field: Refresh is expected to be set.
+ redirectURL = getRefreshURL(rm2Url);
+ assertTrue(redirectURL != null
+ && redirectURL.contains(YarnWebParams.NEXT_REFRESH_INTERVAL)
+ && redirectURL.contains(rm2Url));
+
}
// set up http connection with the given url and get the redirection url from the response
@@ -323,4 +337,17 @@ public class TestRMFailover extends ClientBaseWithFixes {
return redirectUrl;
}
+ static String getRefreshURL(String url) {
+ String redirectUrl = null;
+ try {
+ HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
+ // do not automatically follow the redirection
+ // otherwise we get too many redirections exception
+ conn.setInstanceFollowRedirects(false);
+ redirectUrl = conn.getHeaderField("Refresh");
+ } catch (Exception e) {
+ // throw new RuntimeException(e);
+ }
+ return redirectUrl;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3fd2ccc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 679e1d6..3792649 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -40,4 +40,5 @@ public interface YarnWebParams {
String NODE_STATE = "node.state";
String NODE_LABEL = "node.label";
String WEB_UI_TYPE = "web.ui.type";
+ String NEXT_REFRESH_INTERVAL = "next.fresh.interval";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3fd2ccc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
index 500f17a..a8f793a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
import java.io.PrintWriter;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Random;
import java.util.Set;
import javax.inject.Inject;
@@ -29,8 +33,11 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HtmlQuoting;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
import com.google.common.collect.Sets;
import com.google.inject.Injector;
@@ -48,11 +55,26 @@ public class RMWebAppFilter extends GuiceContainer {
// define a set of URIs which do not need to do redirection
private static final Set<String> NON_REDIRECTED_URIS = Sets.newHashSet(
"/conf", "/stacks", "/logLevel", "/logs");
+ private String path;
+ private static final int BASIC_SLEEP_TIME = 5;
+ private static final int MAX_SLEEP_TIME = 5 * 60;
@Inject
- public RMWebAppFilter(Injector injector) {
+ public RMWebAppFilter(Injector injector, Configuration conf) {
super(injector);
this.injector=injector;
+ InetSocketAddress sock = YarnConfiguration.useHttps(conf)
+ ? conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT)
+ : conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+
+ path = sock.getHostName() + ":" + Integer.toString(sock.getPort());
+ path = YarnConfiguration.useHttps(conf)
+ ? "https://" + path
+ : "http://" + path;
}
@Override
@@ -69,9 +91,11 @@ public class RMWebAppFilter extends GuiceContainer {
rmWebApp.checkIfStandbyRM();
if (rmWebApp.isStandby()
&& shouldRedirect(rmWebApp, uri)) {
- String redirectPath = rmWebApp.getRedirectPath() + uri;
+
+ String redirectPath = rmWebApp.getRedirectPath();
if (redirectPath != null && !redirectPath.isEmpty()) {
+ redirectPath += uri;
String redirectMsg =
"This is standby RM. The redirect url is: " + redirectPath;
PrintWriter out = response.getWriter();
@@ -79,11 +103,40 @@ public class RMWebAppFilter extends GuiceContainer {
response.setHeader("Location", redirectPath);
response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
return;
+ } else {
+ boolean doRetry = true;
+ String retryIntervalStr =
+ request.getParameter(YarnWebParams.NEXT_REFRESH_INTERVAL);
+ int retryInterval = 0;
+ if (retryIntervalStr != null) {
+ try {
+ retryInterval = Integer.parseInt(retryIntervalStr.trim());
+ } catch (NumberFormatException ex) {
+ doRetry = false;
+ }
+ }
+ int next = calculateExponentialTime(retryInterval);
+
+ String redirectUrl =
+ appendOrReplaceParamter(path + uri,
+ YarnWebParams.NEXT_REFRESH_INTERVAL + "=" + (retryInterval + 1));
+ if (redirectUrl == null || next > MAX_SLEEP_TIME) {
+ doRetry = false;
+ }
+ String redirectMsg =
+ doRetry ? "Can not find any active RM. Will retry in next " + next
+ + " seconds." : "There is no active RM right now.";
+ PrintWriter out = response.getWriter();
+ out.println(redirectMsg);
+ if (doRetry) {
+ response.setHeader("Refresh", next + ";url=" + redirectUrl);
+ response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+ }
}
+ return;
}
super.doFilter(request, response, chain);
-
}
private boolean shouldRedirect(RMWebApp rmWebApp, String uri) {
@@ -92,4 +145,33 @@ public class RMWebAppFilter extends GuiceContainer {
&& !uri.startsWith(ProxyUriUtils.PROXY_BASE)
&& !NON_REDIRECTED_URIS.contains(uri);
}
-}
+
+ private String appendOrReplaceParamter(String uri, String newQuery) {
+ if (uri.contains(YarnWebParams.NEXT_REFRESH_INTERVAL + "=")) {
+ return uri.replaceAll(YarnWebParams.NEXT_REFRESH_INTERVAL + "=[^&]+",
+ newQuery);
+ }
+ try {
+ URI oldUri = new URI(uri);
+ String appendQuery = oldUri.getQuery();
+ if (appendQuery == null) {
+ appendQuery = newQuery;
+ } else {
+ appendQuery += "&" + newQuery;
+ }
+
+ URI newUri =
+ new URI(oldUri.getScheme(), oldUri.getAuthority(), oldUri.getPath(),
+ appendQuery, oldUri.getFragment());
+
+ return newUri.toString();
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+
+ private static int calculateExponentialTime(int retries) {
+ long baseTime = BASIC_SLEEP_TIME * (1L << retries);
+ return (int) (baseTime * ((new Random()).nextDouble() + 0.5));
+ }
+}
\ No newline at end of file