You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2016/10/03 22:59:10 UTC
[57/57] [abbrv] hadoop git commit: YARN-4767. Network issues can
cause persistent RM UI outage. (Daniel Templeton via kasha)
YARN-4767. Network issues can cause persistent RM UI outage. (Daniel Templeton via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/736d33cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/736d33cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/736d33cd
Branch: refs/heads/HDFS-10467
Commit: 736d33cddd88a0cec925a451940b2523999a9c51
Parents: 607705c
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Mon Oct 3 14:35:57 2016 -0700
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Mon Oct 3 14:35:57 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../resourcemanager/webapp/ErrorBlock.java | 39 +++
.../server/resourcemanager/webapp/RMWebApp.java | 1 +
.../webapp/RedirectionErrorPage.java | 47 ++++
.../resourcemanager/webapp/RmController.java | 4 +
.../webapp/TestRedirectionErrorPage.java | 68 +++++
.../server/webproxy/WebAppProxyServlet.java | 274 ++++++++++++++-----
.../server/webproxy/amfilter/AmIpFilter.java | 64 +++--
.../server/webproxy/TestWebAppProxyServlet.java | 24 +-
.../server/webproxy/amfilter/TestAmFilter.java | 29 +-
10 files changed, 454 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index a34273c..ee9100f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -41,4 +41,5 @@ public interface YarnWebParams {
String NODE_LABEL = "node.label";
String WEB_UI_TYPE = "web.ui.type";
String NEXT_REFRESH_INTERVAL = "next.refresh.interval";
+ String ERROR_MESSAGE = "error.message";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ErrorBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ErrorBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ErrorBlock.java
new file mode 100644
index 0000000..963e53f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ErrorBlock.java
@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import com.google.inject.Inject;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.ERROR_MESSAGE;
+
+/**
+ * This class is used to display an error message to the user in the UI.
+ */
+public class ErrorBlock extends HtmlBlock {
+ @Inject
+ ErrorBlock(ViewContext ctx) {
+ super(ctx);
+ }
+
+ @Override
+ protected void render(Block html) {
+ html.p()._($(ERROR_MESSAGE))._();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index 106065b..2d7139f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -71,6 +71,7 @@ public class RMWebApp extends WebApp implements YarnWebParams {
route("/errors-and-warnings", RmController.class, "errorsAndWarnings");
route(pajoin("/logaggregationstatus", APPLICATION_ID),
RmController.class, "logaggregationstatus");
+ route(pajoin("/failure", APPLICATION_ID), RmController.class, "failure");
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RedirectionErrorPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RedirectionErrorPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RedirectionErrorPage.java
new file mode 100644
index 0000000..beb0cca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RedirectionErrorPage.java
@@ -0,0 +1,47 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+
+/**
+ * This class is used to display a message that the proxy request failed
+ * because of a redirection issue.
+ */
+public class RedirectionErrorPage extends RmView {
+ @Override protected void preHead(Page.HTML<_> html) {
+ String aid = $(YarnWebParams.APPLICATION_ID);
+
+ commonPreHead(html);
+ set(YarnWebParams.ERROR_MESSAGE,
+ "The application master for " + aid + " redirected the "
+ + "resource manager's web proxy's request back to the web proxy, "
+ + "which means your request to view the application master's web UI "
+ + "cannot be fulfilled. The typical cause for this error is a "
+ + "network misconfiguration that causes the resource manager's web "
+ + "proxy host to resolve to an unexpected IP address on the "
+ + "application master host. Please contact your cluster "
+ + "administrator to resolve the issue.");
+ }
+
+ @Override protected Class<? extends SubView> content() {
+ return ErrorBlock.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
index b124d75..a291e05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
@@ -62,6 +62,10 @@ public class RmController extends Controller {
render(ContainerPage.class);
}
+ public void failure() {
+ render(RedirectionErrorPage.class);
+ }
+
public void nodes() {
render(NodesPage.class);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRedirectionErrorPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRedirectionErrorPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRedirectionErrorPage.java
new file mode 100644
index 0000000..408dc9b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRedirectionErrorPage.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.junit.Test;
+
+import com.google.inject.Binder;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+/**
+ * This class tests the RedirectionErrorPage.
+ */
+public class TestRedirectionErrorPage {
+ @Test
+ public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1234L, 0);
+ Injector injector;
+
+ // initialize RM Context, and create RMApp, without creating RMAppAttempt
+ final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
+
+ injector = WebAppTests.createMockInjector(RMContext.class, rmContext,
+ new Module() {
+ @Override
+ public void configure(Binder binder) {
+ try {
+ ResourceManager rm = TestRMWebApp.mockRm(rmContext);
+ binder.bind(ResourceManager.class).toInstance(rm);
+ binder.bind(ApplicationBaseProtocol.class).toInstance(
+ rm.getClientRMService());
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ });
+
+ ErrorBlock instance = injector.getInstance(ErrorBlock.class);
+ instance.set(YarnWebParams.APPLICATION_ID, appId.toString());
+ instance.set(YarnWebParams.ERROR_MESSAGE, "This is an error");
+ instance.render();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
index 0b621aa..b32ee30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
@@ -26,6 +26,7 @@ import java.io.ObjectInputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.InetAddress;
+import java.net.SocketException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
@@ -42,8 +43,10 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.core.UriBuilderException;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -76,7 +79,8 @@ public class WebAppProxyServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(
WebAppProxyServlet.class);
- private static final Set<String> passThroughHeaders =
+ private static final String REDIRECT = "/redirect";
+ private static final Set<String> PASS_THROUGH_HEADERS =
new HashSet<>(Arrays.asList(
"User-Agent",
"Accept",
@@ -93,6 +97,7 @@ public class WebAppProxyServlet extends HttpServlet {
private transient List<TrackingUriPlugin> trackingUriPlugins;
private final String rmAppPageUrlBase;
private final String ahsAppPageUrlBase;
+ private final String failurePageUrlBase;
private transient YarnConfiguration conf;
/**
@@ -126,11 +131,16 @@ public class WebAppProxyServlet extends HttpServlet {
this.trackingUriPlugins =
conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR,
TrackingUriPlugin.class);
- this.rmAppPageUrlBase = StringHelper.pjoin(
- WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "app");
- this.ahsAppPageUrlBase = StringHelper.pjoin(
- WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils
- .getAHSWebAppURLWithoutScheme(conf), "applicationhistory", "app");
+ this.rmAppPageUrlBase =
+ StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf),
+ "cluster", "app");
+ this.failurePageUrlBase =
+ StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf),
+ "cluster", "failure");
+ this.ahsAppPageUrlBase =
+ StringHelper.pjoin(WebAppUtils.getHttpSchemePrefix(conf)
+ + WebAppUtils.getAHSWebAppURLWithoutScheme(conf),
+ "applicationhistory", "app");
}
/**
@@ -220,9 +230,9 @@ public class WebAppProxyServlet extends HttpServlet {
@SuppressWarnings("unchecked")
Enumeration<String> names = req.getHeaderNames();
- while(names.hasMoreElements()) {
+ while (names.hasMoreElements()) {
String name = names.nextElement();
- if(passThroughHeaders.contains(name)) {
+ if (PASS_THROUGH_HEADERS.contains(name)) {
String value = req.getHeader(name);
if (LOG.isDebugEnabled()) {
LOG.debug("REQ HEADER: {} : {}", name, value);
@@ -312,30 +322,49 @@ public class WebAppProxyServlet extends HttpServlet {
boolean userWasWarned = false;
boolean userApproved = Boolean.parseBoolean(userApprovedParamS);
boolean securityEnabled = isSecurityEnabled();
+ boolean isRedirect = false;
+ String pathInfo = req.getPathInfo();
final String remoteUser = req.getRemoteUser();
- final String pathInfo = req.getPathInfo();
String[] parts = null;
+
if (pathInfo != null) {
+ // If there's a redirect, strip the redirect so that the path can be
+ // parsed
+ if (pathInfo.startsWith(REDIRECT)) {
+ pathInfo = pathInfo.substring(REDIRECT.length());
+ isRedirect = true;
+ }
+
parts = pathInfo.split("/", 3);
}
- if(parts == null || parts.length < 2) {
+
+ if ((parts == null) || (parts.length < 2)) {
LOG.warn("{} gave an invalid proxy path {}", remoteUser, pathInfo);
notFound(resp, "Your path appears to be formatted incorrectly.");
return;
}
+
//parts[0] is empty because path info always starts with a /
String appId = parts[1];
String rest = parts.length > 2 ? parts[2] : "";
ApplicationId id = Apps.toAppID(appId);
- if(id == null) {
+
+ if (id == null) {
LOG.warn("{} attempting to access {} that is invalid",
remoteUser, appId);
notFound(resp, appId + " appears to be formatted incorrectly.");
return;
}
-
- if(securityEnabled) {
+
+ // If this call is from an AM redirect, we need to be careful about how
+ // we handle it. If this method returns true, it means the method
+ // already redirected the response, so we can just return.
+ if (isRedirect && handleRedirect(appId, req, resp)) {
+ return;
+ }
+
+ if (securityEnabled) {
String cookieName = getCheckCookieName(id);
Cookie[] cookies = req.getCookies();
if (cookies != null) {
@@ -351,22 +380,21 @@ public class WebAppProxyServlet extends HttpServlet {
boolean checkUser = securityEnabled && (!userWasWarned || !userApproved);
- FetchedAppReport fetchedAppReport = null;
- ApplicationReport applicationReport = null;
+ FetchedAppReport fetchedAppReport;
+
try {
- fetchedAppReport = getApplicationReport(id);
- if (fetchedAppReport != null) {
- if (fetchedAppReport.getAppReportSource() != AppReportSource.RM &&
- fetchedAppReport.getAppReportSource() != AppReportSource.AHS) {
- throw new UnsupportedOperationException("Application report not "
- + "fetched from RM or history server.");
- }
- applicationReport = fetchedAppReport.getApplicationReport();
- }
+ fetchedAppReport = getFetchedAppReport(id);
} catch (ApplicationNotFoundException e) {
- applicationReport = null;
+ fetchedAppReport = null;
}
- if(applicationReport == null) {
+
+ ApplicationReport applicationReport = null;
+
+ if (fetchedAppReport != null) {
+ applicationReport = fetchedAppReport.getApplicationReport();
+ }
+
+ if (applicationReport == null) {
LOG.warn("{} attempting to access {} that was not found",
remoteUser, id);
@@ -382,57 +410,31 @@ public class WebAppProxyServlet extends HttpServlet {
"in RM or history server");
return;
}
- String original = applicationReport.getOriginalTrackingUrl();
- URI trackingUri;
- if (original == null || original.equals("N/A") || original.equals("")) {
- if (fetchedAppReport.getAppReportSource() == AppReportSource.RM) {
- // fallback to ResourceManager's app page if no tracking URI provided
- // and Application Report was fetched from RM
- LOG.debug("Original tracking url is '{}'. Redirecting to RM app page",
- original == null? "NULL" : original);
- ProxyUtils.sendRedirect(req, resp,
- StringHelper.pjoin(rmAppPageUrlBase, id.toString()));
- } else if (fetchedAppReport.getAppReportSource()
- == AppReportSource.AHS) {
- // fallback to Application History Server app page if the application
- // report was fetched from AHS
- LOG.debug("Original tracking url is '{}'. Redirecting to AHS app page"
- , original == null? "NULL" : original);
- ProxyUtils.sendRedirect(req, resp,
- StringHelper.pjoin(ahsAppPageUrlBase, id.toString()));
- }
+
+ URI trackingUri = getTrackingUri(req, resp, id,
+ applicationReport.getOriginalTrackingUrl(),
+ fetchedAppReport.getAppReportSource());
+
+ // If the tracking URI is null, there was a redirect, so just return.
+ if (trackingUri == null) {
return;
- } else {
- if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) {
- trackingUri = ProxyUriUtils.getUriFromAMUrl(
- WebAppUtils.getHttpSchemePrefix(conf), original);
- } else {
- trackingUri = new URI(original);
- }
}
String runningUser = applicationReport.getUser();
- if(checkUser && !runningUser.equals(remoteUser)) {
+
+ if (checkUser && !runningUser.equals(remoteUser)) {
LOG.info("Asking {} if they want to connect to the "
+ "app master GUI of {} owned by {}",
remoteUser, appId, runningUser);
warnUserPage(resp, ProxyUriUtils.getPathAndQuery(id, rest,
req.getQueryString(), true), runningUser, id);
+
return;
}
// Append the user-provided path and query parameter to the original
// tracking url.
- UriBuilder builder = UriBuilder.fromUri(trackingUri);
- String queryString = req.getQueryString();
- if (queryString != null) {
- List<NameValuePair> queryPairs =
- URLEncodedUtils.parse(queryString, null);
- for (NameValuePair pair : queryPairs) {
- builder.queryParam(pair.getName(), pair.getValue());
- }
- }
- URI toFetch = builder.path(rest).build();
+ URI toFetch = buildTrackingUrl(trackingUri, req, rest);
LOG.info("{} is accessing unchecked {}"
+ " which is the app master GUI of {} owned by {}",
@@ -459,6 +461,152 @@ public class WebAppProxyServlet extends HttpServlet {
}
/**
+ * Return a URL based on the {@code trackingUri} that includes the
+ * user-provided path and query parameters.
+ *
+ * @param trackingUri the base tracking URI
+ * @param req the service request
+ * @param rest the user-provided path
+ * @return the new tracking URI
+ * @throws UriBuilderException if there's an error building the URL
+ */
+ private URI buildTrackingUrl(URI trackingUri, final HttpServletRequest req,
+ String rest) throws UriBuilderException {
+ UriBuilder builder = UriBuilder.fromUri(trackingUri);
+ String queryString = req.getQueryString();
+
+ if (queryString != null) {
+ List<NameValuePair> queryPairs = URLEncodedUtils.parse(queryString, null);
+
+ for (NameValuePair pair : queryPairs) {
+ builder.queryParam(pair.getName(), pair.getValue());
+ }
+ }
+
+ return builder.path(rest).build();
+ }
+
+ /**
+ * Locate the tracking URI for the application based on the reported tracking
+ * URI. If the reported URI is invalid, redirect to the history server or RM
+ * app page. If the URI is valid, covert it into a usable URI object with a
+ * schema. If the returned URI is null, that means there was a redirect.
+ *
+ * @param req the servlet request for redirects
+ * @param resp the servlet response for redirects
+ * @param id the application ID
+ * @param originalUri the reported tracking URI
+ * @param appReportSource the source of the application report
+ * @return a valid tracking URI or null if redirected instead
+ * @throws IOException thrown if the redirect fails
+ * @throws URISyntaxException if the tracking URI is invalid
+ */
+ private URI getTrackingUri(HttpServletRequest req, HttpServletResponse resp,
+ ApplicationId id, String originalUri, AppReportSource appReportSource)
+ throws IOException, URISyntaxException {
+ URI trackingUri = null;
+
+ if ((originalUri == null) ||
+ originalUri.equals("N/A") ||
+ originalUri.equals("")) {
+ if (appReportSource == AppReportSource.RM) {
+ // fallback to ResourceManager's app page if no tracking URI provided
+ // and Application Report was fetched from RM
+ LOG.debug("Original tracking url is '{}'. Redirecting to RM app page",
+ originalUri == null ? "NULL" : originalUri);
+ ProxyUtils.sendRedirect(req, resp,
+ StringHelper.pjoin(rmAppPageUrlBase, id.toString()));
+ } else if (appReportSource == AppReportSource.AHS) {
+ // fallback to Application History Server app page if the application
+ // report was fetched from AHS
+ LOG.debug("Original tracking url is '{}'. Redirecting to AHS app page",
+ originalUri == null ? "NULL" : originalUri);
+ ProxyUtils.sendRedirect(req, resp,
+ StringHelper.pjoin(ahsAppPageUrlBase, id.toString()));
+ }
+ } else if (ProxyUriUtils.getSchemeFromUrl(originalUri).isEmpty()) {
+ trackingUri =
+ ProxyUriUtils.getUriFromAMUrl(WebAppUtils.getHttpSchemePrefix(conf),
+ originalUri);
+ } else {
+ trackingUri = new URI(originalUri);
+ }
+
+ return trackingUri;
+ }
+
+ /**
+ * Fetch the application report from the RM.
+ *
+ * @param id the app ID
+ * @return the application report
+ * @throws IOException if the request to the RM fails
+ * @throws YarnException if the request to the RM fails
+ */
+ private FetchedAppReport getFetchedAppReport(ApplicationId id)
+ throws IOException, YarnException {
+ FetchedAppReport fetchedAppReport = getApplicationReport(id);
+
+ if (fetchedAppReport != null) {
+ if ((fetchedAppReport.getAppReportSource() != AppReportSource.RM) &&
+ (fetchedAppReport.getAppReportSource() != AppReportSource.AHS)) {
+ throw new UnsupportedOperationException("Application report not "
+ + "fetched from RM or history server.");
+ }
+ }
+
+ return fetchedAppReport;
+ }
+
+ /**
+ * Check whether the request is a redirect from the AM and handle it
+ * appropriately. This check exists to prevent the AM from forwarding back to
+ * the web proxy, which would contact the AM again, which would forward
+ * again... If this method returns true, there was a redirect, and
+ * it was handled by redirecting the current request to an error page.
+ *
+ * @param path the part of the request path after the app id
+ * @param id the app id
+ * @param req the request object
+ * @param resp the response object
+ * @return whether there was a redirect
+ * @throws IOException if a redirect fails
+ */
+ private boolean handleRedirect(String id, HttpServletRequest req,
+ HttpServletResponse resp) throws IOException {
+ // If this isn't a redirect, we don't care.
+ boolean badRedirect = false;
+
+ // If this is a redirect, check if we're calling ourselves.
+ try {
+ badRedirect = NetUtils.getLocalInetAddress(req.getRemoteHost()) != null;
+ } catch (SocketException ex) {
+ // This exception means we can't determine the calling host. Odds are
+ // that means it's not us. Let it go and hope it works out better next
+ // time.
+ }
+
+ // If the proxy tries to call itself, it gets into an endless
+ // loop and consumes all available handler threads until the
+ // application completes. Redirect to the app page with a flag
+ // that tells it to print an appropriate error message.
+ if (badRedirect) {
+ LOG.error("The AM's web app redirected the RM web proxy's request back "
+ + "to the web proxy. The typical cause is that the AM is resolving "
+ + "the RM's address as something other than what it expects. Check "
+ + "your network configuration and the value of the "
+ + "yarn.web-proxy.address property. Once the host resolution issue "
+ + "has been resolved, you will likely need to delete the "
+ + "misbehaving application, " + id);
+ String redirect = StringHelper.pjoin(failurePageUrlBase, id);
+ LOG.error("REDIRECT: sending redirect to " + redirect);
+ ProxyUtils.sendRedirect(req, resp, redirect);
+ }
+
+ return badRedirect;
+ }
+
+ /**
* This method is used by Java object deserialization, to fill in the
* transient {@link #trackingUriPlugins} field.
* See {@link ObjectInputStream#defaultReadObject()}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
index e7617f0..fe6fc32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
@@ -59,8 +59,9 @@ public class AmIpFilter implements Filter {
public static final String PROXY_HOSTS_DELIMITER = ",";
public static final String PROXY_URI_BASES = "PROXY_URI_BASES";
public static final String PROXY_URI_BASES_DELIMITER = ",";
+ private static final String PROXY_PATH = "/proxy";
//update the proxy IP list about every 5 min
- private static final long updateInterval = 5 * 60 * 1000;
+ private static final long UPDATE_INTERVAL = 5 * 60 * 1000;
private String[] proxyHosts;
private Set<String> proxyAddresses = null;
@@ -96,7 +97,7 @@ public class AmIpFilter implements Filter {
protected Set<String> getProxyAddresses() throws ServletException {
long now = System.currentTimeMillis();
synchronized(this) {
- if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) {
+ if (proxyAddresses == null || (lastUpdate + UPDATE_INTERVAL) >= now) {
proxyAddresses = new HashSet<>();
for (String proxyHost : proxyHosts) {
try {
@@ -131,37 +132,52 @@ public class AmIpFilter implements Filter {
HttpServletRequest httpReq = (HttpServletRequest)req;
HttpServletResponse httpResp = (HttpServletResponse)resp;
+
if (LOG.isDebugEnabled()) {
LOG.debug("Remote address for request is: {}", httpReq.getRemoteAddr());
}
+
if (!getProxyAddresses().contains(httpReq.getRemoteAddr())) {
- String redirectUrl = findRedirectUrl();
- String target = redirectUrl + httpReq.getRequestURI();
- ProxyUtils.sendRedirect(httpReq, httpResp, target);
- return;
- }
+ StringBuilder redirect = new StringBuilder(findRedirectUrl());
+
+ redirect.append(httpReq.getRequestURI());
+
+ int insertPoint = redirect.indexOf(PROXY_PATH);
+
+ if (insertPoint >= 0) {
+ // Add /redirect as the second component of the path so that the RM web
+ // proxy knows that this request was a redirect.
+ insertPoint += PROXY_PATH.length();
+ redirect.insert(insertPoint, "/redirect");
+ }
- String user = null;
+ ProxyUtils.sendRedirect(httpReq, httpResp, redirect.toString());
+ } else {
+ String user = null;
- if (httpReq.getCookies() != null) {
- for(Cookie c: httpReq.getCookies()) {
- if(WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())){
- user = c.getValue();
- break;
+ if (httpReq.getCookies() != null) {
+ for(Cookie c: httpReq.getCookies()) {
+ if(WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())){
+ user = c.getValue();
+ break;
+ }
}
}
- }
- if (user == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME
- + " cookie, so user will not be set");
+ if (user == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not find "
+ + WebAppProxyServlet.PROXY_USER_COOKIE_NAME
+ + " cookie, so user will not be set");
+ }
+
+ chain.doFilter(req, resp);
+ } else {
+ AmIpPrincipal principal = new AmIpPrincipal(user);
+ ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq,
+ principal);
+
+ chain.doFilter(requestWrapper, resp);
}
- chain.doFilter(req, resp);
- } else {
- final AmIpPrincipal principal = new AmIpPrincipal(user);
- ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq,
- principal);
- chain.doFilter(requestWrapper, resp);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
index 330e4de..7236982 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
@@ -155,7 +155,7 @@ public class TestWebAppProxyServlet {
URL emptyUrl = new URL("http://localhost:" + proxyPort + "/proxy");
HttpURLConnection emptyProxyConn = (HttpURLConnection) emptyUrl
.openConnection();
- emptyProxyConn.connect();;
+ emptyProxyConn.connect();
assertEquals(HttpURLConnection.HTTP_NOT_FOUND, emptyProxyConn.getResponseCode());
// wrong url. Set wrong app ID
@@ -176,6 +176,25 @@ public class TestWebAppProxyServlet {
assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode());
assertTrue(isResponseCookiePresent(
proxyConn, "checked_application_0_0000", "true"));
+
+ // test that redirection is squashed correctly
+ URL redirectUrl = new URL("http://localhost:" + proxyPort
+ + "/proxy/redirect/application_00_0");
+ proxyConn = (HttpURLConnection) redirectUrl.openConnection();
+ proxyConn.setInstanceFollowRedirects(false);
+ proxyConn.connect();
+ assertEquals("The proxy returned an unexpected status code rather than"
+ + "redirecting the connection (302)",
+ HttpURLConnection.HTTP_MOVED_TEMP, proxyConn.getResponseCode());
+
+ String expected =
+ WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration)
+ + "/cluster/failure/application_00_0";
+ String redirect = proxyConn.getHeaderField(ProxyUtils.LOCATION);
+
+ assertEquals("The proxy did not redirect the connection to the failure "
+ + "page of the RM", expected, redirect);
+
// cannot found application 1: null
appReportFetcher.answer = 1;
proxyConn = (HttpURLConnection) url.openConnection();
@@ -185,6 +204,7 @@ public class TestWebAppProxyServlet {
proxyConn.getResponseCode());
assertFalse(isResponseCookiePresent(
proxyConn, "checked_application_0_0000", "true"));
+
// cannot found application 2: ApplicationNotFoundException
appReportFetcher.answer = 4;
proxyConn = (HttpURLConnection) url.openConnection();
@@ -194,6 +214,7 @@ public class TestWebAppProxyServlet {
proxyConn.getResponseCode());
assertFalse(isResponseCookiePresent(
proxyConn, "checked_application_0_0000", "true"));
+
// wrong user
appReportFetcher.answer = 2;
proxyConn = (HttpURLConnection) url.openConnection();
@@ -203,6 +224,7 @@ public class TestWebAppProxyServlet {
assertTrue(s
.contains("to continue to an Application Master web interface owned by"));
assertTrue(s.contains("WARNING: The following page may not be safe!"));
+
//case if task has a not running status
appReportFetcher.answer = 3;
proxyConn = (HttpURLConnection) url.openConnection();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736d33cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
index 6f64777..9dc0ce0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.webproxy.amfilter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.net.HttpURLConnection;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -147,8 +148,8 @@ public class TestAmFilter {
testFilter.init(config);
HttpServletResponseForTest response = new HttpServletResponseForTest();
- // Test request should implements HttpServletRequest
+ // Test request should implements HttpServletRequest
ServletRequest failRequest = Mockito.mock(ServletRequest.class);
try {
testFilter.doFilter(failRequest, response, chain);
@@ -159,22 +160,32 @@ public class TestAmFilter {
// request with HttpServletRequest
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
- Mockito.when(request.getRemoteAddr()).thenReturn("redirect");
- Mockito.when(request.getRequestURI()).thenReturn("/redirect");
+ Mockito.when(request.getRemoteAddr()).thenReturn("nowhere");
+ Mockito.when(request.getRequestURI()).thenReturn("/app/application_00_0");
+
+ // address "redirect" is not in host list for non-proxy connection
testFilter.doFilter(request, response, chain);
- // address "redirect" is not in host list
- assertEquals(302, response.status);
+ assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, response.status);
String redirect = response.getHeader(ProxyUtils.LOCATION);
- assertEquals("http://bogus/redirect", redirect);
+ assertEquals("http://bogus/app/application_00_0", redirect);
+
+ // address "redirect" is not in host list for proxy connection
+ Mockito.when(request.getRequestURI()).thenReturn("/proxy/application_00_0");
+ testFilter.doFilter(request, response, chain);
+ assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, response.status);
+ redirect = response.getHeader(ProxyUtils.LOCATION);
+ assertEquals("http://bogus/proxy/redirect/application_00_0", redirect);
+
// "127.0.0.1" contains in host list. Without cookie
Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
testFilter.doFilter(request, response, chain);
-
assertTrue(doFilterRequest
.contains("javax.servlet.http.HttpServletRequest"));
+
// cookie added
- Cookie[] cookies = new Cookie[1];
- cookies[0] = new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user");
+ Cookie[] cookies = new Cookie[] {
+ new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user")
+ };
Mockito.when(request.getCookies()).thenReturn(cookies);
testFilter.doFilter(request, response, chain);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org