You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/02/01 19:41:02 UTC
[43/50] [abbrv] hadoop git commit: YARN-4428. Redirect RM page to AHS
page when AHS turned on and RM page is not available. Contributed by Chang Li
YARN-4428. Redirect RM page to AHS page when AHS turned on and RM page is not available. Contributed by Chang Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/772ea7b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/772ea7b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/772ea7b4
Branch: refs/heads/HDFS-7240
Commit: 772ea7b41b06beaa1f4ac4fa86eac8d6e6c8cd36
Parents: f4a57d4
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jan 29 21:48:54 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jan 29 21:48:54 2016 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../rmapp/attempt/RMAppAttemptImpl.java | 23 ++++-
.../server/resourcemanager/webapp/RMWebApp.java | 5 ++
.../resourcemanager/webapp/RMWebAppFilter.java | 90 ++++++++++++++++++++
.../attempt/TestRMAppAttemptTransitions.java | 78 ++++++++++++++++-
5 files changed, 197 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/772ea7b4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ebbf0f4..d82a9be 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1441,6 +1441,9 @@ Release 2.7.3 - UNRELEASED
YARN-4598. Invalid event: RESOURCE_FAILED at
CONTAINER_CLEANEDUP_AFTER_KILL (tangshangwen via jlowe)
+ YARN-4428. Redirect RM page to AHS page when AHS turned on and RM page is
+ not available (Chang Li via jlowe)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/772ea7b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 99f1b21..16456ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -628,6 +629,21 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
}
+ private void setTrackingUrlToAHSPage(RMAppAttemptState stateToBeStored) {
+ originalTrackingUrl = pjoin(
+ WebAppUtils.getHttpSchemePrefix(conf) +
+ WebAppUtils.getAHSWebAppURLWithoutScheme(conf),
+ "applicationhistory", "app", getAppAttemptId().getApplicationId());
+ switch (stateToBeStored) {
+ case KILLED:
+ case FAILED:
+ proxiedTrackingUrl = originalTrackingUrl;
+ break;
+ default:
+ break;
+ }
+ }
+
private void invalidateAMHostAndPort() {
this.host = "N/A";
this.rpcPort = -1;
@@ -1211,7 +1227,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
String diags = null;
// don't leave the tracking URL pointing to a non-existent AM
- setTrackingUrlToRMAppPage(stateToBeStored);
+ if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
+ setTrackingUrlToAHSPage(stateToBeStored);
+ } else {
+ setTrackingUrlToRMAppPage(stateToBeStored);
+ }
String finalTrackingUrl = getOriginalTrackingUrl();
FinalApplicationStatus finalStatus = null;
int exitStatus = ContainerExitStatus.INVALID;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/772ea7b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index 91c3d72..106065b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.RMHAUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
@@ -122,4 +123,8 @@ public class RMWebApp extends WebApp implements YarnWebParams {
return rm.getRMContext().getRMAdminService()
.getHAZookeeperConnectionState();
}
+
+ public RMContext getRMContext() {
+ return rm.getRMContext();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/772ea7b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
index 9f35b6c..7413748 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
+
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetSocketAddress;
@@ -35,9 +37,19 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HtmlQuoting;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
import com.google.inject.Injector;
@@ -45,6 +57,8 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
@Singleton
public class RMWebAppFilter extends GuiceContainer {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RMWebAppFilter.class);
private Injector injector;
/**
@@ -56,6 +70,8 @@ public class RMWebAppFilter extends GuiceContainer {
private static final Set<String> NON_REDIRECTED_URIS = Sets.newHashSet(
"/conf", "/stacks", "/logLevel", "/logs");
private String path;
+ private boolean ahsEnabled;
+ private String ahsPageURLPrefix;
private static final int BASIC_SLEEP_TIME = 5;
private static final int MAX_SLEEP_TIME = 5 * 60;
private static final Random randnum = new Random();
@@ -76,6 +92,13 @@ public class RMWebAppFilter extends GuiceContainer {
path = YarnConfiguration.useHttps(conf)
? "https://" + path
: "http://" + path;
+ ahsEnabled = conf.getBoolean(
+ YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED);
+ ahsPageURLPrefix = pjoin(
+ WebAppUtils.getHttpSchemePrefix(conf) +
+ WebAppUtils.getAHSWebAppURLWithoutScheme(
+ conf), "applicationhistory");
}
@Override
@@ -137,11 +160,78 @@ public class RMWebAppFilter extends GuiceContainer {
}
}
return;
+ } else if (ahsEnabled) {
+ String ahsRedirectUrl = ahsRedirectPath(uri, rmWebApp);
+ if(ahsRedirectUrl != null) {
+ response.setHeader("Location", ahsRedirectUrl);
+ response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+ return;
+ }
}
super.doFilter(request, response, chain);
}
+ private String ahsRedirectPath(String uri, RMWebApp rmWebApp) {
+ // TODO: Commonize URL parsing code. Will be done in YARN-4642.
+ String redirectPath = null;
+ if(uri.contains("/cluster/")) {
+ String[] parts = uri.split("/");
+ if(parts.length > 3) {
+ RMContext context = rmWebApp.getRMContext();
+ String type = parts[2];
+ ApplicationId appId = null;
+ ApplicationAttemptId appAttemptId = null;
+ ContainerId containerId = null;
+ switch(type){
+ case "app":
+ try {
+ appId = Apps.toAppID(parts[3]);
+ } catch (YarnRuntimeException | NumberFormatException e) {
+ LOG.debug("Error parsing {} as an ApplicationId",
+ parts[3], e);
+ return redirectPath;
+ }
+ if(!context.getRMApps().containsKey(appId)) {
+ redirectPath = pjoin(ahsPageURLPrefix, "app", appId);
+ }
+ break;
+ case "appattempt":
+ try{
+ appAttemptId = ConverterUtils.toApplicationAttemptId(parts[3]);
+ } catch (IllegalArgumentException e) {
+ LOG.debug("Error parsing {} as an ApplicationAttemptId",
+ parts[3], e);
+ return redirectPath;
+ }
+ if(!context.getRMApps().containsKey(
+ appAttemptId.getApplicationId())) {
+ redirectPath = pjoin(ahsPageURLPrefix,
+ "appattempt", appAttemptId);
+ }
+ break;
+ case "container":
+ try {
+ containerId = ContainerId.fromString(parts[3]);
+ } catch (IllegalArgumentException e) {
+ LOG.debug("Error parsing {} as an ContainerId",
+ parts[3], e);
+ return redirectPath;
+ }
+ if(!context.getRMApps().containsKey(
+ containerId.getApplicationAttemptId().getApplicationId())) {
+ redirectPath = pjoin(ahsPageURLPrefix,
+ "container", containerId);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ return redirectPath;
+ }
+
private boolean shouldRedirect(RMWebApp rmWebApp, String uri) {
return !uri.equals("/" + rmWebApp.wsName() + "/v1/cluster/info")
&& !uri.equals("/" + rmWebApp.name() + "/cluster")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/772ea7b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index e7985fa..93f17e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.event.InlineDispatcher;
@@ -137,7 +138,10 @@ public class TestRMAppAttemptTransitions {
private static final String FAILED_DIAGNOSTICS = "Attempt failed by user.";
private static final String RM_WEBAPP_ADDR =
WebAppUtils.getResolvedRMWebAppURLWithScheme(new Configuration());
-
+ private static final String AHS_WEBAPP_ADDR =
+ WebAppUtils.getHttpSchemePrefix(new Configuration()) +
+ WebAppUtils.getAHSWebAppURLWithoutScheme(new Configuration());
+
private boolean isSecurityEnabled;
private RMContext rmContext;
private RMContext spyRMContext;
@@ -1090,6 +1094,78 @@ public class TestRMAppAttemptTransitions {
verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}
+ @SuppressWarnings("unchecked")
+ @Test(timeout=10000)
+ public void testLaunchedFailWhileAHSEnabled() {
+ Configuration myConf = new Configuration(conf);
+ myConf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
+ ApplicationId applicationId = MockApps.newAppID(appId);
+ ApplicationAttemptId applicationAttemptId =
+ ApplicationAttemptId.newInstance(applicationId, 2);
+ RMAppAttempt myApplicationAttempt =
+ new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
+ spyRMContext, scheduler,masterService,
+ submissionContext, myConf, false,
+ BuilderUtils.newResourceRequest(
+ RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
+ submissionContext.getResource(), 1));
+
+ //submit, schedule and allocate app attempt
+ myApplicationAttempt.handle(
+ new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(),
+ RMAppAttemptEventType.START));
+ myApplicationAttempt.handle(
+ new RMAppAttemptEvent(
+ myApplicationAttempt.getAppAttemptId(),
+ RMAppAttemptEventType.ATTEMPT_ADDED));
+
+ Container amContainer = mock(Container.class);
+ Resource resource = BuilderUtils.newResource(2048, 1);
+ when(amContainer.getId()).thenReturn(
+ BuilderUtils.newContainerId(myApplicationAttempt.getAppAttemptId(), 1));
+ when(amContainer.getResource()).thenReturn(resource);
+ Allocation allocation = mock(Allocation.class);
+ when(allocation.getContainers()).
+ thenReturn(Collections.singletonList(amContainer));
+ when(scheduler.allocate(any(ApplicationAttemptId.class), any(List.class),
+ any(List.class), any(List.class), any(List.class), any(List.class),
+ any(List.class))).thenReturn(allocation);
+ RMContainer rmContainer = mock(RMContainerImpl.class);
+ when(scheduler.getRMContainer(amContainer.getId())).thenReturn(rmContainer);
+
+ myApplicationAttempt.handle(
+ new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(),
+ RMAppAttemptEventType.CONTAINER_ALLOCATED));
+ assertEquals(RMAppAttemptState.ALLOCATED_SAVING,
+ myApplicationAttempt.getAppAttemptState());
+ myApplicationAttempt.handle(
+ new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(),
+ RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
+
+ // launch app attempt
+ myApplicationAttempt.handle(
+ new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(),
+ RMAppAttemptEventType.LAUNCHED));
+ assertEquals(YarnApplicationAttemptState.LAUNCHED,
+ myApplicationAttempt.createApplicationAttemptState());
+
+ //fail container right after launched
+ NodeId anyNodeId = NodeId.newInstance("host", 1234);
+ myApplicationAttempt.handle(
+ new RMAppAttemptContainerFinishedEvent(
+ myApplicationAttempt.getAppAttemptId(),
+ BuilderUtils.newContainerStatus(amContainer.getId(),
+ ContainerState.COMPLETE, "", 0,
+ amContainer.getResource()), anyNodeId));
+ sendAttemptUpdateSavedEvent(myApplicationAttempt);
+ assertEquals(RMAppAttemptState.FAILED,
+ myApplicationAttempt.getAppAttemptState());
+ String rmAppPageUrl = pjoin(AHS_WEBAPP_ADDR, "applicationhistory", "app",
+ myApplicationAttempt.getAppAttemptId().getApplicationId());
+ assertEquals(rmAppPageUrl, myApplicationAttempt.getOriginalTrackingUrl());
+ assertEquals(rmAppPageUrl, myApplicationAttempt.getTrackingUrl());
+ }
+
@Test(timeout=20000)
public void testRunningExpire() {
Container amContainer = allocateApplicationAttempt();