You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2012/12/22 01:28:02 UTC
svn commit: r1425211 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/
hadoop-yarn/hadoop-ya...
Author: vinodkv
Date: Sat Dec 22 00:28:02 2012
New Revision: 1425211
URL: http://svn.apache.org/viewvc?rev=1425211&view=rev
Log:
YARN-285. Added a temporary plugin interface for RM to be able to redirect to JobHistory server for apps that it no longer tracks. Contributed by Derek Dagit.
svn merge --ignore-ancestry -c 1425210 ../../trunk/
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
- copied unchanged from r1425210, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1425211&r1=1425210&r2=1425211&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Sat Dec 22 00:28:02 2012
@@ -11,6 +11,10 @@ Release 2.0.3-alpha - Unreleased
YARN-230. RM Restart phase 1 - includes support for saving/restarting all
applications on an RM bounce. (Bikas Saha via acmurthy)
+ YARN-285. Added a temporary plugin interface for RM to be able to redirect
+ to JobHistory server for apps that it no longer tracks. (Derek Dagit via
+ vinodkv)
+
IMPROVEMENTS
YARN-223. Update process tree instead of getting new process trees.
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1425211&r1=1425210&r2=1425211&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Sat Dec 22 00:28:02 2012
@@ -400,6 +400,9 @@ public class YarnConfiguration extends C
public static final String YARN_LOG_SERVER_URL =
YARN_PREFIX + "log.server.url";
+
+ public static final String YARN_TRACKING_URL_GENERATOR =
+ YARN_PREFIX + "tracking.url.generator";
/** Amount of memory in GB that can be allocated for containers.*/
public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb";
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java?rev=1425211&r1=1425210&r2=1425211&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java Sat Dec 22 00:28:02 2012
@@ -24,11 +24,13 @@ import java.io.UnsupportedEncodingExcept
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.TrackingUriPlugin;
public class ProxyUriUtils {
@SuppressWarnings("unused")
@@ -143,4 +145,28 @@ public class ProxyUriUtils {
throws URISyntaxException {
return new URI(HttpConfig.getSchemePrefix() + noSchemeUrl);
}
+
+ /**
+ * Returns the first valid tracking link, if any, from the given id from the
+ * given list of plug-ins, if any.
+ *
+ * @param id the id of the application for which the tracking link is desired
+ * @param trackingUriPlugins list of plugins from which to get the tracking link
+ * @return the desired link if possible, otherwise null
+ * @throws URISyntaxException
+ */
+ public static URI getUriFromTrackingPlugins(ApplicationId id,
+ List<TrackingUriPlugin> trackingUriPlugins)
+ throws URISyntaxException {
+ URI toRet = null;
+ for(TrackingUriPlugin plugin : trackingUriPlugins)
+ {
+ toRet = plugin.getTrackingUri(id);
+ if (toRet != null)
+ {
+ return toRet;
+ }
+ }
+ return null;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java?rev=1425211&r1=1425210&r2=1425211&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java Sat Dec 22 00:28:02 2012
@@ -30,6 +30,7 @@ import java.util.Arrays;
import java.util.EnumSet;
import java.util.Enumeration;
import java.util.HashSet;
+import java.util.List;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServlet;
@@ -48,8 +49,10 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.StringHelper;
+import org.apache.hadoop.yarn.util.TrackingUriPlugin;
import org.apache.hadoop.yarn.webapp.MimeType;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
@@ -61,8 +64,9 @@ public class WebAppProxyServlet extends
"Accept-Language", "Accept-Charset"));
public static final String PROXY_USER_COOKIE_NAME = "proxy-user";
-
-
+
+ private final List<TrackingUriPlugin> trackingUriPlugins;
+
private static class _ implements Hamlet._ {
//Empty
}
@@ -78,6 +82,18 @@ public class WebAppProxyServlet extends
}
/**
+ * Default constructor
+ */
+ public WebAppProxyServlet()
+ {
+ super();
+ YarnConfiguration conf = new YarnConfiguration();
+ this.trackingUriPlugins =
+ conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR,
+ TrackingUriPlugin.class);
+ }
+
+ /**
* Output 404 with appropriate message.
* @param resp the http response.
* @param message the message to include on the page.
@@ -253,6 +269,16 @@ public class WebAppProxyServlet extends
if(applicationReport == null) {
LOG.warn(req.getRemoteUser()+" Attempting to access "+id+
" that was not found");
+
+ URI toFetch =
+ ProxyUriUtils
+ .getUriFromTrackingPlugins(id, this.trackingUriPlugins);
+ if (toFetch != null)
+ {
+ resp.sendRedirect(resp.encodeRedirectURL(toFetch.toString()));
+ return;
+ }
+
notFound(resp, "Application "+appId+" could not be found, " +
"please try the history server");
return;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java?rev=1425211&r1=1425210&r2=1425211&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java Sat Dec 22 00:28:02 2012
@@ -21,11 +21,16 @@ package org.apache.hadoop.yarn.server.we
import static org.junit.Assert.*;
import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.TrackingUriPlugin;
import org.junit.Test;
+import com.google.common.collect.Lists;
+
public class TestProxyUriUtils {
@Test
public void testGetPathApplicationId() {
@@ -83,4 +88,36 @@ public class TestProxyUriUtils {
URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
assertEquals(expected, result);
}
+
+ @Test
+ public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins()
+ throws URISyntaxException {
+ ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
+ List<TrackingUriPlugin> list =
+ Lists.newArrayListWithExpectedSize(0);
+ assertNull(ProxyUriUtils.getUriFromTrackingPlugins(id, list));
+ }
+
+ @Test
+ public void testGetProxyUriFromPluginsReturnsValidUriWhenAble()
+ throws URISyntaxException {
+ ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
+ List<TrackingUriPlugin> list =
+ Lists.newArrayListWithExpectedSize(2);
+ // Insert a plugin that returns null.
+ list.add(new TrackingUriPlugin() {
+ public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
+ return null;
+ }
+ });
+ // Insert a plugin that returns a valid URI.
+ list.add(new TrackingUriPlugin() {
+ public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
+ return new URI("http://history.server.net/");
+ }
+ });
+ URI result = ProxyUriUtils.getUriFromTrackingPlugins(id, list);
+ assertNotNull(result);
+
+ }
}