You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ec...@apache.org on 2015/10/23 21:01:12 UTC
[45/75] [abbrv] hadoop git commit: YARN-2513. Host framework UIs in
YARN for use with the ATS (jeagles)
YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25f8f801
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25f8f801
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25f8f801
Branch: refs/heads/HADOOP-11890
Commit: 25f8f801d15e3d9f27f4a2a198262407203e14a5
Parents: d759b4b
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Wed Oct 21 15:38:08 2015 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Wed Oct 21 15:38:08 2015 -0500
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../hadoop/yarn/conf/YarnConfiguration.java | 17 +++++++
.../org/apache/hadoop/yarn/webapp/WebApps.java | 18 +++++--
.../src/main/resources/yarn-default.xml | 6 +++
.../ApplicationHistoryServer.java | 44 ++++++++++++++--
.../TestApplicationHistoryServer.java | 53 +++++++++++++++++++-
.../src/site/markdown/TimelineServer.md | 8 +++
7 files changed, 140 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8277e05..ae26386 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -989,6 +989,8 @@ Release 2.7.2 - UNRELEASED
YARN-2801. Add documentation for node labels feature. (Wangda Tan and Naganarasimha
G R via ozawa)
+ YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 55b829f..3e89259 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1524,6 +1524,23 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_PREFIX =
YARN_PREFIX + "timeline-service.";
+ /**
+ * Comma seperated list of names for UIs hosted in the timeline server
+ * (For pluggable UIs).
+ */
+ public static final String TIMELINE_SERVICE_UI_NAMES =
+ TIMELINE_SERVICE_PREFIX + "ui-names";
+
+ /** Relative web path that will serve up this UI (For pluggable UIs). */
+ public static final String TIMELINE_SERVICE_UI_WEB_PATH_PREFIX =
+ TIMELINE_SERVICE_PREFIX + "ui-web-path.";
+
+ /**
+ * Path to war file or static content directory for this UI
+ * (For pluggable UIs).
+ */
+ public static final String TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX =
+ TIMELINE_SERVICE_PREFIX + "ui-on-disk-path.";
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index bda24aa..0c6edad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -166,7 +166,7 @@ public class WebApps {
return this;
}
- public WebApp start(WebApp webapp) {
+ public WebApp build(WebApp webapp) {
if (webapp == null) {
webapp = new WebApp() {
@Override
@@ -271,8 +271,7 @@ public class WebApps {
webapp.setConf(conf);
webapp.setHttpServer(server);
- server.start();
- LOG.info("Web app /"+ name +" started at "+ server.getConnectorAddress(0).getPort());
+
} catch (ClassNotFoundException e) {
throw new WebAppException("Error starting http server", e);
} catch (IOException e) {
@@ -300,6 +299,19 @@ public class WebApps {
return start(null);
}
+ public WebApp start(WebApp webapp) {
+ WebApp webApp = build(webapp);
+ HttpServer2 httpServer = webApp.httpServer();
+ try {
+ httpServer.start();
+ LOG.info("Web app " + name + " started at "
+ + httpServer.getConnectorAddress(0).getPort());
+ } catch (IOException e) {
+ throw new WebAppException("Error starting http server", e);
+ }
+ return webApp;
+ }
+
private String inferHostClass() {
String thisClass = this.getClass().getName();
Throwable t = new Throwable();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 383e002..5dc4590 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1869,6 +1869,12 @@
</property>
<property>
+ <description>Comma separated list of UIs that will be hosted</description>
+ <name>yarn.timeline-service.ui-names</name>
+ <value></value>
+ </property>
+
+ <property>
<description>
Default maximum number of retries for timeline service client
and value -1 means no limit.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index dd571f8..dfacaf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
@@ -54,6 +55,8 @@ import org.apache.hadoop.yarn.server.timeline.webapp.CrossOriginFilterInitialize
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.mortbay.jetty.servlet.FilterHolder;
+import org.mortbay.jetty.webapp.WebAppContext;
import com.google.common.annotations.VisibleForTesting;
@@ -137,6 +140,12 @@ public class ApplicationHistoryServer extends CompositeService {
return this.ahsClientService;
}
+ @Private
+ @VisibleForTesting
+ int getPort() {
+ return this.webApp.httpServer().getConnectorAddress(0).getPort();
+ }
+
/**
* @return ApplicationTimelineStore
*/
@@ -223,6 +232,7 @@ public class ApplicationHistoryServer extends CompositeService {
timelineStore, new TimelineACLsManager(conf));
}
+ @SuppressWarnings("unchecked")
private void startWebApp() {
Configuration conf = getConfig();
TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager(
@@ -278,14 +288,42 @@ public class ApplicationHistoryServer extends CompositeService {
String bindAddress = WebAppUtils.getWebAppBindURL(conf,
YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
- LOG.info("Instantiating AHSWebApp at " + bindAddress);
try {
+ AHSWebApp ahsWebApp = new AHSWebApp(timelineDataManager, ahsClientService);
webApp =
WebApps
.$for("applicationhistory", ApplicationHistoryClientService.class,
ahsClientService, "ws")
- .with(conf).at(bindAddress).start(
- new AHSWebApp(timelineDataManager, ahsClientService));
+ .with(conf).withAttribute(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS)).at(bindAddress).build(ahsWebApp);
+ HttpServer2 httpServer = webApp.httpServer();
+
+ String[] names = conf.getTrimmedStrings(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES);
+ WebAppContext webAppContext = httpServer.getWebAppContext();
+
+ for (String name : names) {
+ String webPath = conf.get(
+ YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + name);
+ String onDiskPath = conf.get(
+ YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX + name);
+ WebAppContext uiWebAppContext = new WebAppContext();
+ uiWebAppContext.setContextPath(webPath);
+ uiWebAppContext.setWar(onDiskPath);
+ final String[] ALL_URLS = { "/*" };
+ FilterHolder[] filterHolders =
+ webAppContext.getServletHandler().getFilters();
+ for (FilterHolder filterHolder: filterHolders) {
+ if (!"guice".equals(filterHolder.getName())) {
+ HttpServer2.defineFilter(uiWebAppContext, filterHolder.getName(),
+ filterHolder.getClassName(), filterHolder.getInitParameters(),
+ ALL_URLS);
+ }
+ }
+ LOG.info("Hosting " + name + " from " + onDiskPath + " at " + webPath);
+ httpServer.addContext(uiWebAppContext, true);
+ }
+ httpServer.start();
+ LOG.info("Instantiating AHSWebApp at " + getPort());
} catch (Exception e) {
String msg = "AHSWebApp failed to start.";
LOG.error(msg, e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
index 01c309c..8dbccaa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
@@ -28,16 +28,20 @@ import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
-import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.util.HashMap;
import java.util.Map;
@@ -173,4 +177,49 @@ public class TestApplicationHistoryServer {
}
}
+ @Test(timeout = 240000)
+ public void testHostedUIs() throws Exception {
+
+ ApplicationHistoryServer historyServer = new ApplicationHistoryServer();
+ Configuration config = new YarnConfiguration();
+ config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
+ MemoryTimelineStore.class, TimelineStore.class);
+ config.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
+ MemoryTimelineStateStore.class, TimelineStateStore.class);
+ config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ "localhost:0");
+ final String UI1 = "UI1";
+ String connFileStr = "";
+
+ File diskFile = new File("./pom.xml");
+ String diskFileStr = readInputStream(new FileInputStream(diskFile));
+ try {
+ config.set(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES, UI1);
+ config.set(YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + UI1,
+ "/" + UI1);
+ config.set(YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX
+ + UI1, "./");
+ historyServer.init(config);
+ historyServer.start();
+ URL url = new URL("http://localhost:" + historyServer.getPort() + "/"
+ + UI1 + "/pom.xml");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.connect();
+ assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ connFileStr = readInputStream(conn.getInputStream());
+ } finally {
+ historyServer.stop();
+ }
+ assertEquals("Web file contents should be the same as on disk contents",
+ diskFileStr, connFileStr);
+ }
+ private String readInputStream(InputStream input) throws Exception {
+ ByteArrayOutputStream data = new ByteArrayOutputStream();
+ byte[] buffer = new byte[512];
+ int read;
+ while ((read = input.read(buffer)) >= 0) {
+ data.write(buffer, 0, read);
+ }
+ return new String(data.toByteArray(), "UTF-8");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f8f801/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 1f0388e..2048012 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -185,6 +185,14 @@ selected if this policy is either of `HTTPS_ONLY` or `HTTP_AND_HTTPS`.
| `yarn.timeline-service.client.retry-interval-ms` | The interval in milliseconds between retries for the timeline service client. Defaults to `1000`. |
| `yarn.timeline-service.generic-application-history.max-applications` | The max number of applications could be fetched by using REST API or application history protocol and shown in timeline server web ui. Defaults to `10000`. |
+#### UI Hosting Configuration
+
+The timeline service can host multiple UIs if enabled. The service can support both static web sites hosted in a directory or war files bundled. The web UI is then hosted on the timeline service HTTP port under the path configured.
+| Configuration Property | Description |
+|:---- |:---- |
+| `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be hosted. Defaults to `none`. |
+| `yarn.timeline-service.ui-on-disk-path.$name` | For each of the ui-names, an on disk path should be specified to the directory service static content or the location of a web archive (war file). |
+| `yarn.timeline-service.ui-web-path.$name` | For each of the ui-names, the web path should be specified relative to the Timeline server root. Paths should begin with a starting slash. |
#### Security Configuration