You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2016/12/02 03:18:36 UTC
[24/24] hadoop git commit: MAPREDUCE-6787. Allow job_conf.xml to be
downloadable on the job overview page in JHS (haibochen via rkanter)
MAPREDUCE-6787. Allow job_conf.xml to be downloadable on the job overview page in JHS (haibochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c87b3a44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c87b3a44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c87b3a44
Branch: refs/heads/YARN-5085
Commit: c87b3a448a00df97149a4e93a8c39d9ad0268bdb
Parents: 2d77dc7
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Dec 1 17:29:16 2016 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Dec 1 17:29:38 2016 -0800
----------------------------------------------------------------------
.../mapreduce/v2/app/webapp/AppController.java | 34 ++++++++++++++++++++
.../mapreduce/v2/app/webapp/ConfBlock.java | 2 +-
.../v2/app/webapp/TestAppController.java | 14 ++++++++
.../hadoop/mapreduce/v2/hs/webapp/HsWebApp.java | 2 ++
.../org/apache/hadoop/yarn/webapp/Router.java | 23 ++++++++++---
.../org/apache/hadoop/yarn/webapp/WebApp.java | 13 ++++++++
6 files changed, 83 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c87b3a44/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
index 305ec7e..e30e1b9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
@@ -324,6 +324,40 @@ public class AppController extends Controller implements AMParams {
}
/**
+ * Handle requests to download the job configuration.
+ */
+ public void downloadConf() {
+ try {
+ requireJob();
+ } catch (Exception e) {
+ renderText(e.getMessage());
+ return;
+ }
+ writeJobConf();
+ }
+
+ private void writeJobConf() {
+ String jobId = $(JOB_ID);
+ assert(!jobId.isEmpty());
+
+ JobId jobID = MRApps.toJobID($(JOB_ID));
+ Job job = app.context.getJob(jobID);
+ assert(job != null);
+
+ try {
+ Configuration jobConf = job.loadConfFile();
+ response().setContentType("text/xml");
+ response().setHeader("Content-Disposition",
+ "attachment; filename=" + jobId + ".xml");
+ jobConf.writeXml(writer());
+ } catch (IOException e) {
+ LOG.error("Error reading/writing job" +
+ " conf file for job: " + jobId, e);
+ renderText(e.getMessage());
+ }
+ }
+
+ /**
* Render a BAD_REQUEST error.
* @param s the error message to include.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c87b3a44/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
index 4cb79bf..532c2bd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
@@ -70,7 +70,7 @@ public class ConfBlock extends HtmlBlock {
try {
ConfInfo info = new ConfInfo(job);
- html.div().h3(confPath.toString())._();
+ html.div().a("/jobhistory/downloadconf/" + jid, confPath.toString());
TBODY<TABLE<Hamlet>> tbody = html.
// Tasks table
table("#conf").
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c87b3a44/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
index 92786e3..3f685b0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
@@ -59,6 +60,8 @@ public class TestAppController {
Task task = mock(Task.class);
when(job.getTask(any(TaskId.class))).thenReturn(task);
+ when(job.loadConfFile()).thenReturn(new Configuration());
+ when(job.getConfFile()).thenReturn(new Path("/"));
JobId jobID = MRApps.toJobID("job_01_01");
when(context.getJob(jobID)).thenReturn(job);
@@ -266,6 +269,17 @@ public class TestAppController {
}
/**
+ * Test downloadConf request handling.
+ */
+ @Test
+ public void testDownloadConfiguration() {
+ appController.downloadConf();
+ String jobConfXml = appController.getData();
+ assertTrue("Error downloading the job configuration file.",
+ !jobConfXml.contains("Error"));
+ }
+
+ /**
* Test method 'conf'. Should set AttemptsPage class for rendering or print information about error
*/
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c87b3a44/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java
index ebc6d46..d130910 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java
@@ -50,6 +50,8 @@ public class HsWebApp extends WebApp implements AMParams {
route("/app", HsController.class);
route(pajoin("/job", JOB_ID), HsController.class, "job");
route(pajoin("/conf", JOB_ID), HsController.class, "conf");
+ routeWithoutDefaultView(pajoin("/downloadconf", JOB_ID),
+ HsController.class, "downloadConf");
route(pajoin("/jobcounters", JOB_ID), HsController.class, "jobCounters");
route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME),
HsController.class, "singleJobCounter");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c87b3a44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java
index c46b50e..f2eca04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java
@@ -74,17 +74,32 @@ class Router {
final TreeMap<String, Dest> routes = Maps.newTreeMap(); // path->dest
+ synchronized Dest add(WebApp.HTTP httpMethod, String path,
+ Class<? extends Controller> cls,
+ String action, List<String> names){
+ return addWithOptionalDefaultView(
+ httpMethod, path, cls, action, names, true);
+ }
+
+ synchronized Dest addWithoutDefaultView(WebApp.HTTP httpMethod,
+ String path, Class<? extends Controller> cls, String action,
+ List<String> names){
+ return addWithOptionalDefaultView(httpMethod, path, cls, action,
+ names, false);
+ }
/**
* Add a route to the router.
* e.g., add(GET, "/foo/show", FooController.class, "show", [name...]);
* The name list is from /foo/show/:name/...
*/
- synchronized Dest add(WebApp.HTTP httpMethod, String path,
- Class<? extends Controller> cls,
- String action, List<String> names) {
+ synchronized Dest addWithOptionalDefaultView(WebApp.HTTP httpMethod,
+ String path, Class<? extends Controller> cls,
+ String action, List<String> names, boolean defaultViewNeeded) {
LOG.debug("adding {}({})->{}#{}", new Object[]{path, names, cls, action});
Dest dest = addController(httpMethod, path, cls, action, names);
- addDefaultView(dest);
+ if (defaultViewNeeded) {
+ addDefaultView(dest);
+ }
return dest;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c87b3a44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
index fe800f0..de6a52b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
@@ -210,6 +210,19 @@ public abstract class WebApp extends ServletModule {
res.subList(R_PARAMS, res.size()));
}
+ /**
+ * Setup of a webapp serving route without default views added to the page.
+ * @param pathSpec the path spec in the form of /controller/action/:args etc.
+ * @param cls the controller class
+ * @param action the controller method
+ */
+ public void routeWithoutDefaultView(String pathSpec,
+ Class<? extends Controller> cls, String action) {
+ List<String> res = parseRoute(pathSpec);
+ router.addWithoutDefaultView(HTTP.GET, res.get(R_PATH), cls, action,
+ res.subList(R_PARAMS, res.size()));
+ }
+
public void route(String pathSpec, Class<? extends Controller> cls,
String action) {
route(HTTP.GET, pathSpec, cls, action);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org