You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2018/06/13 01:17:35 UTC
[05/50] hadoop git commit: MapReduce JHS should honor
yarn.webapp.filter-entity-list-by-user(addendum). Contributed by Sunil
Govindan.
MapReduce JHS should honor yarn.webapp.filter-entity-list-by-user(addendum). Contributed by Sunil Govindan.
(cherry picked from commit d5e69d89942fd5dcfafd56bf653565557ff85501)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61ae43f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61ae43f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61ae43f7
Branch: refs/remotes/origin/branch-3.1
Commit: 61ae43f7254c853a5ddef8b6e71524506d56f49e
Parents: f7d0ca7
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Sat Jun 2 09:27:03 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Sat Jun 2 09:57:20 2018 +0530
----------------------------------------------------------------------
.../mapreduce/v2/hs/webapp/HsJobBlock.java | 18 +--------
.../mapreduce/v2/hs/webapp/HsJobsBlock.java | 40 +++++++++++++++++++-
.../mapreduce/v2/hs/webapp/TestBlocks.java | 17 +++++++--
.../mapreduce/v2/hs/webapp/TestHsJobBlock.java | 20 +---------
4 files changed, 56 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61ae43f7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
index 9b845cd..18040f0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
@@ -27,8 +27,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
import java.util.Date;
import java.util.List;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -41,10 +39,8 @@ import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobInfo;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.mapreduce.v2.util.MRApps.TaskAttemptStateUI;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.ResponseInfo;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
@@ -60,14 +56,9 @@ import com.google.inject.Inject;
*/
public class HsJobBlock extends HtmlBlock {
final AppContext appContext;
- private UserGroupInformation ugi;
- private boolean isFilterAppListByUserEnabled;
- @Inject HsJobBlock(Configuration conf, AppContext appctx, ViewContext ctx) {
- super(ctx);
+ @Inject HsJobBlock(AppContext appctx) {
appContext = appctx;
- isFilterAppListByUserEnabled = conf
- .getBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, false);
}
/*
@@ -87,13 +78,6 @@ public class HsJobBlock extends HtmlBlock {
html.p().__("Sorry, ", jid, " not found.").__();
return;
}
- ugi = getCallerUGI();
- if (isFilterAppListByUserEnabled && ugi != null
- && !j.checkAccess(ugi, JobACL.VIEW_JOB)) {
- html.p().__("Sorry, ", jid, " could not be viewed for '",
- ugi.getUserName(), "'.").__();
- return;
- }
if(j instanceof UnparsedJob) {
final int taskCount = j.getTotalMaps() + j.getTotalReduces();
UnparsedJob oversizedJob = (UnparsedJob) j;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61ae43f7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java
index ef563f6..0c7a2df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java
@@ -22,10 +22,15 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobInfo;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
@@ -42,9 +47,19 @@ public class HsJobsBlock extends HtmlBlock {
final AppContext appContext;
final SimpleDateFormat dateFormat =
new SimpleDateFormat("yyyy.MM.dd HH:mm:ss z");
+ private UserGroupInformation ugi;
+ private boolean isFilterAppListByUserEnabled;
+ private boolean areAclsEnabled;
+ private AccessControlList adminAclList;
- @Inject HsJobsBlock(AppContext appCtx) {
+ @Inject
+ HsJobsBlock(Configuration conf, AppContext appCtx, ViewContext ctx) {
+ super(ctx);
appContext = appCtx;
+ isFilterAppListByUserEnabled = conf
+ .getBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, false);
+ areAclsEnabled = conf.getBoolean(MRConfig.MR_ACLS_ENABLED, false);
+ adminAclList = new AccessControlList(conf.get(MRConfig.MR_ADMINS, " "));
}
/*
@@ -77,6 +92,12 @@ public class HsJobsBlock extends HtmlBlock {
StringBuilder jobsTableData = new StringBuilder("[\n");
for (Job j : appContext.getAllJobs().values()) {
JobInfo job = new JobInfo(j);
+ ugi = getCallerUGI();
+ // Allow to list only per-user apps if incoming ugi has permission.
+ if (isFilterAppListByUserEnabled && ugi != null
+ && !checkAccess(job.getUserName())) {
+ continue;
+ }
jobsTableData.append("[\"")
.append(dateFormat.format(new Date(job.getSubmitTime()))).append("\",\"")
.append(job.getFormattedStartTimeStr(dateFormat)).append("\",\"")
@@ -139,4 +160,21 @@ public class HsJobsBlock extends HtmlBlock {
__().
__();
}
+
+ private boolean checkAccess(String userName) {
+ if(!areAclsEnabled) {
+ return true;
+ }
+
+ // User could see its own job.
+ if (ugi.getShortUserName().equals(userName)) {
+ return true;
+ }
+
+ // Admin could also see all jobs
+ if (adminAclList != null && adminAclList.isUserAllowed(ugi)) {
+ return true;
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61ae43f7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java
index d1a704f..27c412b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java
@@ -59,6 +59,8 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.webapp.Controller.RequestContext;
+import org.apache.hadoop.yarn.webapp.View.ViewContext;
+import org.apache.hadoop.yarn.webapp.Controller;
import org.apache.hadoop.yarn.webapp.Params;
import org.apache.hadoop.yarn.webapp.View;
import org.apache.hadoop.yarn.webapp.log.AggregatedLogsPage;
@@ -223,7 +225,14 @@ public class TestBlocks {
jobs.put(job.getID(), job);
when(ctx.getAllJobs()).thenReturn(jobs);
- HsJobsBlock block = new HsJobsBlockForTest(ctx);
+ Controller.RequestContext rc = mock(Controller.RequestContext.class);
+ ViewContext view = mock(ViewContext.class);
+ HttpServletRequest req =mock(HttpServletRequest.class);
+ when(rc.getRequest()).thenReturn(req);
+ when(view.requestContext()).thenReturn(rc);
+
+ Configuration conf = new Configuration();
+ HsJobsBlock block = new HsJobsBlockForTest(conf, ctx, view);
PrintWriter pWriter = new PrintWriter(data);
Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);
block.render(html);
@@ -400,8 +409,10 @@ public class TestBlocks {
}
private class HsJobsBlockForTest extends HsJobsBlock {
- HsJobsBlockForTest(AppContext appCtx) {
- super(appCtx);
+
+ HsJobsBlockForTest(Configuration conf, AppContext appCtx,
+ ViewContext view) {
+ super(conf, appCtx, view);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61ae43f7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
index 48e3d3b..7fa238e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
@@ -33,10 +33,8 @@ import org.apache.hadoop.mapreduce.v2.hs.UnparsedJob;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.StringHelper;
-import org.apache.hadoop.yarn.webapp.Controller;
import org.apache.hadoop.yarn.webapp.ResponseInfo;
import org.apache.hadoop.yarn.webapp.SubView;
-import org.apache.hadoop.yarn.webapp.View.ViewContext;
import org.apache.hadoop.yarn.webapp.view.BlockForTest;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.HtmlBlockForTest;
@@ -51,8 +49,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
-import javax.servlet.http.HttpServletRequest;
-
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -73,13 +69,7 @@ public class TestHsJobBlock {
new JobHistoryStubWithAllOversizeJobs(maxAllowedTaskNum);
jobHistory.init(config);
- Controller.RequestContext rc = mock(Controller.RequestContext.class);
- ViewContext view = mock(ViewContext.class);
- HttpServletRequest req =mock(HttpServletRequest.class);
- when(rc.getRequest()).thenReturn(req);
- when(view.requestContext()).thenReturn(rc);
-
- HsJobBlock jobBlock = new HsJobBlock(config, jobHistory, view) {
+ HsJobBlock jobBlock = new HsJobBlock(jobHistory) {
// override this so that job block can fetch a job id.
@Override
public Map<String, String> moreParams() {
@@ -111,13 +101,7 @@ public class TestHsJobBlock {
JobHistory jobHistory = new JobHitoryStubWithAllNormalSizeJobs();
jobHistory.init(config);
- Controller.RequestContext rc = mock(Controller.RequestContext.class);
- ViewContext view = mock(ViewContext.class);
- HttpServletRequest req =mock(HttpServletRequest.class);
- when(rc.getRequest()).thenReturn(req);
- when(view.requestContext()).thenReturn(rc);
-
- HsJobBlock jobBlock = new HsJobBlock(config, jobHistory, view) {
+ HsJobBlock jobBlock = new HsJobBlock(jobHistory) {
// override this so that the job block can fetch a job id.
@Override
public Map<String, String> moreParams() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org