You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ar...@apache.org on 2013/12/03 18:33:47 UTC
svn commit: r1547492 - in
/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/t...
Author: arp
Date: Tue Dec 3 17:33:46 2013
New Revision: 1547492
URL: http://svn.apache.org/r1547492
Log:
Merging r1547121 through r1547473 from trunk to branch HDFS-2832
Added:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
- copied unchanged from r1547473, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderJobs.java
- copied unchanged from r1547473, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderJobs.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderJobs.java
- copied unchanged from r1547473, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderJobs.java
Removed:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1547121-1547473
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt?rev=1547492&r1=1547491&r2=1547492&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt Tue Dec 3 17:33:46 2013
@@ -137,7 +137,7 @@ Trunk (Unreleased)
MAPREDUCE-5191. TestQueue#testQueue fails with timeout on Windows. (Ivan
Mitic via hitesh)
-Release 2.3.0 - UNRELEASED
+Release 2.4.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -226,7 +226,7 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-5631. TestJobEndNotifier.testNotifyRetries fails with Should
have taken more than 5 seconds in jdk7 (Jonathan Eagles via jlowe)
-Release 2.2.1 - UNRELEASED
+Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -1512,6 +1512,9 @@ Release 0.23.10 - UNRELEASED
IMPROVEMENTS
+ MAPREDUCE-5640. Rename TestLineRecordReader in jobclient module (Jason Lowe
+ via jeagles)
+
OPTIMIZATIONS
MAPREDUCE-1981. Improve getSplits performance by using listLocatedStatus
Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1544304-1547473
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java?rev=1547492&r1=1547491&r2=1547492&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java Tue Dec 3 17:33:46 2013
@@ -20,17 +20,21 @@ package org.apache.hadoop.mapreduce.v2.h
import java.io.IOException;
+import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
import javax.ws.rs.core.UriInfo;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
@@ -55,11 +59,13 @@ import org.apache.hadoop.mapreduce.v2.hs
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobInfo;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.hadoop.yarn.webapp.WebApp;
+import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
@Path("/ws/v1/history")
@@ -78,11 +84,31 @@ public class HsWebServices {
this.webapp = webapp;
}
+ private boolean hasAccess(Job job, HttpServletRequest request) {
+ String remoteUser = request.getRemoteUser();
+ if (remoteUser != null) {
+ return job.checkAccess(UserGroupInformation.createRemoteUser(remoteUser),
+ JobACL.VIEW_JOB);
+ }
+ return true;
+ }
+
+ private void checkAccess(Job job, HttpServletRequest request) {
+ if (!hasAccess(job, request)) {
+ throw new WebApplicationException(Status.UNAUTHORIZED);
+ }
+ }
+
private void init() {
//clear content type
response.setContentType(null);
}
+ @VisibleForTesting
+ void setResponse(HttpServletResponse response) {
+ this.response = response;
+ }
+
@GET
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public HistoryInfo get() {
@@ -190,10 +216,12 @@ public class HsWebServices {
@GET
@Path("/mapreduce/jobs/{jobid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public JobInfo getJob(@PathParam("jobid") String jid) {
+ public JobInfo getJob(@Context HttpServletRequest hsr,
+ @PathParam("jobid") String jid) {
init();
Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
+ checkAccess(job, hsr);
return new JobInfo(job);
}
@@ -217,20 +245,24 @@ public class HsWebServices {
@GET
@Path("/mapreduce/jobs/{jobid}/counters")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public JobCounterInfo getJobCounters(@PathParam("jobid") String jid) {
+ public JobCounterInfo getJobCounters(@Context HttpServletRequest hsr,
+ @PathParam("jobid") String jid) {
init();
Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
+ checkAccess(job, hsr);
return new JobCounterInfo(this.ctx, job);
}
@GET
@Path("/mapreduce/jobs/{jobid}/conf")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public ConfInfo getJobConf(@PathParam("jobid") String jid) {
+ public ConfInfo getJobConf(@Context HttpServletRequest hsr,
+ @PathParam("jobid") String jid) {
init();
Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
+ checkAccess(job, hsr);
ConfInfo info;
try {
info = new ConfInfo(job);
@@ -244,11 +276,12 @@ public class HsWebServices {
@GET
@Path("/mapreduce/jobs/{jobid}/tasks")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public TasksInfo getJobTasks(@PathParam("jobid") String jid,
- @QueryParam("type") String type) {
+ public TasksInfo getJobTasks(@Context HttpServletRequest hsr,
+ @PathParam("jobid") String jid, @QueryParam("type") String type) {
init();
Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
+ checkAccess(job, hsr);
TasksInfo allTasks = new TasksInfo();
for (Task task : job.getTasks().values()) {
TaskType ttype = null;
@@ -270,11 +303,12 @@ public class HsWebServices {
@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public TaskInfo getJobTask(@PathParam("jobid") String jid,
- @PathParam("taskid") String tid) {
+ public TaskInfo getJobTask(@Context HttpServletRequest hsr,
+ @PathParam("jobid") String jid, @PathParam("taskid") String tid) {
init();
Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
+ checkAccess(job, hsr);
Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
return new TaskInfo(task);
@@ -284,10 +318,12 @@ public class HsWebServices {
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/counters")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public JobTaskCounterInfo getSingleTaskCounters(
- @PathParam("jobid") String jid, @PathParam("taskid") String tid) {
+ @Context HttpServletRequest hsr, @PathParam("jobid") String jid,
+ @PathParam("taskid") String tid) {
init();
Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
+ checkAccess(job, hsr);
TaskId taskID = MRApps.toTaskID(tid);
if (taskID == null) {
throw new NotFoundException("taskid " + tid + " not found or invalid");
@@ -302,12 +338,13 @@ public class HsWebServices {
@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public TaskAttemptsInfo getJobTaskAttempts(@PathParam("jobid") String jid,
- @PathParam("taskid") String tid) {
+ public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
+ @PathParam("jobid") String jid, @PathParam("taskid") String tid) {
init();
TaskAttemptsInfo attempts = new TaskAttemptsInfo();
Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
+ checkAccess(job, hsr);
Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
for (TaskAttempt ta : task.getAttempts().values()) {
if (ta != null) {
@@ -324,11 +361,13 @@ public class HsWebServices {
@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public TaskAttemptInfo getJobTaskAttemptId(@PathParam("jobid") String jid,
- @PathParam("taskid") String tid, @PathParam("attemptid") String attId) {
+ public TaskAttemptInfo getJobTaskAttemptId(@Context HttpServletRequest hsr,
+ @PathParam("jobid") String jid, @PathParam("taskid") String tid,
+ @PathParam("attemptid") String attId) {
init();
Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
+ checkAccess(job, hsr);
Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
TaskAttempt ta = AMWebServices.getTaskAttemptFromTaskAttemptString(attId,
task);
@@ -343,11 +382,12 @@ public class HsWebServices {
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}/counters")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public JobTaskAttemptCounterInfo getJobTaskAttemptIdCounters(
- @PathParam("jobid") String jid, @PathParam("taskid") String tid,
- @PathParam("attemptid") String attId) {
+ @Context HttpServletRequest hsr, @PathParam("jobid") String jid,
+ @PathParam("taskid") String tid, @PathParam("attemptid") String attId) {
init();
Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
+ checkAccess(job, hsr);
Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
TaskAttempt ta = AMWebServices.getTaskAttemptFromTaskAttemptString(attId,
task);