You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2011/01/28 22:32:49 UTC
svn commit: r1064894 - in /hadoop/mapreduce/branches/branch-0.22: ./
src/docs/src/documentation/content/xdocs/ src/java/
src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapreduce/
src/test/mapred/org/apache/hadoop/mapred/
Author: shv
Date: Fri Jan 28 21:32:48 2011
New Revision: 1064894
URL: http://svn.apache.org/viewvc?rev=1064894&view=rev
Log:
MAPREDUCE-1754. Merge -r 1064891:1064892 from trunk to branch 0.22.
Modified:
hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java
hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java
hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Fri Jan 28 21:32:48 2011
@@ -440,6 +440,9 @@ Release 0.22.0 - Unreleased
MAPREDUCE-2283. Add timeout for Raid Tests (Ramkumar Vadali via schen)
+ MAPREDUCE-1754. Replace mapred.persmissions.supergroup with an
+ acl : mapreduce.cluster.administrators (Amareshwari Sriramadasu via shv)
+
Release 0.21.1 - Unreleased
NEW FEATURES
Modified: hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml Fri Jan 28 21:32:48 2011
@@ -1778,8 +1778,8 @@
nobody is given access in these properties.</p>
<p>However, irrespective of the job ACLs configured, a job's owner,
- the user who started the cluster and members of an admin configured
- supergroup (<code>mapreduce.cluster.permissions.supergroup</code>)
+ the user who started the cluster and cluster administrators
+ (<code>mapreduce.cluster.administrators</code>)
and queue administrators of the queue to which the job was submitted
to (<code>acl-administer-jobs</code>) always have access to view and
modify a job.</p>
@@ -1809,8 +1809,8 @@
the queue level ACL, "acl-administer-jobs", configured via
mapred-queue-acls.xml. The caller will be able to do the operation
if he/she is part of either queue admins ACL or job modification ACL
- or the user who started the cluster or a member of an admin configured
- supergroup (<code>mapreduce.cluster.permissions.supergroup</code>).
+ or the user who started the cluster or a cluster administrator
+ (<code>mapreduce.cluster.administrators</code>).
</p>
<p>The format of a job level ACL is the same as the format for a
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/mapred-default.xml Fri Jan 28 21:32:48 2011
@@ -967,14 +967,14 @@
job-level ACL.
Irrespective of this ACL configuration, (a) job-owner, (b) the user who
- started the cluster, (c) members of an admin configured supergroup
- configured via mapreduce.cluster.permissions.supergroup and (d) queue
+ started the cluster, (c) cluster administrators
+ configured via mapreduce.cluster.administrators and (d) queue
administrators of the queue to which this job was submitted to configured
via acl-administer-jobs for the specific queue in mapred-queues.xml can
do all the modification operations on a job.
By default, nobody else besides job-owner, the user who started the cluster,
- members of supergroup and queue administrators can perform modification
+ cluster administrators and queue administrators can perform modification
operations on a job.
</description>
</property>
@@ -1001,14 +1001,14 @@
user, for e.g., JobStatus, JobProfile, list of jobs in the queue, etc.
Irrespective of this ACL configuration, (a) job-owner, (b) the user who
- started the cluster, (c) members of an admin configured supergroup
- configured via mapreduce.cluster.permissions.supergroup and (d) queue
+ started the cluster, (c) cluster administrators
+ configured via mapreduce.cluster.administrators and (d) queue
administrators of the queue to which this job was submitted to configured
via acl-administer-jobs for the specific queue in mapred-queues.xml can
do all the view operations on a job.
By default, nobody else besides job-owner, the user who started the
- cluster, memebers of supergroup and queue administrators can perform
+ cluster, cluster administrators and queue administrators can perform
view operations on a job.
</description>
</property>
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/ACLsManager.java Fri Jan 28 21:32:48 2011
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.AuditLogger.Constants;
@@ -36,10 +38,11 @@ import org.apache.hadoop.security.author
@InterfaceAudience.Private
class ACLsManager {
+ static Log LOG = LogFactory.getLog(ACLsManager.class);
// MROwner(user who started this mapreduce cluster)'s ugi
private final UserGroupInformation mrOwner;
- // members of supergroup are mapreduce cluster administrators
- private final String superGroup;
+ // mapreduce cluster administrators
+ private final AccessControlList adminAcl;
private final JobACLsManager jobACLsManager;
private final QueueManager queueManager;
@@ -50,8 +53,15 @@ class ACLsManager {
QueueManager queueManager) throws IOException {
mrOwner = UserGroupInformation.getCurrentUser();
- superGroup = conf.get(MRConfig.MR_SUPERGROUP, "supergroup");
+ adminAcl = new AccessControlList(conf.get(MRConfig.MR_ADMINS, " "));
+ adminAcl.addUser(mrOwner.getShortUserName());
+ String deprecatedSuperGroup = conf.get(MRConfig.MR_SUPERGROUP);
+ if (deprecatedSuperGroup != null) {
+ LOG.warn(MRConfig.MR_SUPERGROUP + " is deprecated. Use "
+ + MRConfig.MR_ADMINS + " instead");
+ adminAcl.addGroup(deprecatedSuperGroup);
+ }
aclsEnabled = conf.getBoolean(MRConfig.MR_ACLS_ENABLED, false);
this.jobACLsManager = jobACLsManager;
@@ -62,8 +72,8 @@ class ACLsManager {
return mrOwner;
}
- String getSuperGroup() {
- return superGroup;
+ AccessControlList getAdminsAcl() {
+ return adminAcl;
}
JobACLsManager getJobACLsManager() {
@@ -72,20 +82,13 @@ class ACLsManager {
/**
* Is the calling user an admin for the mapreduce cluster ?
- * i.e. either cluster owner or member of the supergroup
- * mapreduce.cluster.permissions.supergroup.
+ * i.e. either cluster owner or cluster administrator
* @return true, if user is an admin
*/
boolean isMRAdmin(UserGroupInformation callerUGI) {
- if (mrOwner.getShortUserName().equals(callerUGI.getShortUserName())) {
+ if (adminAcl.isUserAllowed(callerUGI)) {
return true;
}
- String[] groups = callerUGI.getGroupNames();
- for(int i=0; i < groups.length; ++i) {
- if (groups[i].equals(superGroup)) {
- return true;
- }
- }
return false;
}
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobACLsManager.java Fri Jan 28 21:32:48 2011
@@ -62,7 +62,7 @@ class JobACLsManager {
String aclConfigured = conf.get(aclConfigName);
if (aclConfigured == null) {
// If ACLs are not configured at all, we grant no access to anyone. So
- // jobOwner and superuser/supergroup _only_ can do 'stuff'
+ // jobOwner and cluster administrator _only_ can do 'stuff'
aclConfigured = " ";
}
acls.put(aclName, new AccessControlList(aclConfigured));
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Jan 28 21:32:48 2011
@@ -1444,8 +1444,7 @@ public class JobTracker implements MRCon
aclsManager = new ACLsManager(conf, new JobACLsManager(conf), queueManager);
LOG.info("Starting jobtracker with owner as " +
- getMROwner().getShortUserName() + " and supergroup as " +
- getSuperGroup());
+ getMROwner().getShortUserName());
// Create the scheduler
Class<? extends TaskScheduler> schedulerClass
@@ -1483,7 +1482,7 @@ public class JobTracker implements MRCon
int tmpInfoPort = infoSocAddr.getPort();
this.startTime = clock.getTime();
infoServer = new HttpServer("job", infoBindAddress, tmpInfoPort,
- tmpInfoPort == 0, conf);
+ tmpInfoPort == 0, conf, aclsManager.getAdminsAcl());
infoServer.setAttribute("job.tracker", this);
// initialize history parameters.
jobHistory = new JobHistory();
@@ -4139,9 +4138,9 @@ public class JobTracker implements MRCon
public synchronized void refreshNodes() throws IOException {
String user = UserGroupInformation.getCurrentUser().getShortUserName();
// check access
- if (!isMRAdmin(UserGroupInformation.getCurrentUser())) {
+ if (!aclsManager.isMRAdmin(UserGroupInformation.getCurrentUser())) {
AuditLogger.logFailure(user, Constants.REFRESH_NODES,
- getMROwner() + " " + getSuperGroup(), Constants.JOBTRACKER,
+ aclsManager.getAdminsAcl().toString(), Constants.JOBTRACKER,
Constants.UNAUTHORIZED_USER);
throw new AccessControlException(user +
" is not authorized to refresh nodes.");
@@ -4156,14 +4155,6 @@ public class JobTracker implements MRCon
return aclsManager.getMROwner();
}
- String getSuperGroup() {
- return aclsManager.getSuperGroup();
- }
-
- boolean isMRAdmin(UserGroupInformation ugi) {
- return aclsManager.isMRAdmin(ugi);
- }
-
private synchronized void refreshHosts() throws IOException {
// Reread the config to get HOSTS and HOSTS_EXCLUDE filenames.
// Update the file names and refresh internal includes and excludes list
@@ -4604,8 +4595,7 @@ public class JobTracker implements MRCon
aclsManager = new ACLsManager(conf, new JobACLsManager(conf), queueManager);
LOG.info("Starting jobtracker with owner as " +
- getMROwner().getShortUserName() + " and supergroup as " +
- getSuperGroup());
+ getMROwner().getShortUserName());
// Create the scheduler
Class<? extends TaskScheduler> schedulerClass
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskLogServlet.java Fri Jan 28 21:32:48 2011
@@ -121,7 +121,7 @@ public class TaskLogServlet extends Http
/**
* Validates if the given user has job view permissions for this job.
* conf contains jobOwner and job-view-ACLs.
- * We allow jobOwner, superUser(i.e. mrOwner) and members of superGroup and
+ * We allow jobOwner, superUser(i.e. mrOwner) and cluster administrators and
* users and groups specified in configuration using
* mapreduce.job.acl-view-job to view job.
*/
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskTracker.java Fri Jan 28 21:32:48 2011
@@ -589,6 +589,11 @@ public class TaskTracker
}
}
+
+ int getHttpPort() {
+ return httpPort;
+ }
+
/**
* Do the real constructor work here. It's in a separate method
* so we can call it again and "recycle" the object after calling
@@ -596,10 +601,8 @@ public class TaskTracker
*/
synchronized void initialize() throws IOException, InterruptedException {
- aclsManager = new ACLsManager(fConf, new JobACLsManager(fConf), null);
LOG.info("Starting tasktracker with owner as " +
- getMROwner().getShortUserName() + " and supergroup as " +
- getSuperGroup());
+ aclsManager.getMROwner().getShortUserName());
localFs = FileSystem.getLocal(fConf);
// use configured nameserver & interface to get local hostname
@@ -739,18 +742,6 @@ public class TaskTracker
fConf.getBoolean(TT_OUTOFBAND_HEARBEAT, false);
}
- UserGroupInformation getMROwner() {
- return aclsManager.getMROwner();
- }
-
- String getSuperGroup() {
- return aclsManager.getSuperGroup();
- }
-
- boolean isMRAdmin(UserGroupInformation ugi) {
- return aclsManager.isMRAdmin(ugi);
- }
-
/**
* Are ACLs for authorization checks enabled on the MR cluster ?
*/
@@ -1355,13 +1346,14 @@ public class TaskTracker
fConf = conf;
maxMapSlots = conf.getInt(TT_MAP_SLOTS, 2);
maxReduceSlots = conf.getInt(TT_REDUCE_SLOTS, 2);
+ aclsManager = new ACLsManager(fConf, new JobACLsManager(fConf), null);
this.jobTrackAddr = JobTracker.getAddress(conf);
InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(
conf.get(TT_HTTP_ADDRESS, "0.0.0.0:50060"));
String httpBindAddress = infoSocAddr.getHostName();
int httpPort = infoSocAddr.getPort();
this.server = new HttpServer("task", httpBindAddress, httpPort,
- httpPort == 0, conf);
+ httpPort == 0, conf, aclsManager.getAdminsAcl());
workerThreads = conf.getInt(TT_HTTP_THREADS, 40);
this.shuffleServerMetrics = new ShuffleServerMetrics(conf);
server.setThreads(1, workerThreads);
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapreduce/MRConfig.java Fri Jan 28 21:32:48 2011
@@ -38,6 +38,9 @@ public interface MRConfig {
public static final String REDUCEMEMORY_MB =
"mapreduce.cluster.reducememory.mb";
public static final String MR_ACLS_ENABLED = "mapreduce.cluster.acls.enabled";
+ public static final String MR_ADMINS =
+ "mapreduce.cluster.administrators";
+ @Deprecated
public static final String MR_SUPERGROUP =
"mapreduce.cluster.permissions.supergroup";
Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java Fri Jan 28 21:32:48 2011
@@ -59,7 +59,7 @@ public class TestNodeRefresh extends Tes
private JobTracker jt = null;
private String[] hosts = null;
private String[] trackerHosts = null;
- private UserGroupInformation owner, user1, user2, user3, user4;
+ private UserGroupInformation owner, user1, user2, user3, user4, user5;
private static final Log LOG =
LogFactory.getLog(TestNodeRefresh.class);
@@ -82,6 +82,8 @@ public class TestNodeRefresh extends Tes
new String[] {"abc"});
user4= UserGroupInformation.createUserForTesting("user4",
new String[] {"supergroup"});
+ user5= UserGroupInformation.createUserForTesting("user5",
+ new String[] {"user5"});
conf.setBoolean("dfs.replication.considerLoad", false);
// prepare hosts info
@@ -152,7 +154,7 @@ public class TestNodeRefresh extends Tes
/**
* Check default value of HOSTS_EXCLUDE. Also check if only
- * owner/supergroup user is allowed to this command.
+ * owner is allowed to this command.
*/
public void testMRRefreshDefault() throws IOException {
// start a cluster with 2 hosts and no exclude-hosts file
@@ -182,14 +184,14 @@ public class TestNodeRefresh extends Tes
assertTrue("Privileged user denied permission for refresh operation",
success);
- // refresh with super user
+ // refresh with invalid user
success = false;
client = getClient(conf, user4);
try {
client.refreshNodes();
success = true;
} catch (IOException ioe){}
- assertTrue("Super user denied permission for refresh operation",
+ assertFalse("Invalid user performed privileged refresh operation",
success);
// check the cluster status and tracker size
@@ -210,13 +212,15 @@ public class TestNodeRefresh extends Tes
}
/**
- * Check refresh with a specific user is set in the conf along with supergroup
+ * Check refresh with a specific user/group set in the conf
*/
public void testMRSuperUsers() throws IOException {
- // start a cluster with 1 host and specified superuser and supergroup
+ // start a cluster with 1 host and specified cluster administrators
Configuration conf = new Configuration();
// set the supergroup
- conf.set(MRConfig.MR_SUPERGROUP, "abc");
+ conf.set(MRConfig.MR_SUPERGROUP, "supergroup");
+ // set the admin acl
+ conf.set(MRConfig.MR_ADMINS, "user5 abc");
startCluster(2, 1, 0, UserGroupInformation.createRemoteUser("user1"), conf);
conf = mr.createJobConf(new JobConf(conf));
@@ -241,22 +245,42 @@ public class TestNodeRefresh extends Tes
assertTrue("Privileged user denied permission for refresh operation",
success);
- // refresh with super user
+ // refresh with admin group
success = false;
client = getClient(conf, user3);
try {
client.refreshNodes();
success = true;
} catch (IOException ioe){}
- assertTrue("Super user denied permission for refresh operation",
+ assertTrue("Admin group member denied permission for refresh operation",
success);
+ // refresh with admin user
+ success = false;
+ client = getClient(conf, user5);
+ try {
+ client.refreshNodes();
+ success = true;
+ } catch (IOException ioe){}
+ assertTrue("Admin user denied permission for refresh operation",
+ success);
+
+ // refresh with deprecated super group member
+ success = false;
+ client = getClient(conf, user4);
+ try {
+ client.refreshNodes();
+ success = true;
+ } catch (IOException ioe){}
+ assertTrue("Deprecated Super group member denied permission for refresh" +
+ " operation", success);
+
stopCluster();
}
/**
* Check node refresh for decommissioning. Check if an allowed host is
- * disallowed upon refresh. Also check if only owner/supergroup user is
+ * disallowed upon refresh. Also check if only owner/cluster administrator is
* allowed to fire this command.
*/
public void testMRRefreshDecommissioning() throws IOException {
Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java Fri Jan 28 21:32:48 2011
@@ -64,6 +64,10 @@ public class TestQueueManagerWithJobTrac
deleteQueuesConfigFile();
}
+ String adminUser = "adminUser";
+ String adminGroup = "adminGroup";
+ String deprecatedSuperGroup = "superGroup";
+
private void startCluster(boolean aclsEnabled)
throws Exception {
@@ -74,6 +78,8 @@ public class TestQueueManagerWithJobTrac
conf = new Configuration();
conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
conf.setBoolean(MRConfig.MR_ACLS_ENABLED, aclsEnabled);
+ conf.set(MRConfig.MR_SUPERGROUP, deprecatedSuperGroup);
+ conf.set(MRConfig.MR_ADMINS, adminUser + " " + adminGroup);
JobConf jobConf = new JobConf(conf);
String namenode = "file:///";
@@ -122,6 +128,21 @@ public class TestQueueManagerWithJobTrac
fail("user u1 cannot submit jobs to queue p1:p13");
} catch (Exception e) {
}
+
+ // check access to admins
+ job = submitSleepJob(0, 0, 0, 0, true, adminUser+ ",g1",
+ "p1" + NAME_SEPARATOR + "p13", conf);
+ assertTrue("Admin user cannot submit jobs to queue p1:p13",
+ job.isSuccessful());
+ job = submitSleepJob(0, 0, 0, 0, true, "u1,"+ adminGroup,
+ "p1" + NAME_SEPARATOR + "p13", conf);
+ assertTrue("Admin group member cannot submit jobs to queue p1:p13",
+ job.isSuccessful());
+ job = submitSleepJob(0, 0, 0, 0, true, "u1,"+ deprecatedSuperGroup,
+ "p1" + NAME_SEPARATOR + "p13", conf);
+ assertTrue("Deprecated super group member cannot submit jobs to queue" +
+ " p1:p13", job.isSuccessful());
+
// check for access to submit the job
try {
job = submitSleepJob(0, 0, 0, 0, false, "u2,g1", "p1" + NAME_SEPARATOR
@@ -241,6 +262,41 @@ public class TestQueueManagerWithJobTrac
assertEquals("job submitted for u1 and queue p1:p11 is not killed.",
cluster.getJob(jobID).getStatus().getState(), (State.KILLED));
}
+ // check kill access to admins
+ ugi =
+ UserGroupInformation.createUserForTesting("adminUser", new String[]{"g3"});
+ checkAccessToKill(tracker, jobConf, ugi);
+
+ ugi =
+ UserGroupInformation.createUserForTesting("u3", new String[]{adminGroup});
+ checkAccessToKill(tracker, jobConf, ugi);
+
+ ugi =
+ UserGroupInformation.createUserForTesting("u3",
+ new String[]{deprecatedSuperGroup});
+ checkAccessToKill(tracker, jobConf, ugi);
+
+ }
+
+ private void checkAccessToKill(JobTracker tracker, final JobConf mrConf,
+ UserGroupInformation killer) throws IOException, InterruptedException,
+ ClassNotFoundException {
+ Job job = submitSleepJob(1, 1, 100, 100, false, "u1,g1",
+ "p1" + NAME_SEPARATOR + "p11", conf);
+ JobID jobID = job.getStatus().getJobID();
+ //Ensure that the jobinprogress is initied before we issue a kill
+ //signal to the job.
+ JobInProgress jip = tracker.getJob(
+ org.apache.hadoop.mapred.JobID.downgrade(jobID));
+ tracker.initJob(jip);
+ Cluster cluster = killer.doAs(new PrivilegedExceptionAction<Cluster>() {
+ public Cluster run() throws IOException {
+ return new Cluster(mrConf);
+ }
+ });
+ cluster.getJob(jobID).killJob();
+ assertEquals("job not killed by " + killer,
+ cluster.getJob(jobID).getStatus().getState(), (State.KILLED));
}
/**
Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=1064894&r1=1064893&r2=1064894&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java Fri Jan 28 21:32:48 2011
@@ -26,6 +26,7 @@ import java.net.HttpURLConnection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -40,6 +41,7 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
import org.apache.hadoop.security.Groups;
@@ -69,6 +71,12 @@ public class TestWebUIAuthorization exte
private static String mrOwner = null;
// member of supergroup
private static final String superGroupMember = "user2";
+ // mrAdmin
+ private static final String mrAdminUser = "user4";
+ // Group for mrAdmins
+ private static final String mrAdminGroup = "admingroup";
+ // member of mrAdminGroup
+ private static final String mrAdminGroupMember = "user5";
// admin of "default" queue
private static final String qAdmin = "user3";
// "colleague1" is there in job-view-acls config
@@ -127,7 +135,7 @@ public class TestWebUIAuthorization exte
* Validates the given jsp/servlet against different user names who
* can(or cannot) view the job.
* (1) jobSubmitter can view the job
- * (2) superGroupMember can view any job
+ * (2) mrAdmin and deprecated superGroupMember can view any job
* (3) mrOwner can view any job
* (4) qAdmins of the queue to which job is submitted to can view any job in
* that queue.
@@ -145,6 +153,12 @@ public class TestWebUIAuthorization exte
assertEquals("Incorrect return code for supergroup-member " +
superGroupMember, HttpURLConnection.HTTP_OK,
getHttpStatusCode(url, superGroupMember, method));
+ assertEquals("Incorrect return code for admin user " +
+ mrAdminUser, HttpURLConnection.HTTP_OK,
+ getHttpStatusCode(url, mrAdminUser, method));
+ assertEquals("Incorrect return code for admingroup-member " +
+ mrAdminGroupMember, HttpURLConnection.HTTP_OK,
+ getHttpStatusCode(url, mrAdminGroupMember, method));
assertEquals("Incorrect return code for MR-owner " + mrOwner,
HttpURLConnection.HTTP_OK, getHttpStatusCode(url, mrOwner, method));
assertEquals("Incorrect return code for queue admin " + qAdmin,
@@ -166,7 +180,8 @@ public class TestWebUIAuthorization exte
/**
* Validates the given jsp/servlet against different user names who
* can(or cannot) modify the job.
- * (1) jobSubmitter, mrOwner, qAdmin and superGroupMember can modify the job.
+ * (1) jobSubmitter, mrOwner, qAdmin, mrAdmin and deprecated superGroupMember
+ * can modify the job.
* But we are not validating this in this method. Let the caller
* explicitly validate this, if needed.
* (2) user mentioned in job-view-acl but not in job-modify-acl cannot
@@ -253,11 +268,36 @@ public class TestWebUIAuthorization exte
}
}
- public void testAuthorizationForJobHistoryPages() throws Exception {
- JobConf conf = new JobConf();
+ static void setupGroupsProvider() throws IOException {
+ Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
MyGroupsProvider.class.getName());
Groups.getUserToGroupsMappingService(conf);
+ MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
+ MyGroupsProvider.mapping.put(viewColleague, Arrays.asList("group2"));
+ MyGroupsProvider.mapping.put(modifyColleague, Arrays.asList("group1"));
+ MyGroupsProvider.mapping.put(unauthorizedUser,
+ Arrays.asList("evilSociety"));
+ MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
+ MyGroupsProvider.mapping.put(mrAdminGroupMember,
+ Arrays.asList(mrAdminGroup));
+ MyGroupsProvider.mapping.put(viewAndModifyColleague,
+ Arrays.asList("group3"));
+ MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
+
+ mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
+ MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
+ new String[] { "group5", "group6" }));
+
+ MyGroupsProvider.mapping.put(jobSubmitter1, Arrays.asList("group7"));
+ MyGroupsProvider.mapping.put(jobSubmitter2, Arrays.asList("group7"));
+ MyGroupsProvider.mapping.put(jobSubmitter3, Arrays.asList("group7"));
+
+ MyGroupsProvider.mapping.put(mrAdminUser, Arrays.asList("group8"));
+ }
+
+ public void testAuthorizationForJobHistoryPages() throws Exception {
+ setupGroupsProvider();
Properties props = new Properties();
props.setProperty("hadoop.http.filter.initializers",
DummyFilterInitializer.class.getName());
@@ -269,18 +309,8 @@ public class TestWebUIAuthorization exte
"historyDoneFolderOnHDFS");
props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
-
- MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
- MyGroupsProvider.mapping.put(viewColleague, Arrays.asList("group2"));
- MyGroupsProvider.mapping.put(modifyColleague, Arrays.asList("group1"));
- MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety"));
- MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
- MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3"));
- MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
-
- mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
- MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
- new String[] { "group5", "group6" }));
+ props.setProperty(MRConfig.MR_ADMINS, mrAdminUser + " " + mrAdminGroup);
+ props.setProperty(JTConfig.JT_RETIREJOBS, "true");
String[] queueNames = {"default"};
String[] submitAclStrings = new String[] { jobSubmitter };
@@ -290,7 +320,7 @@ public class TestWebUIAuthorization exte
MiniMRCluster cluster = getMRCluster();
int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
- conf = new JobConf(cluster.createJobConf());
+ JobConf conf = new JobConf(cluster.createJobConf());
conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague + " group3");
// Let us add group1 and group3 to modify-job-acl. So modifyColleague and
@@ -312,11 +342,16 @@ public class TestWebUIAuthorization exte
org.apache.hadoop.mapreduce.JobID jobid = job.getJobID();
+ // Wait till job retires.
+ for (int i = 0; i < 10 && !job.isRetired(); i++) {
+ UtilsForTests.waitFor(1000);
+ LOG.info("waiting for the job " + jobid + " to retire");
+ }
+ assertTrue("Job did not retire", job.isRetired());
+
String historyFileName = job.getStatus().getHistoryFile();
String jtURL = "http://localhost:" + infoPort;
- // Job will automatically be retired. Now test jsps..
-
// validate access of jobdetails_history.jsp
String jobDetailsJSP =
jtURL + "/jobdetailshistory.jsp?logFile=" + historyFileName;
@@ -448,12 +483,13 @@ public class TestWebUIAuthorization exte
/**
* Starts a sleep job and tries to kill the job using jobdetails.jsp as
* (1) viewColleague (2) unauthorizedUser (3) modifyColleague
- * (4) viewAndModifyColleague (5) mrOwner (6) superGroupMember and
- * (7) jobSubmitter
+ * (4) viewAndModifyColleague (5) mrOwner (6) deprecated superGroupMember
+ * (7) mrAdmin and (8) jobSubmitter
*
* Validates the given jsp/servlet against different user names who
* can(or cannot) do both view and modify on the job.
- * (1) jobSubmitter, mrOwner and superGroupMember can do both view and modify
+ * (1) jobSubmitter, mrOwner, mrAdmin and deprecated superGroupMember can do
+ * both view and modify
* on the job. But we are not validating this in this method. Let the
* caller explicitly validate this, if needed.
* (2) user mentioned in job-view-acls and job-modify-acls can do this
@@ -513,6 +549,10 @@ public class TestWebUIAuthorization exte
confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
superGroupMember);
confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
+ mrAdminGroupMember);
+ confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
+ mrAdminUser);
+ confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
qAdmin);
}
@@ -565,7 +605,7 @@ public class TestWebUIAuthorization exte
conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, " ");
// Let us start 4 jobs as 4 different users(none of these 4 users is
- // mrOwner and none of these users is a member of superGroup). So only
+ // mrOwner and none of these users is a member of mrAdmin/superGroup). Only
// based on the config MRJobConfig.JOB_ACL_MODIFY_JOB being set here
// and the jobSubmitter, killJob on each of the jobs will be succeeded.
@@ -626,10 +666,7 @@ public class TestWebUIAuthorization exte
*/
@Test
public void testWebUIAuthorization() throws Exception {
- JobConf conf = new JobConf();
- conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
- MyGroupsProvider.class.getName());
- Groups.getUserToGroupsMappingService(conf);
+ setupGroupsProvider();
Properties props = new Properties();
props.setProperty("hadoop.http.filter.initializers",
DummyFilterInitializer.class.getName());
@@ -640,22 +677,7 @@ public class TestWebUIAuthorization exte
props.setProperty(JSPUtil.PRIVATE_ACTIONS_KEY, "true");
props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
-
- MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
- MyGroupsProvider.mapping.put(viewColleague, Arrays.asList("group2"));
- MyGroupsProvider.mapping.put(modifyColleague, Arrays.asList("group1"));
- MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety"));
- MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
- MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3"));
- MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
-
- mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
- MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
- new String[] { "group5", "group6" }));
-
- MyGroupsProvider.mapping.put(jobSubmitter1, Arrays.asList("group7"));
- MyGroupsProvider.mapping.put(jobSubmitter2, Arrays.asList("group7"));
- MyGroupsProvider.mapping.put(jobSubmitter3, Arrays.asList("group7"));
+ props.setProperty(MRConfig.MR_ADMINS, mrAdminUser + " " + mrAdminGroup);
String[] queueNames = {"default"};
String[] submitAclStrings = {jobSubmitter + "," + jobSubmitter1 + ","
@@ -667,7 +689,7 @@ public class TestWebUIAuthorization exte
int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
JobConf clusterConf = cluster.createJobConf();
- conf = new JobConf(clusterConf);
+ JobConf conf = new JobConf(clusterConf);
conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague + " group3");
// Let us add group1 and group3 to modify-job-acl. So modifyColleague and
@@ -711,12 +733,14 @@ public class TestWebUIAuthorization exte
validateJobDetailsJSPKillJob(cluster, clusterConf, jtURL);
// validate killJob of jobtracker.jsp as users viewAndModifyColleague,
- // jobSubmitter, mrOwner and superGroupMember
+ // jobSubmitter, mrOwner, mrAdmin and superGroupMember
confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL,
viewAndModifyColleague);
confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, jobSubmitter);
confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrOwner);
confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, superGroupMember);
+ confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrAdminUser);
+ confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrAdminGroupMember);
confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, qAdmin);
// validate killing of multiple jobs using jobtracker jsp and check
@@ -725,6 +749,53 @@ public class TestWebUIAuthorization exte
validateKillMultipleJobs(cluster, conf, jtURL);
}
+ public void testWebUIAuthorizationForCommonServlets() throws Exception {
+ setupGroupsProvider();
+ Properties props = new Properties();
+ props.setProperty("hadoop.http.filter.initializers",
+ DummyFilterInitializer.class.getName());
+ props.setProperty(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
+ "true");
+ props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
+ props.setProperty(MRConfig.MR_ADMINS, mrAdminUser + " " + mrAdminGroup);
+
+ startCluster(true, props);
+ validateCommonServlets(getMRCluster());
+ stopCluster();
+ }
+
+ private void validateCommonServlets(MiniMRCluster cluster)
+ throws IOException {
+ int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
+ String jtURL = "http://localhost:" + infoPort;
+ for (String servlet : new String[] { "logs", "stacks", "logLevel" }) {
+ String url = jtURL + "/" + servlet;
+ checkAccessToCommonServlet(url);
+ }
+ // validate access to common servlets for TaskTracker.
+ String ttURL = "http://localhost:"
+ + cluster.getTaskTrackerRunner(0).getTaskTracker().getHttpPort();
+ for (String servlet : new String[] { "logs", "stacks", "logLevel" }) {
+ String url = ttURL + "/" + servlet;
+ checkAccessToCommonServlet(url);
+ }
+ }
+
+ private void checkAccessToCommonServlet(String url) throws IOException {
+ url = url + "?a=b";
+ assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(url, mrAdminUser,
+ "GET"));
+ assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(url,
+ mrAdminGroupMember, "GET"));
+ assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(url,
+ mrOwner, "GET"));
+ assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(url,
+ superGroupMember, "GET"));
+ // no access for any other user
+ assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(url,
+ jobSubmitter, "GET"));
+ }
+
// validate killJob of jobtracker.jsp
private void validateJobTrackerJSPKillJobAction(
org.apache.hadoop.mapreduce.JobID jobid, String jtURL)
@@ -762,7 +833,8 @@ public class TestWebUIAuthorization exte
"&changeJobPriority=true&setJobPriority="+"HIGH"+"&jobCheckBox=" +
jobid.toString();
validateModifyJob(jobTrackerJSPSetJobPriorityAction, "GET");
- // jobSubmitter, mrOwner, qAdmin and superGroupMember are not validated for
+ // jobSubmitter, mrOwner, qAdmin, mrAdmin and superGroupMember are not
+ // validated for
// job-modify permission in validateModifyJob(). So let us do it
// explicitly here
assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
@@ -770,6 +842,10 @@ public class TestWebUIAuthorization exte
assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
jobTrackerJSPSetJobPriorityAction, superGroupMember, "GET"));
assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
+ jobTrackerJSPSetJobPriorityAction, mrAdminUser, "GET"));
+ assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
+ jobTrackerJSPSetJobPriorityAction, mrAdminGroupMember, "GET"));
+ assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
jobTrackerJSPSetJobPriorityAction, qAdmin, "GET"));
assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
jobTrackerJSPSetJobPriorityAction, mrOwner, "GET"));
@@ -850,5 +926,9 @@ public class TestWebUIAuthorization exte
getHttpStatusCode(jobTrackerJSP, qAdmin, "GET"));
assertEquals(HttpURLConnection.HTTP_OK,
getHttpStatusCode(jobTrackerJSP, superGroupMember, "GET"));
- }
+ assertEquals(HttpURLConnection.HTTP_OK,
+ getHttpStatusCode(jobTrackerJSP, mrAdminUser, "GET"));
+ assertEquals(HttpURLConnection.HTTP_OK,
+ getHttpStatusCode(jobTrackerJSP, mrAdminGroupMember, "GET"));
+ }
}