You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2010/09/17 09:34:40 UTC
svn commit: r998003 [3/3] - in /hadoop/mapreduce/trunk: ./ conf/
src/c++/task-controller/ src/c++/task-controller/tests/
src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/
src/contrib/mumak/src/test/org/apache/hadoop/mapred/ src/docs/src...
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerRefresh.java Fri Sep 17 07:34:39 2010
@@ -18,7 +18,6 @@
package org.apache.hadoop.mapred;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -37,6 +36,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.QueueState;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
+
import org.junit.After;
import org.junit.Test;
@@ -48,19 +49,12 @@ public class TestQueueManagerRefresh {
private static final Log LOG =
LogFactory.getLog(TestQueueManagerRefresh.class);
- String queueConfigPath =
- System.getProperty("test.build.extraconf", "build/test/extraconf");
- File queueConfigFile =
- new File(queueConfigPath, QueueManager.QUEUE_CONF_FILE_NAME);
-
/**
* Remove the configuration file after the test's done.
*/
@After
public void tearDown() {
- if (queueConfigFile.exists()) {
- queueConfigFile.delete();
- }
+ deleteQueuesConfigFile();
}
/**
@@ -96,8 +90,8 @@ public class TestQueueManagerRefresh {
JobQueueInfo[] queues = getSimpleQueueHierarchy();
// write the configuration file
- QueueManagerTestUtils.writeQueueConfigurationFile(
- queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+ writeQueueConfigurationFile(
+ QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
QueueManager qManager = new QueueManager();
@@ -107,8 +101,8 @@ public class TestQueueManagerRefresh {
queues[0].addChild(newQueue);
// Rewrite the configuration file
- QueueManagerTestUtils.writeQueueConfigurationFile(
- queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+ writeQueueConfigurationFile(
+ QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
testRefreshFailureWithChangeOfHierarchy(qManager);
@@ -127,8 +121,8 @@ public class TestQueueManagerRefresh {
JobQueueInfo[] queues = getSimpleQueueHierarchy();
// write the configuration file
- QueueManagerTestUtils.writeQueueConfigurationFile(
- queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+ writeQueueConfigurationFile(
+ QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
QueueManager qManager = new QueueManager();
@@ -137,8 +131,8 @@ public class TestQueueManagerRefresh {
queues[0].removeChild(q2);
// Rewrite the configuration file
- QueueManagerTestUtils.writeQueueConfigurationFile(
- queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+ writeQueueConfigurationFile(
+ QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
testRefreshFailureWithChangeOfHierarchy(qManager);
}
@@ -187,8 +181,8 @@ public class TestQueueManagerRefresh {
JobQueueInfo[] queues = getSimpleQueueHierarchy();
// write the configuration file
- QueueManagerTestUtils.writeQueueConfigurationFile(
- queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+ writeQueueConfigurationFile(
+ QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
QueueManager qManager = new QueueManager();
@@ -226,8 +220,8 @@ public class TestQueueManagerRefresh {
}
// write the configuration file
- QueueManagerTestUtils.writeQueueConfigurationFile(
- queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+ writeQueueConfigurationFile(
+ QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
QueueManager qManager = new QueueManager();
@@ -261,8 +255,8 @@ public class TestQueueManagerRefresh {
JobQueueInfo[] queues = getSimpleQueueHierarchy();
// write the configuration file
- QueueManagerTestUtils.writeQueueConfigurationFile(
- queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
+ writeQueueConfigurationFile(
+ QUEUES_CONFIG_FILE_PATH, new JobQueueInfo[] { queues[0] });
QueueManager qManager = new QueueManager();
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithDeprecatedConf.java Fri Sep 17 07:34:39 2010
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.QueueState;
import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
@@ -48,16 +49,21 @@ import org.apache.hadoop.mapreduce.Sleep
import org.apache.hadoop.security.UserGroupInformation;
import static org.apache.hadoop.mapred.DeprecatedQueueConfigurationParser.*;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
+import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
public class TestQueueManagerWithDeprecatedConf extends TestCase {
- static final Log LOG = LogFactory.getLog(TestQueueManagerWithDeprecatedConf.class);
-
+ static final Log LOG = LogFactory.getLog(
+ TestQueueManagerWithDeprecatedConf.class);
+
+ String submitAcl = QueueACL.SUBMIT_JOB.getAclName();
+ String adminAcl = QueueACL.ADMINISTER_JOBS.getAclName();
public void testMultipleQueues() {
JobConf conf = new JobConf();
- conf.set("mapred.queue.names", "q1,q2,Q3");
+ conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY,
+ "q1,q2,Q3");
QueueManager qMgr = new QueueManager(conf);
Set<String> expQueues = new TreeSet<String>();
expQueues.add("q1");
@@ -68,7 +74,8 @@ public class TestQueueManagerWithDepreca
public void testSchedulerInfo() {
JobConf conf = new JobConf();
- conf.set("mapred.queue.names", "qq1,qq2");
+ conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY,
+ "qq1,qq2");
QueueManager qMgr = new QueueManager(conf);
qMgr.setSchedulerInfo("qq1", "queueInfoForqq1");
qMgr.setSchedulerInfo("qq2", "queueInfoForqq2");
@@ -85,44 +92,50 @@ public class TestQueueManagerWithDepreca
try {
// queue properties with which the cluster is started.
Properties hadoopConfProps = new Properties();
- hadoopConfProps.put("mapred.queue.names", "default,q1,q2");
- hadoopConfProps.put("mapred.acls.enabled", "true");
- UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile);
+ hadoopConfProps.put(DeprecatedQueueConfigurationParser.
+ MAPRED_QUEUE_NAMES_KEY, "default,q1,q2");
+ hadoopConfProps.put(MRConfig.MR_ACLS_ENABLED, "true");
- //properties for mapred-queue-acls.xml
UserGroupInformation ugi =
UserGroupInformation.createRemoteUser("unknownUser");
- hadoopConfProps.put("mapred.queue.default.acl-submit-job", ugi.getUserName());
- hadoopConfProps.put("mapred.queue.q1.acl-submit-job", "u1");
- hadoopConfProps.put("mapred.queue.q2.acl-submit-job", "*");
- hadoopConfProps.put("mapred.queue.default.acl-administer-jobs", ugi.getUserName());
- hadoopConfProps.put("mapred.queue.q1.acl-administer-jobs", "u1");
- hadoopConfProps.put("mapred.queue.q2.acl-administer-jobs", "*");
+ hadoopConfProps.put(toFullPropertyName(
+ "default", submitAcl), ugi.getUserName());
+ hadoopConfProps.put(toFullPropertyName(
+ "q1", submitAcl), "u1");
+ hadoopConfProps.put(toFullPropertyName(
+ "q2", submitAcl), "*");
+ hadoopConfProps.put(toFullPropertyName(
+ "default", adminAcl), ugi.getUserName());
+ hadoopConfProps.put(toFullPropertyName(
+ "q1", adminAcl), "u2");
+ hadoopConfProps.put(toFullPropertyName(
+ "q2", adminAcl), "*");
UtilsForTests.setUpConfigFile(hadoopConfProps, hadoopConfigFile);
Configuration conf = new JobConf();
+ conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
QueueManager queueManager = new QueueManager(conf);
- //Testing access to queue.
+ //Testing job submission access to queues.
assertTrue("User Job Submission failed.",
- queueManager.hasAccess("default", Queue.QueueOperation.
+ queueManager.hasAccess("default", QueueACL.
SUBMIT_JOB, ugi));
assertFalse("User Job Submission failed.",
- queueManager.hasAccess("q1", Queue.QueueOperation.
+ queueManager.hasAccess("q1", QueueACL.
SUBMIT_JOB, ugi));
assertTrue("User Job Submission failed.",
- queueManager.hasAccess("q2", Queue.QueueOperation.
+ queueManager.hasAccess("q2", QueueACL.
SUBMIT_JOB, ugi));
- //Testing the admin acls
+ //Testing the administer-jobs acls
assertTrue("User Job Submission failed.",
- queueManager.hasAccess("default", Queue.QueueOperation.ADMINISTER_JOBS, ugi));
- assertFalse("User Job Submission failed.",
- queueManager.hasAccess("q1", Queue.QueueOperation.
- ADMINISTER_JOBS, ugi));
- assertTrue("User Job Submission failed.",
- queueManager.hasAccess("q2", Queue.QueueOperation.
- ADMINISTER_JOBS, ugi));
-
+ queueManager.hasAccess("default",
+ QueueACL.ADMINISTER_JOBS, ugi));
+ assertFalse("User Job Submission failed.",
+ queueManager.hasAccess("q1", QueueACL.
+ ADMINISTER_JOBS, ugi));
+ assertTrue("User Job Submission failed.",
+ queueManager.hasAccess("q2", QueueACL.
+ ADMINISTER_JOBS, ugi));
} finally {
//Cleanup the configuration files in all cases
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManagerWithJobTracker.java Fri Sep 17 07:34:39 2010
@@ -19,8 +19,8 @@
package org.apache.hadoop.mapred;
import static org.apache.hadoop.mapred.QueueConfigurationParser.NAME_SEPARATOR;
-import static org.apache.hadoop.mapred.QueueManagerTestUtils.CONFIG;
-import static org.apache.hadoop.mapred.QueueManagerTestUtils.checkForConfigFile;
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.QUEUES_CONFIG_FILE_PATH;
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.deleteQueuesConfigFile;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.createAcls;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.createDocument;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.createProperties;
@@ -29,7 +29,6 @@ import static org.apache.hadoop.mapred.Q
import static org.apache.hadoop.mapred.QueueManagerTestUtils.createSimpleDocumentWithAcls;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.createState;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.miniMRCluster;
-import static org.apache.hadoop.mapred.QueueManagerTestUtils.setUpCluster;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.submitSleepJob;
import static org.apache.hadoop.mapred.QueueManagerTestUtils.writeToFile;
import static org.junit.Assert.assertEquals;
@@ -37,7 +36,6 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Properties;
@@ -47,11 +45,12 @@ import org.apache.hadoop.mapred.tools.MR
import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.QueueState;
import org.apache.hadoop.mapreduce.JobStatus.State;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.AfterClass;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@@ -60,21 +59,26 @@ public class TestQueueManagerWithJobTrac
private static Configuration conf;
- @BeforeClass
- public static void setUp() throws Exception {
- checkForConfigFile();
+ @AfterClass
+ public static void tearDown() throws Exception {
+ deleteQueuesConfigFile();
+ }
+
+ private void startCluster(boolean aclsEnabled)
+ throws Exception {
+
+ deleteQueuesConfigFile();
Document doc = createDocument();
- createSimpleDocumentWithAcls(doc, "true");
- writeToFile(doc, CONFIG);
+ createSimpleDocumentWithAcls(doc);
+ writeToFile(doc, QUEUES_CONFIG_FILE_PATH);
conf = new Configuration();
- conf.addResource(CONFIG);
- conf.set("mapred.committer.job.setup.cleanup.needed", "false");
- setUpCluster(conf);
- }
+ conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
+ conf.setBoolean(MRConfig.MR_ACLS_ENABLED, aclsEnabled);
+
+ JobConf jobConf = new JobConf(conf);
+ String namenode = "file:///";
+ miniMRCluster = new MiniMRCluster(0, namenode, 3, null, null, jobConf);
- @AfterClass
- public static void tearDown() throws Exception {
- new File(CONFIG).delete();
}
/**
@@ -83,6 +87,8 @@ public class TestQueueManagerWithJobTrac
*/
@Test(expected = IOException.class)
public void testSubmitJobForStoppedQueue() throws Exception {
+ startCluster(true);
+
submitSleepJob(10, 10, 100, 100, false, null,
"p1" + NAME_SEPARATOR + "p14", conf);
fail("queue p1:p14 is in stopped state and should not accept jobs");
@@ -94,8 +100,10 @@ public class TestQueueManagerWithJobTrac
*/
@Test(expected = IOException.class)
public void testSubmitJobForContainerQueue() throws Exception {
- submitSleepJob(10, 10, 100, 100, false, null, "p1", conf);
- fail("queue p1 is a container queue and cannot have jobs");
+ startCluster(true);
+
+ submitSleepJob(10, 10, 100, 100, false, null, "p1", conf);
+ fail("queue p1 is a container queue and cannot have jobs");
}
/**
@@ -104,12 +112,16 @@ public class TestQueueManagerWithJobTrac
*/
@Test
public void testAclsForSubmitJob() throws Exception {
+ startCluster(true);
+
Job job;
+ try {
// submit job to queue p1:p13 with unspecified acls
job = submitSleepJob(0, 0, 0, 0, true, "u1,g1", "p1" + NAME_SEPARATOR
+ "p13", conf);
- assertTrue("Job submission for u1 failed in queue : p1:p13.",
- job.isSuccessful());
+ fail("user u1 cannot submit jobs to queue p1:p13");
+ } catch (Exception e) {
+ }
// check for access to submit the job
try {
job = submitSleepJob(0, 0, 0, 0, false, "u2,g1", "p1" + NAME_SEPARATOR
@@ -117,11 +129,17 @@ public class TestQueueManagerWithJobTrac
fail("user u2 cannot submit jobs to queue p1:p11");
} catch (Exception e) {
}
- // submit job to queue p1:p11 with acls-submit-job as u1
+ // submit job to queue p1:p11 with acl-submit-job as u1
job = submitSleepJob(0, 0, 0, 0, true, "u1,g1", "p1"
+ NAME_SEPARATOR + "p11", conf);
assertTrue("Job submission for u1 failed in queue : p1:p11.",
job.isSuccessful());
+
+ // submit job to queue p1:p12 with acl-submit-job as *
+ job = submitSleepJob(0, 0, 0, 0, true, "u2,g1", "p1"
+ + NAME_SEPARATOR + "p12", conf);
+ assertTrue("Job submission for u2 failed in queue : p1:p12.",
+ job.isSuccessful());
}
/**
@@ -130,6 +148,8 @@ public class TestQueueManagerWithJobTrac
*/
@Test
public void testAccessToKillJob() throws Exception {
+ startCluster(true);
+
Job job = submitSleepJob(1, 1, 100, 100, false, "u1,g1", "p1"
+ NAME_SEPARATOR + "p11", conf);
final JobConf jobConf = miniMRCluster.createJobConf();
@@ -229,11 +249,13 @@ public class TestQueueManagerWithJobTrac
*/
@Test
public void testSubmitJobsAfterRefresh() throws Exception {
+ startCluster(true);
+
// test for refresh
- checkForConfigFile();
+ deleteQueuesConfigFile();
Document doc = createDocument();
refreshDocument(doc);
- writeToFile(doc, CONFIG);
+ writeToFile(doc, QUEUES_CONFIG_FILE_PATH);
MRAdmin admin = new MRAdmin(miniMRCluster.createJobConf());
admin.run(new String[] { "-refreshQueues" });
try {
@@ -242,15 +264,15 @@ public class TestQueueManagerWithJobTrac
fail("user u1 is not in the submit jobs' list");
} catch (Exception e) {
}
- checkForConfigFile();
+ deleteQueuesConfigFile();
doc = createDocument();
- createSimpleDocumentWithAcls(doc, "true");
- writeToFile(doc, CONFIG);
+ createSimpleDocumentWithAcls(doc);
+ writeToFile(doc, QUEUES_CONFIG_FILE_PATH);
admin.run(new String[] { "-refreshQueues" });
}
private void refreshDocument(Document doc) {
- Element queues = createQueuesNode(doc, "true");
+ Element queues = createQueuesNode(doc);
// Create parent level queue q1.
Element q1 = createQueue(doc, "q1");
@@ -298,12 +320,7 @@ public class TestQueueManagerWithJobTrac
*/
@Test
public void testAclsDisabled() throws Exception {
- checkForConfigFile();
- Document doc = createDocument();
- createSimpleDocumentWithAcls(doc, "false");
- writeToFile(doc, CONFIG);
- MRAdmin admin = new MRAdmin(miniMRCluster.createJobConf());
- admin.run(new String[] { "-refreshQueues" });
+ startCluster(false);
// submit job to queue p1:p11 by any user not in acls-submit-job
Job job = submitSleepJob(0, 0, 0, 0, true, "u2,g1", "p1" + NAME_SEPARATOR
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java Fri Sep 17 07:34:39 2010
@@ -32,6 +32,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.JobTracker.RecoveryManager;
+import org.apache.hadoop.mapreduce.MRConfig;
+
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.createQueuesConfigFile;
+import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.security.UserGroupInformation;
@@ -156,11 +160,9 @@ public class TestRecoveryManager extends
// clean up
FileSystem fs = FileSystem.get(new Configuration());
fs.delete(TEST_DIR, true);
-
+
JobConf conf = new JobConf();
conf.set(JTConfig.JT_JOBHISTORY_BLOCK_SIZE, "1024");
- conf.set(
- DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY, "default");
MiniMRCluster mr = new MiniMRCluster(1, "file:///", 1, null, null, conf);
JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
@@ -239,10 +241,11 @@ public class TestRecoveryManager extends
true);
mr.getJobTrackerConf().setInt(JTConfig.JT_TASKS_PER_JOB, 25);
- mr.getJobTrackerConf().setBoolean("mapred.acls.enabled" , true);
+ mr.getJobTrackerConf().setBoolean(MRConfig.MR_ACLS_ENABLED, true);
+
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
- mr.getJobTrackerConf().set("mapred.queue.default.acl-submit-job",
- ugi.getUserName());
+ mr.getJobTrackerConf().set(toFullPropertyName(
+ "default", QueueACL.SUBMIT_JOB.getAclName()), ugi.getUserName());
// start the jobtracker
LOG.info("Starting jobtracker");
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java Fri Sep 17 07:34:39 2010
@@ -39,12 +39,12 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.server.tasktracker.Localizer;
import org.apache.hadoop.mapreduce.util.MRAsyncDiskService;
+
+import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.mapred.JvmManager.JvmEnv;
import org.apache.hadoop.mapred.TaskController.JobInitializationContext;
@@ -122,12 +122,14 @@ public class TestTaskTrackerLocalization
System.setProperty("hadoop.log.dir", HADOOP_LOG_DIR.getAbsolutePath());
trackerFConf = new JobConf();
+
trackerFConf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
localDirs = new String[numLocalDirs];
for (int i = 0; i < numLocalDirs; i++) {
localDirs[i] = new File(ROOT_MAPRED_LOCAL_DIR, "0_" + i).getPath();
}
trackerFConf.setStrings(MRConfig.LOCAL_DIR, localDirs);
+ trackerFConf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
// Create the job configuration file. Same as trackerConf in this test.
jobConf = new JobConf(trackerFConf);
@@ -139,6 +141,15 @@ public class TestTaskTrackerLocalization
jobConf.setInt(MRJobConfig.USER_LOG_RETAIN_HOURS, 0);
jobConf.setUser(getJobOwner().getShortUserName());
+ String queue = "default";
+ // set job queue name in job conf
+ jobConf.setQueueName(queue);
+ // Set queue admins acl in job conf similar to what JobClient does so that
+ // it goes into job conf also.
+ jobConf.set(toFullPropertyName(queue,
+ QueueACL.ADMINISTER_JOBS.getAclName()),
+ "qAdmin1,qAdmin2 qAdminsGroup1,qAdminsGroup2");
+
Job job = Job.getInstance(jobConf);
String jtIdentifier = "200907202331";
jobId = new JobID(jtIdentifier, 1);
@@ -527,6 +538,37 @@ public class TestTaskTrackerLocalization
.exists());
checkFilePermissions(jobLogDir.toString(), "drwx------", task.getUser(),
taskTrackerUGI.getGroupNames()[0]);
+
+ // Make sure that the job ACLs file job-acls.xml exists in job userlog dir
+ File jobACLsFile = new File(jobLogDir, TaskTracker.jobACLsFile);
+ assertTrue("JobACLsFile is missing in the job userlog dir " + jobLogDir,
+ jobACLsFile.exists());
+
+ // With default task controller, the job-acls.xml file is owned by TT and
+ // permissions are 700
+ checkFilePermissions(jobACLsFile.getAbsolutePath(), "-rwx------",
+ taskTrackerUGI.getShortUserName(), taskTrackerUGI.getGroupNames()[0]);
+
+ validateJobACLsFileContent();
+ }
+
+ // Validate the contents of jobACLsFile ( i.e. user name, job-view-acl, queue
+ // name and queue-admins-acl ).
+ protected void validateJobACLsFileContent() {
+ JobConf jobACLsConf = TaskLogServlet.getConfFromJobACLsFile(jobId);
+
+ assertTrue(jobACLsConf.get("user.name").equals(
+ localizedJobConf.getUser()));
+ assertTrue(jobACLsConf.get(MRJobConfig.JOB_ACL_VIEW_JOB).
+ equals(localizedJobConf.get(MRJobConfig.JOB_ACL_VIEW_JOB)));
+
+ String queue = localizedJobConf.getQueueName();
+ assertTrue(queue.equalsIgnoreCase(jobACLsConf.getQueueName()));
+
+ String qACLName = toFullPropertyName(queue,
+ QueueACL.ADMINISTER_JOBS.getAclName());
+ assertTrue(jobACLsConf.get(qACLName).equals(
+ localizedJobConf.get(qACLName)));
}
/**
@@ -645,24 +687,6 @@ public class TestTaskTrackerLocalization
+ expectedStderr.toString() + " Observed : "
+ attemptLogFiles[1].toString(), expectedStderr.toString().equals(
attemptLogFiles[1].toString()));
-
- // Make sure that the job ACLs file exists in the task log dir
- File jobACLsFile = new File(logDir, TaskRunner.jobACLsFile);
- assertTrue("JobACLsFile is missing in the task log dir " + logDir,
- jobACLsFile.exists());
-
- // With default task controller, the job-acls file is owned by TT and
- // permissions are 700
- checkFilePermissions(jobACLsFile.getAbsolutePath(), "-rwx------",
- taskTrackerUGI.getShortUserName(), taskTrackerUGI.getGroupNames()[0]);
-
- // Validate the contents of jobACLsFile(both user name and job-view-acls)
- Configuration jobACLsConf = TaskLogServlet.getConfFromJobACLsFile(task
- .getTaskID(), task.isTaskCleanupTask());
- assertTrue(jobACLsConf.get(MRJobConfig.USER_NAME).equals(
- localizedJobConf.getUser()));
- assertTrue(jobACLsConf.get(MRJobConfig.JOB_ACL_VIEW_JOB).
- equals(localizedJobConf.get(MRJobConfig.JOB_ACL_VIEW_JOB)));
}
/**
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestUserLogCleanup.java Fri Sep 17 07:34:39 2010
@@ -55,6 +55,7 @@ public class TestUserLogCleanup {
taskLogCleanupThread = new UserLogCleaner(conf);
taskLogCleanupThread.setClock(myClock);
tt = new TaskTracker();
+ tt.setConf(new JobConf(conf));
tt.setLocalizer(localizer);
tt.setTaskLogCleanupThread(taskLogCleanupThread);
}
@@ -67,8 +68,9 @@ public class TestUserLogCleanup {
private File localizeJob(JobID jobid) throws IOException {
File jobUserlog = TaskLog.getJobDir(jobid);
+ JobConf conf = new JobConf();
// localize job log directory
- tt.initializeJobLogDir(jobid);
+ tt.initializeJobLogDir(jobid, conf);
assertTrue(jobUserlog + " directory is not created.", jobUserlog.exists());
return jobUserlog;
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java Fri Sep 17 07:34:39 2010
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
import org.apache.hadoop.security.UserGroupInformation;
@@ -57,12 +59,18 @@ public class TestWebUIAuthorization exte
private static final Log LOG = LogFactory.getLog(
TestWebUIAuthorization.class);
- // user1 submits the jobs
+ // users who submit the jobs
private static final String jobSubmitter = "user1";
+ private static final String jobSubmitter1 = "user11";
+ private static final String jobSubmitter2 = "user12";
+ private static final String jobSubmitter3 = "user13";
+
// mrOwner starts the cluster
private static String mrOwner = null;
// member of supergroup
private static final String superGroupMember = "user2";
+ // admin of "default" queue
+ private static final String qAdmin = "user3";
// "colleague1" is there in job-view-acls config
private static final String viewColleague = "colleague1";
// "colleague2" is there in job-modify-acls config
@@ -72,10 +80,17 @@ public class TestWebUIAuthorization exte
// "evilJohn" is not having view/modify access on the jobs
private static final String unauthorizedUser = "evilJohn";
+ @Override
protected void setUp() throws Exception {
// do not do anything
};
+ @Override
+ protected void tearDown() throws Exception {
+ deleteQueuesConfigFile();
+ super.tearDown();
+ }
+
/** access a url, ignoring some IOException such as the page does not exist */
static int getHttpStatusCode(String urlstring, String userName,
String method) throws IOException {
@@ -112,45 +127,51 @@ public class TestWebUIAuthorization exte
* Validates the given jsp/servlet against different user names who
* can(or cannot) view the job.
* (1) jobSubmitter can view the job
- * (2) superGroupMember can view the job
- * (3) user mentioned in job-view-acls should be able to view the job
- * (4) user mentioned in job-modify-acls but not in job-view-acls
+ * (2) superGroupMember can view any job
+ * (3) mrOwner can view any job
+ * (4) qAdmins of the queue to which job is submitted to can view any job in
+ * that queue.
+ * (5) user mentioned in job-view-acl should be able to view the
+ * job irrespective of job-modify-acl.
+ * (6) user mentioned in job-modify-acl but not in job-view-acl
* cannot view the job
- * (5) other unauthorized users cannot view the job
+ * (7) other unauthorized users cannot view the job
*/
private void validateViewJob(String url, String method)
throws IOException {
- assertEquals("Incorrect return code for " + jobSubmitter,
+ assertEquals("Incorrect return code for job submitter " + jobSubmitter,
HttpURLConnection.HTTP_OK, getHttpStatusCode(url, jobSubmitter,
method));
- assertEquals("Incorrect return code for " + superGroupMember,
- HttpURLConnection.HTTP_OK, getHttpStatusCode(url, superGroupMember,
- method));
- assertEquals("Incorrect return code for " + mrOwner,
+ assertEquals("Incorrect return code for supergroup-member " +
+ superGroupMember, HttpURLConnection.HTTP_OK,
+ getHttpStatusCode(url, superGroupMember, method));
+ assertEquals("Incorrect return code for MR-owner " + mrOwner,
HttpURLConnection.HTTP_OK, getHttpStatusCode(url, mrOwner, method));
- assertEquals("Incorrect return code for " + viewColleague,
- HttpURLConnection.HTTP_OK, getHttpStatusCode(url, viewColleague,
- method));
- assertEquals("Incorrect return code for " + viewAndModifyColleague,
- HttpURLConnection.HTTP_OK, getHttpStatusCode(url,
- viewAndModifyColleague, method));
- assertEquals("Incorrect return code for " + modifyColleague,
- HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(url,
- modifyColleague, method));
- assertEquals("Incorrect return code for " + unauthorizedUser,
- HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode(url,
- unauthorizedUser, method));
+ assertEquals("Incorrect return code for queue admin " + qAdmin,
+ HttpURLConnection.HTTP_OK, getHttpStatusCode(url, qAdmin, method));
+ assertEquals("Incorrect return code for user in job-view-acl " +
+ viewColleague, HttpURLConnection.HTTP_OK,
+ getHttpStatusCode(url, viewColleague, method));
+ assertEquals("Incorrect return code for user in job-view-acl and " +
+ "job-modify-acl " + viewAndModifyColleague, HttpURLConnection.HTTP_OK,
+ getHttpStatusCode(url, viewAndModifyColleague, method));
+ assertEquals("Incorrect return code for user in job-modify-acl " +
+ modifyColleague, HttpURLConnection.HTTP_UNAUTHORIZED,
+ getHttpStatusCode(url, modifyColleague, method));
+ assertEquals("Incorrect return code for unauthorizedUser " +
+ unauthorizedUser, HttpURLConnection.HTTP_UNAUTHORIZED,
+ getHttpStatusCode(url, unauthorizedUser, method));
}
/**
* Validates the given jsp/servlet against different user names who
* can(or cannot) modify the job.
- * (1) jobSubmitter and superGroupMember can modify the job. But we are not
- * validating this in this method. Let the caller explicitly validate
- * this, if needed.
- * (2) user mentioned in job-view-acls but not in job-modify-acls cannot
+ * (1) jobSubmitter, mrOwner, qAdmin and superGroupMember can modify the job.
+ * But we are not validating this in this method. Let the caller
+ * explicitly validate this, if needed.
+ * (2) user mentioned in job-view-acl but not in job-modify-acl cannot
* modify the job
- * (3) user mentioned in job-modify-acls (irrespective of job-view-acls)
+ * (3) user mentioned in job-modify-acl (irrespective of job-view-acl)
* can modify the job
* (4) other unauthorized users cannot modify the job
*/
@@ -240,13 +261,13 @@ public class TestWebUIAuthorization exte
Properties props = new Properties();
props.setProperty("hadoop.http.filter.initializers",
DummyFilterInitializer.class.getName());
- props.setProperty(MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG,
+ props.setProperty(MRConfig.MR_ACLS_ENABLED,
String.valueOf(true));
+
props.setProperty("dfs.permissions.enabled", "false");
props.setProperty("mapred.job.tracker.history.completed.location",
"historyDoneFolderOnHDFS");
- props.setProperty("mapreduce.job.committer.setup.cleanup.needed",
- "false");
+ props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
@@ -255,11 +276,17 @@ public class TestWebUIAuthorization exte
MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety"));
MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3"));
+ MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
+
mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
- new String[] { "group4", "group5" }));
+ new String[] { "group5", "group6" }));
+
+ String[] queueNames = {"default"};
+ String[] submitAclStrings = new String[] { jobSubmitter };
+ String[] adminsAclStrings = new String[] { qAdmin };
+ startCluster(props, queueNames, submitAclStrings, adminsAclStrings);
- startCluster(true, props);
MiniMRCluster cluster = getMRCluster();
int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
@@ -338,30 +365,57 @@ public class TestWebUIAuthorization exte
Integer.toString(attemptsMap.get(attempt).getHttpPort()),
attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR;
validateViewJob(stderrURL, "GET");
+ }
+ }
- // delete job-acls.xml file from the task log dir of attempt and verify
- // if unauthorized users can view task logs of attempt.
- File attemptLogDir = TaskLog.getAttemptDir(
- org.apache.hadoop.mapred.TaskAttemptID.downgrade(attempt), false);
- Path jobACLsFilePath = new Path(attemptLogDir.toString(),
- TaskRunner.jobACLsFile);
- new File(jobACLsFilePath.toUri().getPath()).delete();
+ // For each tip, let us test the effect of deletion of job-acls.xml file and
+ // deletion of task log dir for each of the attempts of the tip.
+
+ // delete job-acls.xml file from the job userlog dir and verify
+ // if unauthorized users can view task logs of each attempt.
+ Path jobACLsFilePath = new Path(TaskLog.getJobDir(jobid).toString(),
+ TaskTracker.jobACLsFile);
+ assertTrue("Could not delete job-acls.xml file.",
+ new File(jobACLsFilePath.toUri().getPath()).delete());
+
+ for (TaskID tip : tipsMap.keySet()) {
+
+ Map<TaskAttemptID, TaskAttemptInfo> attemptsMap =
+ tipsMap.get(tip).getAllTaskAttempts();
+ for (TaskAttemptID attempt : attemptsMap.keySet()) {
+
+ String stdoutURL = TaskLogServlet.getTaskLogUrl("localhost",
+ Integer.toString(attemptsMap.get(attempt).getHttpPort()),
+ attempt.toString()) + "&filter=" + TaskLog.LogName.STDOUT;;
+
+ String stderrURL = TaskLogServlet.getTaskLogUrl("localhost",
+ Integer.toString(attemptsMap.get(attempt).getHttpPort()),
+ attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR;
+
+ // unauthorized users can view task logs of each attempt because
+ // job-acls.xml file is deleted.
assertEquals("Incorrect return code for " + unauthorizedUser,
HttpURLConnection.HTTP_OK, getHttpStatusCode(stdoutURL,
- unauthorizedUser, "GET"));
+ unauthorizedUser, "GET"));
assertEquals("Incorrect return code for " + unauthorizedUser,
HttpURLConnection.HTTP_OK, getHttpStatusCode(stderrURL,
- unauthorizedUser, "GET"));
+ unauthorizedUser, "GET"));
// delete the whole task log dir of attempt and verify that we get
// correct response code (i.e. HTTP_GONE) when task logs are accessed.
+ File attemptLogDir = TaskLog.getAttemptDir(
+ org.apache.hadoop.mapred.TaskAttemptID.downgrade(attempt), false);
FileUtil.fullyDelete(attemptLogDir);
+
+ // Try accessing tasklogs - STDOUT and STDERR now(after the whole
+ // attempt log dir is deleted).
assertEquals("Incorrect return code for " + jobSubmitter,
HttpURLConnection.HTTP_GONE, getHttpStatusCode(stdoutURL,
- jobSubmitter, "GET"));
+ jobSubmitter, "GET"));
+
assertEquals("Incorrect return code for " + jobSubmitter,
HttpURLConnection.HTTP_GONE, getHttpStatusCode(stderrURL,
- jobSubmitter, "GET"));
+ jobSubmitter, "GET"));
}
}
@@ -377,6 +431,21 @@ public class TestWebUIAuthorization exte
}
/**
+ * Creates queues configuration file with the given queues and acls and starts
+ * cluster with that queues configuration file.
+ * @param props configuration properties to inject to the mini cluster
+ * @param queueNames the job queues on the cluster
+ * @param submitAclStrings acl-submit-job acls for all queues
+ * @param adminsAclStrings acl-administer-jobs acls for all queues
+ * @throws Exception
+ */
+ private void startCluster(Properties props, String[] queueNames,
+ String[] submitAclStrings, String[] adminsAclStrings) throws Exception {
+ createQueuesConfigFile(queueNames, submitAclStrings, adminsAclStrings);
+ startCluster(true, props);
+ }
+
+ /**
* Starts a sleep job and tries to kill the job using jobdetails.jsp as
* (1) viewColleague (2) unauthorizedUser (3) modifyColleague
* (4) viewAndModifyColleague (5) mrOwner (6) superGroupMember and
@@ -387,11 +456,13 @@ public class TestWebUIAuthorization exte
* (1) jobSubmitter, mrOwner and superGroupMember can do both view and modify
* on the job. But we are not validating this in this method. Let the
* caller explicitly validate this, if needed.
- * (2) user mentioned in job-view-acls but not in job-modify-acls cannot
+ * (2) user mentioned in job-view-acls and job-modify-acls can do this
+ * (3) user mentioned in job-view-acls but not in job-modify-acls cannot
* do this
- * (3) user mentioned in job-modify-acls but not in job-view-acls cannot
+ * (4) user mentioned in job-modify-acls but not in job-view-acls cannot
* do this
- * (4) other unauthorized users cannot do this
+ * (5) qAdmin cannot do this because he doesn't have view access to the job
+ * (6) other unauthorized users cannot do this
*
* @throws Exception
*/
@@ -419,8 +490,11 @@ public class TestWebUIAuthorization exte
getHttpStatusCode(url, unauthorizedUser, "POST"));
assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
getHttpStatusCode(url, modifyColleague, "POST"));
+
assertEquals(HttpURLConnection.HTTP_OK,
getHttpStatusCode(url, viewAndModifyColleague, "POST"));
+ assertTrue("killJob using jobdetails.jsp failed for a job for which "
+ + "user has job-view and job-modify permissions", job.isComplete());
} finally {
if (!job.isComplete()) {
LOG.info("Killing job " + jobid + " from finally block");
@@ -430,14 +504,16 @@ public class TestWebUIAuthorization exte
}
}
- // check if jobSubmitter, mrOwner and superGroupMember can do killJob
- // using jobdetails.jsp url
+ // Check if jobSubmitter, mrOwner, superGroupMember and queueAdmins
+ // can do killJob using jobdetails.jsp url
+ confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
+ jobSubmitter);
confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
- jobSubmitter);
+ mrOwner);
confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
- mrOwner);
+ superGroupMember);
confirmJobDetailsJSPKillJobAsUser(cluster, conf, jtURL, jobTrackerJSP,
- superGroupMember);
+ qAdmin);
}
/**
@@ -486,47 +562,47 @@ public class TestWebUIAuthorization exte
// jobTrackerJSP killJob url
String url = jobTrackerJSP + "&killJobs=true";
// view-job-acl doesn't matter for killJob from jobtracker jsp page
- conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "");
+ conf.set(MRJobConfig.JOB_ACL_VIEW_JOB, " ");
- // Let us start jobs as 4 different users(none of these 4 users is
+ // Let us start 4 jobs as 4 different users(none of these 4 users is
// mrOwner and none of these users is a member of superGroup). So only
- // based on the config JobContext.JOB_ACL_MODIFY_JOB being set here,
- // killJob on each of the jobs will be succeeded.
+ // based on the config MRJobConfig.JOB_ACL_MODIFY_JOB being set here
+ // and the jobSubmitter, killJob on each of the jobs will be succeeded.
// start 1st job.
// Out of these 4 users, only jobSubmitter can do killJob on 1st job
- conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, "");
+ conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, " ");
Job job1 = startSleepJobAsUser(jobSubmitter, conf);
org.apache.hadoop.mapreduce.JobID jobid = job1.getJobID();
getTIPId(cluster, jobid);// wait till the map task is started
url = url.concat("&jobCheckBox=" + jobid.toString());
// start 2nd job.
- // Out of these 4 users, only viewColleague can do killJob on 2nd job
- Job job2 = startSleepJobAsUser(viewColleague, conf);
+ // Out of these 4 users, only jobSubmitter1 can do killJob on 2nd job
+ Job job2 = startSleepJobAsUser(jobSubmitter1, conf);
jobid = job2.getJobID();
getTIPId(cluster, jobid);// wait till the map task is started
url = url.concat("&jobCheckBox=" + jobid.toString());
// start 3rd job.
- // Out of these 4 users, only modifyColleague can do killJob on 3rd job
- Job job3 = startSleepJobAsUser(modifyColleague, conf);
+ // Out of these 4 users, only jobSubmitter2 can do killJob on 3rd job
+ Job job3 = startSleepJobAsUser(jobSubmitter2, conf);
jobid = job3.getJobID();
getTIPId(cluster, jobid);// wait till the map task is started
url = url.concat("&jobCheckBox=" + jobid.toString());
// start 4rd job.
- // Out of these 4 users, viewColleague and viewAndModifyColleague
+ // Out of these 4 users, jobSubmitter1 and jobSubmitter3
// can do killJob on 4th job
- conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, viewColleague);
- Job job4 = startSleepJobAsUser(viewAndModifyColleague, conf);
+ conf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, jobSubmitter1);
+ Job job4 = startSleepJobAsUser(jobSubmitter3, conf);
jobid = job4.getJobID();
getTIPId(cluster, jobid);// wait till the map task is started
url = url.concat("&jobCheckBox=" + jobid.toString());
try {
- // Try killing all the 4 jobs as user viewColleague who can kill only
+ // Try killing all the 4 jobs as user jobSubmitter1 who can kill only
// 2nd and 4th jobs. Check if 1st and 3rd jobs are not killed and
// 2nd and 4th jobs got killed
assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
- getHttpStatusCode(url, viewColleague, "POST"));
+ getHttpStatusCode(url, jobSubmitter1, "POST"));
assertFalse("killJob succeeded for a job for which user doesnot "
+ " have job-modify permission", job1.isComplete());
assertFalse("killJob succeeded for a job for which user doesnot "
@@ -557,12 +633,12 @@ public class TestWebUIAuthorization exte
Properties props = new Properties();
props.setProperty("hadoop.http.filter.initializers",
DummyFilterInitializer.class.getName());
- props.setProperty(
- MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG, String.valueOf(true));
+ props.setProperty(MRConfig.MR_ACLS_ENABLED, String.valueOf(true));
+
props.setProperty("dfs.permissions.enabled", "false");
props.setProperty(JSPUtil.PRIVATE_ACTIONS_KEY, "true");
- props.setProperty("mapreduce.job.committer.setup.cleanup.needed", "false");
+ props.setProperty(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
props.setProperty(MRConfig.MR_SUPERGROUP, "superGroup");
MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1"));
@@ -571,12 +647,22 @@ public class TestWebUIAuthorization exte
MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety"));
MyGroupsProvider.mapping.put(superGroupMember, Arrays.asList("superGroup"));
MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3"));
+ MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4"));
mrOwner = UserGroupInformation.getCurrentUser().getShortUserName();
MyGroupsProvider.mapping.put(mrOwner, Arrays.asList(
- new String[] { "group4", "group5" }));
+ new String[] { "group5", "group6" }));
+
+ MyGroupsProvider.mapping.put(jobSubmitter1, Arrays.asList("group7"));
+ MyGroupsProvider.mapping.put(jobSubmitter2, Arrays.asList("group7"));
+ MyGroupsProvider.mapping.put(jobSubmitter3, Arrays.asList("group7"));
+
+ String[] queueNames = {"default"};
+ String[] submitAclStrings = {jobSubmitter + "," + jobSubmitter1 + ","
+ + jobSubmitter2 + "," + jobSubmitter3};
+ String[] adminsAclStrings = new String[]{qAdmin};
+ startCluster(props, queueNames, submitAclStrings, adminsAclStrings);
- startCluster(true, props);
MiniMRCluster cluster = getMRCluster();
int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort();
@@ -630,8 +716,8 @@ public class TestWebUIAuthorization exte
viewAndModifyColleague);
confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, jobSubmitter);
confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, mrOwner);
- confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL,
- superGroupMember);
+ confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, superGroupMember);
+ confirmJobTrackerJSPKillJobAsUser(cluster, conf, jtURL, qAdmin);
// validate killing of multiple jobs using jobtracker jsp and check
// if all the jobs which can be killed by user are actually the ones that
@@ -676,7 +762,7 @@ public class TestWebUIAuthorization exte
"&changeJobPriority=true&setJobPriority="+"HIGH"+"&jobCheckBox=" +
jobid.toString();
validateModifyJob(jobTrackerJSPSetJobPriorityAction, "GET");
- // jobSubmitter, mrOwner and superGroupMember are not validated for
+ // jobSubmitter, mrOwner, qAdmin and superGroupMember are not validated for
// job-modify permission in validateModifyJob(). So let us do it
// explicitly here
assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
@@ -684,6 +770,8 @@ public class TestWebUIAuthorization exte
assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
jobTrackerJSPSetJobPriorityAction, superGroupMember, "GET"));
assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
+ jobTrackerJSPSetJobPriorityAction, qAdmin, "GET"));
+ assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(
jobTrackerJSPSetJobPriorityAction, mrOwner, "GET"));
}
@@ -759,6 +847,8 @@ public class TestWebUIAuthorization exte
assertEquals(HttpURLConnection.HTTP_OK,
getHttpStatusCode(jobTrackerJSP, mrOwner, "GET"));
assertEquals(HttpURLConnection.HTTP_OK,
+ getHttpStatusCode(jobTrackerJSP, qAdmin, "GET"));
+ assertEquals(HttpURLConnection.HTTP_OK,
getHttpStatusCode(jobTrackerJSP, superGroupMember, "GET"));
}
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java Fri Sep 17 07:34:39 2010
@@ -26,11 +26,12 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobInProgress;
import org.apache.hadoop.mapred.JobTracker;
import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.Operation;
+import static org.apache.hadoop.mapred.QueueManagerTestUtils.*;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Before;
@@ -56,23 +57,33 @@ public class TestJobACLs {
TestJobACLs.class.getCanonicalName() + Path.SEPARATOR
+ "completed-job-store");
+ private String jobSubmitter = "jobSubmitter";
+ private String viewColleague = "viewColleague";
+ private String modifyColleague = "modifyColleague";
+ private String qAdmin = "qAdmin";
+
/**
* Start the cluster before running the actual test.
*
* @throws IOException
*/
@Before
- public void setup() throws IOException {
+ public void setup() throws Exception {
// Start the cluster
startCluster(false);
}
- private void startCluster(boolean reStart) throws IOException {
- UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser();
+ private void startCluster(boolean reStart) throws Exception {
+
+ // Configure job queues
+ String[] queueNames = {"default"};
+ createQueuesConfigFile(queueNames,
+ new String[] { jobSubmitter }, new String[] { qAdmin });
+
JobConf conf = new JobConf();
- // Enable job-level authorization
- conf.setBoolean(MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG, true);
+ // Enable queue and job level authorization
+ conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
// Enable CompletedJobStore
FileSystem fs = FileSystem.getLocal(conf);
@@ -84,6 +95,7 @@ public class TestJobACLs {
conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, true);
conf.set(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1");
+ UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser();
mr = new MiniMRCluster(0, 0, 1, "file:///", 1, null, null, MR_UGI, conf);
}
@@ -92,6 +104,7 @@ public class TestJobACLs {
*/
@After
public void tearDown() {
+ deleteQueuesConfigFile();
if (mr != null) {
mr.shutdown();
}
@@ -106,10 +119,10 @@ public class TestJobACLs {
* @throws ClassNotFoundException
*/
@Test
- public void testACLS() throws IOException, InterruptedException,
- ClassNotFoundException {
+ public void testACLS() throws Exception {
verifyACLViewJob();
- verifyACLModifyJob();
+ verifyACLModifyJob(modifyColleague);
+ verifyACLModifyJob(qAdmin);
verifyACLPersistence();
}
@@ -123,18 +136,21 @@ public class TestJobACLs {
// Set the job up.
final Configuration myConf = mr.createJobConf();
- myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user1,user3");
+ myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague);
// Submit the job as user1
- Job job = submitJobAsUser(myConf, "user1");
+ Job job = submitJobAsUser(myConf, jobSubmitter);
final JobID jobId = job.getJobID();
// Try operations as an unauthorized user.
- verifyViewJobAsUnauthorizedUser(myConf, jobId, "user2");
+ verifyViewJobAsUnauthorizedUser(myConf, jobId, modifyColleague);
- // Try operations as an authorized user.
- verifyViewJobAsAuthorizedUser(myConf, jobId, "user3");
+ // Try operations as an authorized user, who is part of view-job-acl.
+ verifyViewJobAsAuthorizedUser(myConf, jobId, viewColleague);
+
+ // Try operations as an authorized user, who is a queue administrator.
+ verifyViewJobAsAuthorizedUser(myConf, jobId, qAdmin);
// Clean up the job
job.killJob();
@@ -242,7 +258,7 @@ public class TestJobACLs {
fail("AccessControlException expected..");
} catch (IOException ioe) {
assertTrue(ioe.getMessage().contains(
- JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.VIEW_JOB));
+ " cannot perform operation " + JobACL.VIEW_JOB));
} catch (InterruptedException e) {
fail("Exception .. interrupted.." + e);
}
@@ -253,7 +269,7 @@ public class TestJobACLs {
fail("AccessControlException expected..");
} catch (IOException ioe) {
assertTrue(ioe.getMessage().contains(
- JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.VIEW_JOB));
+ " cannot perform operation " + JobACL.VIEW_JOB));
} catch (InterruptedException e) {
fail("Exception .. interrupted.." + e);
}
@@ -264,29 +280,29 @@ public class TestJobACLs {
}
/**
- * Verify JobContext.Job_ACL_MODIFY_JOB
+ * Verify MRConfig.Job_ACL_MODIFY_JOB
*
* @throws IOException
* @throws InterruptedException
* @throws ClassNotFoundException
*/
- private void verifyACLModifyJob() throws IOException,
+ private void verifyACLModifyJob(String authorizedUser) throws IOException,
InterruptedException, ClassNotFoundException {
// Set the job up.
final Configuration myConf = mr.createJobConf();
- myConf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, "user1,user3");
+ myConf.set(MRJobConfig.JOB_ACL_MODIFY_JOB, modifyColleague);
// Submit the job as user1
- Job job = submitJobAsUser(myConf, "user1");
+ Job job = submitJobAsUser(myConf, jobSubmitter);
final JobID jobId = job.getJobID();
// Try operations as an unauthorized user.
- verifyModifyJobAsUnauthorizedUser(myConf, jobId, "user2");
+ verifyModifyJobAsUnauthorizedUser(myConf, jobId, viewColleague);
// Try operations as an authorized user.
- verifyModifyJobAsAuthorizedUser(myConf, jobId, "user3");
+ verifyModifyJobAsAuthorizedUser(myConf, jobId, authorizedUser);
}
private void verifyModifyJobAsAuthorizedUser(
@@ -357,7 +373,7 @@ public class TestJobACLs {
fail("AccessControlException expected..");
} catch (IOException ioe) {
assertTrue(ioe.getMessage().contains(
- JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.MODIFY_JOB));
+ " cannot perform operation " + Operation.KILL_JOB));
} catch (InterruptedException e) {
fail("Exception .. interrupted.." + e);
}
@@ -368,7 +384,7 @@ public class TestJobACLs {
fail("AccessControlException expected..");
} catch (IOException ioe) {
assertTrue(ioe.getMessage().contains(
- JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.MODIFY_JOB));
+ " cannot perform operation " + Operation.SET_JOB_PRIORITY));
} catch (InterruptedException e) {
fail("Exception .. interrupted.." + e);
}
@@ -378,15 +394,14 @@ public class TestJobACLs {
});
}
- private void verifyACLPersistence() throws IOException,
- InterruptedException {
+ private void verifyACLPersistence() throws Exception {
// Set the job up.
final Configuration myConf = mr.createJobConf();
- myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user2 group2");
+ myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, viewColleague + " group2");
// Submit the job as user1
- Job job = submitJobAsUser(myConf, "user1");
+ Job job = submitJobAsUser(myConf, jobSubmitter);
final JobID jobId = job.getJobID();
@@ -406,11 +421,14 @@ public class TestJobACLs {
final Configuration myNewJobConf = mr.createJobConf();
// Now verify view-job works off CompletedJobStore
- verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, "user2");
+ verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, viewColleague);
+ verifyViewJobAsAuthorizedUser(myNewJobConf, jobId, qAdmin);
// Only JobCounters is persisted on the JobStore. So test counters only.
UserGroupInformation unauthorizedUGI =
- UserGroupInformation.createUserForTesting("user3", new String[] {});
+ UserGroupInformation.createUserForTesting(
+ modifyColleague, new String[] {});
+
unauthorizedUGI.doAs(new PrivilegedExceptionAction<Object>() {
@SuppressWarnings("null")
@Override
@@ -432,7 +450,7 @@ public class TestJobACLs {
fail("AccessControlException expected..");
} catch (IOException ioe) {
assertTrue(ioe.getMessage().contains(
- JobACLsManager.UNAUTHORIZED_JOB_ACCESS_ERROR + JobACL.VIEW_JOB));
+ " cannot perform operation " + Operation.VIEW_JOB_COUNTERS));
} catch (InterruptedException e) {
fail("Exception .. interrupted.." + e);
}
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/CurrentJHParser.java Fri Sep 17 07:34:39 2010
@@ -21,10 +21,7 @@ import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaParseException;
import org.apache.hadoop.mapreduce.jobhistory.EventReader;
-import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java Fri Sep 17 07:34:39 2010
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.tools.rumen;
-import java.text.ParseException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -72,7 +71,11 @@ public class Job20LineHistoryEventEmitte
String submitTime = line.get("SUBMIT_TIME");
String jobConf = line.get("JOBCONF");
String user = line.get("USER");
+ if (user == null) {
+ user = "nulluser";
+ }
String jobName = line.get("JOBNAME");
+ String jobQueueName = line.get("JOB_QUEUE");// could be null
if (submitTime != null) {
Job20LineHistoryEventEmitter that =
@@ -82,8 +85,8 @@ public class Job20LineHistoryEventEmitte
Map<JobACL, AccessControlList> jobACLs =
new HashMap<JobACL, AccessControlList>();
- return new JobSubmittedEvent(jobID, jobName, user == null ? "nulluser"
- : user, that.originalSubmitTime, jobConf, jobACLs);
+ return new JobSubmittedEvent(jobID, jobName, user,
+ that.originalSubmitTime, jobConf, jobACLs, jobQueueName);
}
return null;
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=998003&r1=998002&r2=998003&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java Fri Sep 17 07:34:39 2010
@@ -544,6 +544,8 @@ public class JobBuilder {
result.setJobName(event.getJobName());
result.setUser(event.getUserName());
result.setSubmitTime(event.getSubmitTime());
+ // job queue name is set when conf file is processed.
+ // See JobBuilder.process(Properties) method for details.
}
private void processJobStatusChangedEvent(JobStatusChangedEvent event) {