You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2010/08/12 10:22:48 UTC
svn commit: r984680 - in /hadoop/mapreduce/trunk/src:
java/org/apache/hadoop/mapred/ java/org/apache/hadoop/mapreduce/
java/org/apache/hadoop/mapreduce/jobhistory/
test/mapred/org/apache/hadoop/mapred/
test/mapred/org/apache/hadoop/mapreduce/
Author: vinodkv
Date: Thu Aug 12 08:22:48 2010
New Revision: 984680
URL: http://svn.apache.org/viewvc?rev=984680&view=rev
Log:
MAPREDUCE-1780. Fixes serialization of ACL in JobStatus.java to use AccessControlList.write() instead of AccessControlList.toString(). Contributed by Ravi Gummadi.
Modified:
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/QueueManager.java Thu Aug 12 08:22:48 2010
@@ -690,18 +690,17 @@ class QueueManager {
queue.getAcls().get(QueueManager.toFullPropertyName(queue.getName(),
Queue.QueueOperation.ADMINISTER_JOBS.getAclName()));
}
- StringBuilder aclsSubmitJobValue = new StringBuilder();
+ String aclsSubmitJobValue = " ";
if (submitJobList != null ) {
- aclsSubmitJobValue = getAclsInfo(submitJobList);
+ aclsSubmitJobValue = submitJobList.getAclString();
}
- dumpGenerator.writeStringField("acl_submit_job",
- aclsSubmitJobValue.toString());
- StringBuilder aclsAdministerValue = new StringBuilder();
+ dumpGenerator.writeStringField("acl_submit_job", aclsSubmitJobValue);
+ String aclsAdministerValue = " ";
if (administerJobsList != null) {
- aclsAdministerValue = getAclsInfo(administerJobsList);
+ aclsAdministerValue = administerJobsList.getAclString();
}
dumpGenerator.writeStringField("acl_administer_jobs",
- aclsAdministerValue.toString());
+ aclsAdministerValue);
dumpGenerator.writeFieldName("properties");
dumpGenerator.writeStartArray();
if (queue.getProperties() != null) {
@@ -725,7 +724,4 @@ class QueueManager {
}
}
- private static StringBuilder getAclsInfo(AccessControlList accessControlList){
- return new StringBuilder(accessControlList.toString());
- }
}
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobStatus.java Thu Aug 12 08:22:48 2010
@@ -350,7 +350,7 @@ public class JobStatus implements Writab
out.writeInt(jobACLs.size());
for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
WritableUtils.writeEnum(out, entry.getKey());
- Text.writeString(out, entry.getValue().toString());
+ entry.getValue().write(out);
}
}
@@ -377,8 +377,9 @@ public class JobStatus implements Writab
int numACLs = in.readInt();
for (int i = 0; i < numACLs; i++) {
JobACL aclType = WritableUtils.readEnum(in, JobACL.class);
- String acl = Text.readString(in);
- this.jobACLs.put(aclType, new AccessControlList(acl));
+ AccessControlList acl = new AccessControlList(" ");
+ acl.readFields(in);
+ this.jobACLs.put(aclType, acl);
}
}
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java Thu Aug 12 08:22:48 2010
@@ -71,7 +71,7 @@ public class JobSubmittedEvent implement
Map<Utf8, Utf8> jobAcls = new HashMap<Utf8, Utf8>();
for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
jobAcls.put(new Utf8(entry.getKey().getAclName()), new Utf8(
- entry.getValue().toString()));
+ entry.getValue().getAclString()));
}
datum.acls = jobAcls;
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobHistory.java Thu Aug 12 08:22:48 2010
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.Clust
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
@@ -585,6 +586,21 @@ public class TestJobHistory extends Test
validateJobLevelKeyValues(mr, job, jobInfo, conf);
validateTaskLevelKeyValues(mr, job, jobInfo);
validateTaskAttemptLevelKeyValues(mr, job, jobInfo);
+
+ // Also JobACLs should be correct
+ if (mr.getJobTrackerRunner().getJobTracker()
+ .isJobLevelAuthorizationEnabled()) {
+ AccessControlList acl = new AccessControlList(
+ conf.get(JobACL.VIEW_JOB.getAclName(), " "));
+ assertTrue("VIEW_JOB ACL is not properly logged to history file.",
+ acl.toString().equals(
+ jobInfo.getJobACLs().get(JobACL.VIEW_JOB).toString()));
+ acl = new AccessControlList(
+ conf.get(JobACL.MODIFY_JOB.getAclName(), " "));
+ assertTrue("MODIFY_JOB ACL is not properly logged to history file.",
+ acl.toString().equals(
+ jobInfo.getJobACLs().get(JobACL.MODIFY_JOB).toString()));
+ }
}
public void testDoneFolderOnHDFS() throws IOException, InterruptedException {
@@ -721,7 +737,10 @@ public class TestJobHistory extends Test
//set the done folder location
String doneFolder = TEST_ROOT_DIR + "history_done";
conf.set(JTConfig.JT_JOBHISTORY_COMPLETED_LOCATION, doneFolder);
-
+
+ // Enable ACLs so that they are logged to history
+ conf.setBoolean(MRConfig.JOB_LEVEL_AUTHORIZATION_ENABLING_FLAG, true);
+
mr = new MiniMRCluster(2, "file:///", 3, null, null, conf);
// run the TCs
@@ -737,6 +756,10 @@ public class TestJobHistory extends Test
//Disable speculative execution
conf.setSpeculativeExecution(false);
+ // set the job acls
+ conf.set(JobACL.VIEW_JOB.getAclName(), "user1,user2 group1,group2");
+ conf.set(JobACL.MODIFY_JOB.getAclName(), "user3,user4 group3,group4");
+
// Make sure that the job is not removed from memory until we do finish
// the validation of history file content
conf.setInt("mapred.jobtracker.completeuserjobs.maximum", 10);
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestQueueManager.java Thu Aug 12 08:22:48 2010
@@ -26,24 +26,18 @@ import static org.junit.Assert.*;
import org.apache.hadoop.mapreduce.QueueState;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AccessControlList;
import org.codehaus.jackson.map.ObjectMapper;
import org.junit.After;
import org.junit.Test;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-
import java.io.File;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.Map.Entry;
public class TestQueueManager {
@@ -596,7 +590,7 @@ public class TestQueueManager {
assertEquals("35", q1_properties.get("maxCapacity").getValue());
// check for acls
- assertEquals("u1", childQueues.get("p1:p12").getAcl_submit_job());
- assertEquals("u2", childQueues.get("p1:p12").getAcl_administer_jobs());
+ assertEquals("u1 ", childQueues.get("p1:p12").getAcl_submit_job());
+ assertEquals("u2 ", childQueues.get("p1:p12").getAcl_administer_jobs());
}
}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java?rev=984680&r1=984679&r2=984680&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapreduce/TestJobACLs.java Thu Aug 12 08:22:48 2010
@@ -383,7 +383,7 @@ public class TestJobACLs {
// Set the job up.
final Configuration myConf = mr.createJobConf();
- myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user1,user2");
+ myConf.set(MRJobConfig.JOB_ACL_VIEW_JOB, "user2 group2");
// Submit the job as user1
Job job = submitJobAsUser(myConf, "user1");