You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/12/13 20:02:41 UTC
svn commit: r1213867 - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/...
Author: atm
Date: Tue Dec 13 19:02:37 2011
New Revision: 1213867
URL: http://svn.apache.org/viewvc?rev=1213867&view=rev
Log:
Merge trunk into HA branch.
Added:
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java
- copied unchanged from r1213862, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
- copied unchanged from r1213862, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
Modified:
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/.gitignore (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/JobClientUnitTest.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/hadoop-mapreduce-project:1152502-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project:1152502-1213862
/hadoop/core/branches/branch-0.19/mapred:713112
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/.gitignore:1161333-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/.gitignore:1161333-1213862
/hadoop/core/branches/branch-0.19/mapred/.gitignore:713112
/hadoop/core/trunk/.gitignore:784664-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt Tue Dec 13 19:02:37 2011
@@ -144,6 +144,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3369. Migrate MR1 tests to run on MR2 using the new interfaces
introduced in MAPREDUCE-3169. (Ahmed Radwan via tomwhite)
+ MAPREDUCE-3518. mapred queue -info <queue> -showJobs throws NPE.
+ (Jonathan Eagles via mahadev)
+
OPTIMIZATIONS
BUG FIXES
@@ -264,6 +267,15 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3527. Fix minor API incompatibilities between 1.0 and 0.23.
(tomwhite)
+ MAPREDUCE-3328. mapred queue -list output inconsistent and missing child
+ queues. (Ravi Prakash via mahadev)
+
+ MAPREDUCE-3510. Capacity Scheduler inherited ACLs not displayed by mapred queue
+ -showacls (Jonathan Eagles via mahadev)
+
+ MAPREDUCE-3537. Fix race condition in DefaultContainerExecutor which led
+ to container localization occuring in wrong directories. (acmurthy)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:1161333-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:1161333-1213862
/hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
/hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/conf:1152502-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/conf:1152502-1213862
/hadoop/core/branches/branch-0.19/mapred/conf:713112
/hadoop/core/trunk/conf:784664-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java Tue Dec 13 19:02:37 2011
@@ -450,9 +450,19 @@ public class TypeConverter {
public static QueueInfo fromYarn(org.apache.hadoop.yarn.api.records.QueueInfo
queueInfo, Configuration conf) {
- return new QueueInfo(queueInfo.getQueueName(),queueInfo.toString(),
- fromYarn(queueInfo.getQueueState()), TypeConverter.fromYarnApps(
- queueInfo.getApplications(), conf));
+ QueueInfo toReturn = new QueueInfo(queueInfo.getQueueName(), "Capacity: " +
+ queueInfo.getCapacity() * 100 + ", MaximumCapacity: " +
+ (queueInfo.getMaximumCapacity() < 0 ? "UNDEFINED" :
+ queueInfo.getMaximumCapacity()) + ", CurrentCapacity: " +
+ queueInfo.getCurrentCapacity() * 100, fromYarn(queueInfo.getQueueState()),
+ TypeConverter.fromYarnApps(queueInfo.getApplications(), conf));
+ List<QueueInfo> childQueues = new ArrayList<QueueInfo>();
+ for(org.apache.hadoop.yarn.api.records.QueueInfo childQueue :
+ queueInfo.getChildQueues()) {
+ childQueues.add(fromYarn(childQueue, conf));
+ }
+ toReturn.setQueueChildren(childQueues);
+ return toReturn;
}
public static QueueInfo[] fromYarnQueueInfo(
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java Tue Dec 13 19:02:37 2011
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.mapreduce;
+import java.util.ArrayList;
+import java.util.List;
+
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -36,6 +39,7 @@ import org.apache.hadoop.yarn.api.record
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestTypeConverter {
@Test
@@ -134,4 +138,33 @@ public class TestTypeConverter {
Assert.assertEquals("queueInfo translation didn't work.",
returned.getState().toString(), queueInfo.getQueueState().toString().toLowerCase());
}
+
+ /**
+ * Test that child queues are converted too during conversion of the parent
+ * queue
+ */
+ @Test
+ public void testFromYarnQueue() {
+ //Define child queue
+ org.apache.hadoop.yarn.api.records.QueueInfo child =
+ Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class);
+ Mockito.when(child.getQueueState()).thenReturn(QueueState.RUNNING);
+
+ //Define parent queue
+ org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
+ Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class);
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> children =
+ new ArrayList<org.apache.hadoop.yarn.api.records.QueueInfo>();
+ children.add(child); //Add one child
+ Mockito.when(queueInfo.getChildQueues()).thenReturn(children);
+ Mockito.when(queueInfo.getQueueState()).thenReturn(QueueState.RUNNING);
+
+ //Call the function we're testing
+ org.apache.hadoop.mapreduce.QueueInfo returned =
+ TypeConverter.fromYarn(queueInfo, new Configuration());
+
+ //Verify that the converted queue has the 1 child we had added
+ Assert.assertEquals("QueueInfo children weren't properly converted",
+ returned.getQueueChildren().size(), 1);
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java Tue Dec 13 19:02:37 2011
@@ -432,7 +432,6 @@ public class JobClient extends CLI {
}
- Cluster cluster;
/**
* Ugi of the client. We store this ugi when the client is created and
* then make sure that the same ugi is used to run the various protocols.
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java Tue Dec 13 19:02:37 2011
@@ -110,40 +110,33 @@ class JobQueueClient extends Configured
}
// format and print information about the passed in job queue.
- void printJobQueueInfo(JobQueueInfo jobQueueInfo, Writer writer)
- throws IOException {
+ void printJobQueueInfo(JobQueueInfo jobQueueInfo, Writer writer,
+ String prefix) throws IOException {
if (jobQueueInfo == null) {
writer.write("No queue found.\n");
writer.flush();
return;
}
- writer.write(String.format("Queue Name : %s \n",
+ writer.write(String.format(prefix + "======================\n"));
+ writer.write(String.format(prefix + "Queue Name : %s \n",
jobQueueInfo.getQueueName()));
- writer.write(String.format("Queue State : %s \n",
+ writer.write(String.format(prefix + "Queue State : %s \n",
jobQueueInfo.getQueueState()));
- writer.write(String.format("Scheduling Info : %s \n",
+ writer.write(String.format(prefix + "Scheduling Info : %s \n",
jobQueueInfo.getSchedulingInfo()));
List<JobQueueInfo> childQueues = jobQueueInfo.getChildren();
if (childQueues != null && childQueues.size() > 0) {
- writer.write(String.format("Child Queues : "));
for (int i = 0; i < childQueues.size(); i++) {
- JobQueueInfo childQueue = childQueues.get(i);
- writer.write(String.format("%s", childQueue.getQueueName()));
- if (i != childQueues.size() - 1) {
- writer.write(String.format(", "));
- }
+ printJobQueueInfo(childQueues.get(i), writer, " " + prefix);
}
- writer.write("\n");
}
- writer.write(String.format("======================\n"));
writer.flush();
}
private void displayQueueList() throws IOException {
JobQueueInfo[] rootQueues = jc.getRootQueues();
- List<JobQueueInfo> allQueues = expandQueueList(rootQueues);
- for (JobQueueInfo queue : allQueues) {
- printJobQueueInfo(queue, new PrintWriter(System.out));
+ for (JobQueueInfo queue : rootQueues) {
+ printJobQueueInfo(queue, new PrintWriter(System.out), "");
}
}
@@ -181,7 +174,7 @@ class JobQueueClient extends Configured
System.out.println("Queue \"" + queue + "\" does not exist.");
return;
}
- printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out));
+ printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out), "");
if (showJobs && (jobQueueInfo.getChildren() == null ||
jobQueueInfo.getChildren().size() == 0)) {
JobStatus[] jobs = jc.getJobsFromQueue(queue);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java Tue Dec 13 19:02:37 2011
@@ -105,7 +105,7 @@ public class JobQueueInfo extends QueueI
public List<JobQueueInfo> getChildren() {
List<JobQueueInfo> list = new ArrayList<JobQueueInfo>();
for (QueueInfo q : super.getQueueChildren()) {
- list.add((JobQueueInfo)q);
+ list.add(new JobQueueInfo(q));
}
return list;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java Tue Dec 13 19:02:37 2011
@@ -55,7 +55,7 @@ import org.apache.hadoop.yarn.logaggrega
@InterfaceStability.Stable
public class CLI extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(CLI.class);
- private Cluster cluster;
+ protected Cluster cluster;
public CLI() {
}
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1166973-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1166973-1213862
/hadoop/core/branches/branch-0.19/mapred/src/java/mapred-default.xml:713112
/hadoop/core/trunk/src/mapred/mapred-default.xml:776175-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Tue Dec 13 19:02:37 2011
@@ -198,13 +198,16 @@ public class ResourceMgrDelegate {
}
private void getChildQueues(org.apache.hadoop.yarn.api.records.QueueInfo parent,
- List<org.apache.hadoop.yarn.api.records.QueueInfo> queues) {
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> queues,
+ boolean recursive) {
List<org.apache.hadoop.yarn.api.records.QueueInfo> childQueues =
parent.getChildQueues();
for (org.apache.hadoop.yarn.api.records.QueueInfo child : childQueues) {
queues.add(child);
- getChildQueues(child, queues);
+ if(recursive) {
+ getChildQueues(child, queues, recursive);
+ }
}
}
@@ -226,7 +229,7 @@ public class ResourceMgrDelegate {
org.apache.hadoop.yarn.api.records.QueueInfo rootQueue =
applicationsManager.getQueueInfo(
getQueueInfoRequest(ROOT, false, true, true)).getQueueInfo();
- getChildQueues(rootQueue, queues);
+ getChildQueues(rootQueue, queues, true);
return TypeConverter.fromYarnQueueInfo(queues, this.conf);
}
@@ -238,8 +241,8 @@ public class ResourceMgrDelegate {
org.apache.hadoop.yarn.api.records.QueueInfo rootQueue =
applicationsManager.getQueueInfo(
- getQueueInfoRequest(ROOT, false, true, false)).getQueueInfo();
- getChildQueues(rootQueue, queues);
+ getQueueInfoRequest(ROOT, false, true, true)).getQueueInfo();
+ getChildQueues(rootQueue, queues, false);
return TypeConverter.fromYarnQueueInfo(queues, this.conf);
}
@@ -252,7 +255,7 @@ public class ResourceMgrDelegate {
org.apache.hadoop.yarn.api.records.QueueInfo parentQueue =
applicationsManager.getQueueInfo(
getQueueInfoRequest(parent, false, true, false)).getQueueInfo();
- getChildQueues(parentQueue, queues);
+ getChildQueues(parentQueue, queues, true);
return TypeConverter.fromYarnQueueInfo(queues, this.conf);
}
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Tue Dec 13 19:02:37 2011
@@ -1 +1,6 @@
target
+.classpath
+.externalToolBuilders
+.launches
+.project
+.settings
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Tue Dec 13 19:02:37 2011
@@ -75,7 +75,7 @@ public class DefaultContainerExecutor ex
}
@Override
- public void startLocalizer(Path nmPrivateContainerTokensPath,
+ public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
List<String> localDirs, List<String> logDirs)
throws IOException, InterruptedException {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Tue Dec 13 19:02:37 2011
@@ -198,19 +198,33 @@ public class CapacitySchedulerConfigurat
private static String getAclKey(QueueACL acl) {
return "acl_" + acl.toString().toLowerCase();
}
-
+
+ public AccessControlList getAcl(String queue, QueueACL acl) {
+ String queuePrefix = getQueuePrefix(queue);
+ String aclString = get(queuePrefix + getAclKey(acl), DEFAULT_ACL);
+ return new AccessControlList(aclString);
+ }
+
+ public void setAcl(String queue, QueueACL acl, String aclString) {
+ String queuePrefix = getQueuePrefix(queue);
+ set(queuePrefix + getAclKey(acl), aclString);
+ }
+
public Map<QueueACL, AccessControlList> getAcls(String queue) {
- Map<QueueACL, AccessControlList> acls =
+ Map<QueueACL, AccessControlList> acls =
new HashMap<QueueACL, AccessControlList>();
- String queuePrefix = getQueuePrefix(queue);
for (QueueACL acl : QueueACL.values()) {
- acls.put(acl,
- new AccessControlList(get(queuePrefix + getAclKey(acl),
- DEFAULT_ACL)));
+ acls.put(acl, getAcl(queue, acl));
}
return acls;
}
+ public void setAcls(String queue, Map<QueueACL, AccessControlList> acls) {
+ for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
+ setAcl(queue, e.getKey(), e.getValue().getAclString());
+ }
+ }
+
public String[] getQueues(String queue) {
LOG.info("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
String[] queues = getStrings(getQueuePrefix(queue) + QUEUES);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Tue Dec 13 19:02:37 2011
@@ -492,11 +492,8 @@ public class LeafQueue implements CSQueu
QueueUserACLInfo userAclInfo =
recordFactory.newRecordInstance(QueueUserACLInfo.class);
List<QueueACL> operations = new ArrayList<QueueACL>();
- for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
- QueueACL operation = e.getKey();
- AccessControlList acl = e.getValue();
-
- if (acl.isUserAllowed(user)) {
+ for (QueueACL operation : QueueACL.values()) {
+ if (hasAccess(operation, user)) {
operations.add(operation);
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Tue Dec 13 19:02:37 2011
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -30,11 +31,14 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -102,20 +106,29 @@ public class TestLeafQueue {
private static final String A = "a";
private static final String B = "b";
+ private static final String C = "c";
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
+ conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setMaximumCapacity(CapacityScheduler.ROOT, 100);
+ conf.setAcl(CapacityScheduler.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
final String Q_A = CapacityScheduler.ROOT + "." + A;
- conf.setCapacity(Q_A, 10);
+ conf.setCapacity(Q_A, 9);
conf.setMaximumCapacity(Q_A, 20);
+ conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*");
final String Q_B = CapacityScheduler.ROOT + "." + B;
conf.setCapacity(Q_B, 90);
conf.setMaximumCapacity(Q_B, 99);
+ conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*");
+
+ final String Q_C = CapacityScheduler.ROOT + "." + C;
+ conf.setCapacity(Q_C, 1);
+ conf.setMaximumCapacity(Q_C, 10);
+ conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " ");
LOG.info("Setup top-level queues a and b");
}
@@ -167,8 +180,8 @@ public class TestLeafQueue {
//can add more sturdy test with 3-layer queues
//once MAPREDUCE:3410 is resolved
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
- assertEquals(0.1, a.getCapacity(), epsilon);
- assertEquals(0.1, a.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.09, a.getCapacity(), epsilon);
+ assertEquals(0.09, a.getAbsoluteCapacity(), epsilon);
assertEquals(0.2, a.getMaximumCapacity(), epsilon);
assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
@@ -177,6 +190,12 @@ public class TestLeafQueue {
assertEquals(0.9, b.getAbsoluteCapacity(), epsilon);
assertEquals(0.99, b.getMaximumCapacity(), epsilon);
assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
+
+ LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
+ assertEquals(0.01, c.getCapacity(), epsilon);
+ assertEquals(0.01, c.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.1, c.getMaximumCapacity(), epsilon);
+ assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon);
}
@Test
@@ -1082,6 +1101,37 @@ public class TestLeafQueue {
assertEquals(0, app_0.getTotalRequiredResources(priority));
}
+
+ public boolean hasQueueACL(List<QueueUserACLInfo> aclInfos, QueueACL acl) {
+ for (QueueUserACLInfo aclInfo : aclInfos) {
+ if (aclInfo.getUserAcls().contains(acl)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Test
+ public void testInheritedQueueAcls() throws IOException {
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+
+ LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+ LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
+ LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
+
+ assertFalse(root.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+ assertTrue(a.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+ assertTrue(b.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+ assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+
+ assertTrue(hasQueueACL(
+ a.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+ assertTrue(hasQueueACL(
+ b.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+ assertFalse(hasQueueACL(
+ c.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+
+ }
@After
public void tearDown() throws Exception {
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1213862
/hadoop/core/branches/branch-0.19/mapred/src/c++:713112
/hadoop/core/trunk/src/c++:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1213862
/hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
/hadoop/core/trunk/src/contrib:784664-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1213862
/hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112
/hadoop/core/branches/branch-0.19/mapred/src/contrib/block_forensics:713112
/hadoop/core/trunk/src/contrib/block_forensics:784664-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1213862
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
/hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1213862
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
/hadoop/core/trunk/src/contrib/build.xml:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1213862
/hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
/hadoop/core/trunk/src/contrib/data_join:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1213862
/hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
/hadoop/core/branches/branch-0.19/mapred/src/contrib/eclipse-plugin:713112
/hadoop/core/trunk/src/contrib/eclipse-plugin:776175-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1213862
/hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
/hadoop/core/trunk/src/contrib/index:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1213862
/hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
/hadoop/core/trunk/src/contrib/vaidya:776175-786373
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1213862
/hadoop/core/branches/branch-0.19/mapred/src/examples:713112
/hadoop/core/trunk/src/examples:776175-784663
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1213862
/hadoop/core/branches/branch-0.19/mapred/src/java:713112
/hadoop/core/trunk/src/mapred:776175-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:1152502-1213862
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
/hadoop/core/trunk/src/test/mapred:776175-785643
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:1159757-1213862
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1152502-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:1152502-1213862
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/hdfs:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/hdfs:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1161333-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java:1161333-1213862
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/FileBench.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1161333-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:1161333-1213862
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1159757-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:1159757-1213862
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs-with-mr/org/apache/hadoop/ipc:713112
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/ipc:713112
/hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/ipc:776175-784663
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/JobClientUnitTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/JobClientUnitTest.java?rev=1213867&r1=1213866&r2=1213867&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/JobClientUnitTest.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/JobClientUnitTest.java Tue Dec 13 19:02:37 2011
@@ -19,21 +19,41 @@
package org.apache.hadoop.mapred;
import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.IOException;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Cluster;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobPriority;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.TaskReport;
import org.junit.Test;
public class JobClientUnitTest {
+ public class TestJobClient extends JobClient {
+
+ TestJobClient(JobConf jobConf) throws IOException {
+ super(jobConf);
+ }
+
+ void setCluster(Cluster cluster) {
+ this.cluster = cluster;
+ }
+ }
+
@SuppressWarnings("deprecation")
@Test
public void testMapTaskReportsWithNullJob() throws Exception {
- JobClient client = new JobClient();
+ TestJobClient client = new TestJobClient(new JobConf());
Cluster mockCluster = mock(Cluster.class);
- client.cluster = mockCluster;
+ client.setCluster(mockCluster);
JobID id = new JobID("test",0);
when(mockCluster.getJob(id)).thenReturn(null);
@@ -47,9 +67,9 @@ public class JobClientUnitTest {
@SuppressWarnings("deprecation")
@Test
public void testReduceTaskReportsWithNullJob() throws Exception {
- JobClient client = new JobClient();
+ TestJobClient client = new TestJobClient(new JobConf());
Cluster mockCluster = mock(Cluster.class);
- client.cluster = mockCluster;
+ client.setCluster(mockCluster);
JobID id = new JobID("test",0);
when(mockCluster.getJob(id)).thenReturn(null);
@@ -63,9 +83,9 @@ public class JobClientUnitTest {
@SuppressWarnings("deprecation")
@Test
public void testSetupTaskReportsWithNullJob() throws Exception {
- JobClient client = new JobClient();
+ TestJobClient client = new TestJobClient(new JobConf());
Cluster mockCluster = mock(Cluster.class);
- client.cluster = mockCluster;
+ client.setCluster(mockCluster);
JobID id = new JobID("test",0);
when(mockCluster.getJob(id)).thenReturn(null);
@@ -79,9 +99,9 @@ public class JobClientUnitTest {
@SuppressWarnings("deprecation")
@Test
public void testCleanupTaskReportsWithNullJob() throws Exception {
- JobClient client = new JobClient();
+ TestJobClient client = new TestJobClient(new JobConf());
Cluster mockCluster = mock(Cluster.class);
- client.cluster = mockCluster;
+ client.setCluster(mockCluster);
JobID id = new JobID("test",0);
when(mockCluster.getJob(id)).thenReturn(null);
@@ -91,4 +111,49 @@ public class JobClientUnitTest {
verify(mockCluster).getJob(id);
}
+
+ @Test
+ public void testShowJob() throws Exception {
+ TestJobClient client = new TestJobClient(new JobConf());
+ JobID jobID = new JobID("test", 0);
+
+ JobStatus mockJobStatus = mock(JobStatus.class);
+ when(mockJobStatus.getJobID()).thenReturn(jobID);
+ when(mockJobStatus.getState()).thenReturn(JobStatus.State.RUNNING);
+ when(mockJobStatus.getStartTime()).thenReturn(0L);
+ when(mockJobStatus.getUsername()).thenReturn("mockuser");
+ when(mockJobStatus.getQueue()).thenReturn("mockqueue");
+ when(mockJobStatus.getPriority()).thenReturn(JobPriority.NORMAL);
+ when(mockJobStatus.getNumUsedSlots()).thenReturn(1);
+ when(mockJobStatus.getNumReservedSlots()).thenReturn(1);
+ when(mockJobStatus.getUsedMem()).thenReturn(1024);
+ when(mockJobStatus.getReservedMem()).thenReturn(512);
+ when(mockJobStatus.getNeededMem()).thenReturn(2048);
+ when(mockJobStatus.getSchedulingInfo()).thenReturn("NA");
+
+ Job mockJob = mock(Job.class);
+ when(mockJob.getTaskReports(isA(TaskType.class))).thenReturn(new TaskReport[0]);
+
+ Cluster mockCluster = mock(Cluster.class);
+ when(mockCluster.getJob(jobID)).thenReturn(mockJob);
+
+ client.setCluster(mockCluster);
+
+
+ client.displayJobList(new JobStatus[] {mockJobStatus});
+ verify(mockJobStatus, atLeastOnce()).getJobID();
+ verify(mockJob, atLeastOnce()).getTaskReports(isA(TaskType.class));
+ verify(mockCluster, atLeastOnce()).getJob(jobID);
+ verify(mockJobStatus).getState();
+ verify(mockJobStatus).getStartTime();
+ verify(mockJobStatus).getUsername();
+ verify(mockJobStatus).getQueue();
+ verify(mockJobStatus).getPriority();
+ verify(mockJobStatus).getNumUsedSlots();
+ verify(mockJobStatus).getNumReservedSlots();
+ verify(mockJobStatus).getUsedMem();
+ verify(mockJobStatus).getReservedMem();
+ verify(mockJobStatus).getNeededMem();
+ verify(mockJobStatus).getSchedulingInfo();
+ }
}
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1161333-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:1161333-1213862
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1161333-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:1161333-1213862
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:713112
/hadoop/core/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:776175-785643
/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java:817878-835934
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec 13 19:02:37 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1213389
+/hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:1152502-1213862
/hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
/hadoop/core/trunk/src/webapps/job:776175-785643