You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2011/12/13 00:58:23 UTC
svn commit: r1213505 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test...
Author: mahadev
Date: Mon Dec 12 23:58:22 2011
New Revision: 1213505
URL: http://svn.apache.org/viewvc?rev=1213505&view=rev
Log:
MAPREDUCE-3328. mapred queue -list output inconsistent and missing child queues. (Ravi Prakash via mahadev) - Merging r1213504 from trunk
Added:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java
- copied unchanged from r1213504, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobQueueClient.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
- copied unchanged from r1213504, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon Dec 12 23:58:22 2011
@@ -218,6 +218,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3527. Fix minor API incompatibilities between 1.0 and 0.23.
(tomwhite)
+ MAPREDUCE-3328. mapred queue -list output inconsistent and missing child
+ queues. (Ravi Prakash via mahadev)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java Mon Dec 12 23:58:22 2011
@@ -450,9 +450,19 @@ public class TypeConverter {
public static QueueInfo fromYarn(org.apache.hadoop.yarn.api.records.QueueInfo
queueInfo, Configuration conf) {
- return new QueueInfo(queueInfo.getQueueName(),queueInfo.toString(),
- fromYarn(queueInfo.getQueueState()), TypeConverter.fromYarnApps(
- queueInfo.getApplications(), conf));
+ QueueInfo toReturn = new QueueInfo(queueInfo.getQueueName(), "Capacity: " +
+ queueInfo.getCapacity() * 100 + ", MaximumCapacity: " +
+ (queueInfo.getMaximumCapacity() < 0 ? "UNDEFINED" :
+ queueInfo.getMaximumCapacity()) + ", CurrentCapacity: " +
+ queueInfo.getCurrentCapacity() * 100, fromYarn(queueInfo.getQueueState()),
+ TypeConverter.fromYarnApps(queueInfo.getApplications(), conf));
+ List<QueueInfo> childQueues = new ArrayList<QueueInfo>();
+ for(org.apache.hadoop.yarn.api.records.QueueInfo childQueue :
+ queueInfo.getChildQueues()) {
+ childQueues.add(fromYarn(childQueue, conf));
+ }
+ toReturn.setQueueChildren(childQueues);
+ return toReturn;
}
public static QueueInfo[] fromYarnQueueInfo(
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java Mon Dec 12 23:58:22 2011
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.mapreduce;
+import java.util.ArrayList;
+import java.util.List;
+
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -36,6 +39,7 @@ import org.apache.hadoop.yarn.api.record
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestTypeConverter {
@Test
@@ -134,4 +138,33 @@ public class TestTypeConverter {
Assert.assertEquals("queueInfo translation didn't work.",
returned.getState().toString(), queueInfo.getQueueState().toString().toLowerCase());
}
+
+ /**
+ * Test that child queues are converted too during conversion of the parent
+ * queue
+ */
+ @Test
+ public void testFromYarnQueue() {
+ //Define child queue
+ org.apache.hadoop.yarn.api.records.QueueInfo child =
+ Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class);
+ Mockito.when(child.getQueueState()).thenReturn(QueueState.RUNNING);
+
+ //Define parent queue
+ org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
+ Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class);
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> children =
+ new ArrayList<org.apache.hadoop.yarn.api.records.QueueInfo>();
+ children.add(child); //Add one child
+ Mockito.when(queueInfo.getChildQueues()).thenReturn(children);
+ Mockito.when(queueInfo.getQueueState()).thenReturn(QueueState.RUNNING);
+
+ //Call the function we're testing
+ org.apache.hadoop.mapreduce.QueueInfo returned =
+ TypeConverter.fromYarn(queueInfo, new Configuration());
+
+ //Verify that the converted queue has the 1 child we had added
+ Assert.assertEquals("QueueInfo children weren't properly converted",
+ returned.getQueueChildren().size(), 1);
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java Mon Dec 12 23:58:22 2011
@@ -110,40 +110,33 @@ class JobQueueClient extends Configured
}
// format and print information about the passed in job queue.
- void printJobQueueInfo(JobQueueInfo jobQueueInfo, Writer writer)
- throws IOException {
+ void printJobQueueInfo(JobQueueInfo jobQueueInfo, Writer writer,
+ String prefix) throws IOException {
if (jobQueueInfo == null) {
writer.write("No queue found.\n");
writer.flush();
return;
}
- writer.write(String.format("Queue Name : %s \n",
+ writer.write(String.format(prefix + "======================\n"));
+ writer.write(String.format(prefix + "Queue Name : %s \n",
jobQueueInfo.getQueueName()));
- writer.write(String.format("Queue State : %s \n",
+ writer.write(String.format(prefix + "Queue State : %s \n",
jobQueueInfo.getQueueState()));
- writer.write(String.format("Scheduling Info : %s \n",
+ writer.write(String.format(prefix + "Scheduling Info : %s \n",
jobQueueInfo.getSchedulingInfo()));
List<JobQueueInfo> childQueues = jobQueueInfo.getChildren();
if (childQueues != null && childQueues.size() > 0) {
- writer.write(String.format("Child Queues : "));
for (int i = 0; i < childQueues.size(); i++) {
- JobQueueInfo childQueue = childQueues.get(i);
- writer.write(String.format("%s", childQueue.getQueueName()));
- if (i != childQueues.size() - 1) {
- writer.write(String.format(", "));
- }
+ printJobQueueInfo(childQueues.get(i), writer, " " + prefix);
}
- writer.write("\n");
}
- writer.write(String.format("======================\n"));
writer.flush();
}
private void displayQueueList() throws IOException {
JobQueueInfo[] rootQueues = jc.getRootQueues();
- List<JobQueueInfo> allQueues = expandQueueList(rootQueues);
- for (JobQueueInfo queue : allQueues) {
- printJobQueueInfo(queue, new PrintWriter(System.out));
+ for (JobQueueInfo queue : rootQueues) {
+ printJobQueueInfo(queue, new PrintWriter(System.out), "");
}
}
@@ -181,7 +174,7 @@ class JobQueueClient extends Configured
System.out.println("Queue \"" + queue + "\" does not exist.");
return;
}
- printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out));
+ printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out), "");
if (showJobs && (jobQueueInfo.getChildren() == null ||
jobQueueInfo.getChildren().size() == 0)) {
JobStatus[] jobs = jc.getJobsFromQueue(queue);
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java Mon Dec 12 23:58:22 2011
@@ -105,7 +105,7 @@ public class JobQueueInfo extends QueueI
public List<JobQueueInfo> getChildren() {
List<JobQueueInfo> list = new ArrayList<JobQueueInfo>();
for (QueueInfo q : super.getQueueChildren()) {
- list.add((JobQueueInfo)q);
+ list.add(new JobQueueInfo(q));
}
return list;
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1213505&r1=1213504&r2=1213505&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Mon Dec 12 23:58:22 2011
@@ -198,13 +198,16 @@ public class ResourceMgrDelegate {
}
private void getChildQueues(org.apache.hadoop.yarn.api.records.QueueInfo parent,
- List<org.apache.hadoop.yarn.api.records.QueueInfo> queues) {
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> queues,
+ boolean recursive) {
List<org.apache.hadoop.yarn.api.records.QueueInfo> childQueues =
parent.getChildQueues();
for (org.apache.hadoop.yarn.api.records.QueueInfo child : childQueues) {
queues.add(child);
- getChildQueues(child, queues);
+ if(recursive) {
+ getChildQueues(child, queues, recursive);
+ }
}
}
@@ -226,7 +229,7 @@ public class ResourceMgrDelegate {
org.apache.hadoop.yarn.api.records.QueueInfo rootQueue =
applicationsManager.getQueueInfo(
getQueueInfoRequest(ROOT, false, true, true)).getQueueInfo();
- getChildQueues(rootQueue, queues);
+ getChildQueues(rootQueue, queues, true);
return TypeConverter.fromYarnQueueInfo(queues, this.conf);
}
@@ -238,8 +241,8 @@ public class ResourceMgrDelegate {
org.apache.hadoop.yarn.api.records.QueueInfo rootQueue =
applicationsManager.getQueueInfo(
- getQueueInfoRequest(ROOT, false, true, false)).getQueueInfo();
- getChildQueues(rootQueue, queues);
+ getQueueInfoRequest(ROOT, false, true, true)).getQueueInfo();
+ getChildQueues(rootQueue, queues, false);
return TypeConverter.fromYarnQueueInfo(queues, this.conf);
}
@@ -252,7 +255,7 @@ public class ResourceMgrDelegate {
org.apache.hadoop.yarn.api.records.QueueInfo parentQueue =
applicationsManager.getQueueInfo(
getQueueInfoRequest(parent, false, true, false)).getQueueInfo();
- getChildQueues(parentQueue, queues);
+ getChildQueues(parentQueue, queues, true);
return TypeConverter.fromYarnQueueInfo(queues, this.conf);
}