You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/04/17 07:19:07 UTC
svn commit: r1094103 [1/2] - in /hadoop/mapreduce/branches/MR-279: ./
mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/
mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/
mr-client/hadoop-ma...
Author: acmurthy
Date: Sun Apr 17 05:19:05 2011
New Revision: 1094103
URL: http://svn.apache.org/viewvc?rev=1094103&view=rev
Log:
Implement 'bin/mapred queue [-info [-showJobs]] [-list] and enhanced 'bin/mapred job -list' to show queue and ApplicationMaster information.
Added:
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoRequest.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoResponse.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoRequestPBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoResponsePBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationState.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/client_RM_protocol.proto
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Sun Apr 17 05:19:05 2011
@@ -4,6 +4,10 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Implement 'bin/mapred queue [-info [-showJobs]] [-list] and enhanced
+ 'bin/mapred job -list' to show queue and ApplicationMaster information.
+ (acmurthy)
+
Implement 'bin/mapred job -list' and 'bin/mapred job
-list-active-trackers'. (acmurthy)
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java Sun Apr 17 05:19:05 2011
@@ -341,15 +341,19 @@ public class TypeConverter {
public static JobStatus.State fromYarn(ApplicationState state) {
switch (state) {
case ALLOCATED:
- case LAUNCHED:
+ case ALLOCATING:
case PENDING:
+ case LAUNCHING:
return State.PREP;
case PAUSED:
+ case LAUNCHED:
case RUNNING:
return State.RUNNING;
case COMPLETED:
+ case CLEANUP:
return State.SUCCEEDED;
case FAILED:
+ case EXPIRED_PENDING:
return State.FAILED;
case KILLED:
return State.KILLED;
@@ -373,7 +377,8 @@ public class TypeConverter {
public static JobStatus fromYarn(Application application) {
String trackingUrl = "";
try {
- if (application.getMasterHost() != null) {
+ if (application.getMasterHost() != null &&
+ !application.getMasterHost().isEmpty()) {
URL url =
new URL("http", application.getMasterHost(),
application.getMasterPort(), "");
@@ -388,9 +393,10 @@ public class TypeConverter {
0.0f, 0.0f, 0.0f, 0.0f,
TypeConverter.fromYarn(application.getState()),
org.apache.hadoop.mapreduce.JobPriority.NORMAL,
- application.getUser(), application.getName(), "",
- trackingUrl
+ application.getUser(), application.getName(),
+ application.getQueue(), "", trackingUrl
);
+ jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url
return jobStatus;
}
@@ -402,5 +408,22 @@ public class TypeConverter {
return jobStatuses.toArray(new JobStatus[jobStatuses.size()]);
}
+
+ public static QueueInfo fromYarn(org.apache.hadoop.yarn.api.records.QueueInfo
+ queueInfo) {
+ return new QueueInfo(queueInfo.getQueueName(),
+ queueInfo.toString(), QueueState.RUNNING,
+ TypeConverter.fromYarn(queueInfo.getApplications()));
+ }
+
+ public static QueueInfo[] fromYarn(
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> queues) {
+ List<QueueInfo> queueInfos = new ArrayList<QueueInfo>(queues.size());
+ for (org.apache.hadoop.yarn.api.records.QueueInfo queue : queues) {
+ queueInfos.add(TypeConverter.fromYarn(queue));
+ }
+ return queueInfos.toArray(new QueueInfo[queueInfos.size()]);
+ }
+
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java Sun Apr 17 05:19:05 2011
@@ -869,10 +869,10 @@ public class JobClient extends CLI {
}
void displayJobList(JobStatus[] jobs) {
- System.out.printf("JobId\tState\tStartTime\tUserName\tPriority\tSchedulingInfo\n");
+ System.out.printf("JobId\tState\tStartTime\tUserName\tQueue\tPriority\tSchedulingInfo\n");
for (JobStatus job : jobs) {
- System.out.printf("%s\t%d\t%d\t%s\t%s\t%s\n", job.getJobID(), job.getRunState(),
- job.getStartTime(), job.getUsername(),
+ System.out.printf("%s\t%d\t%d\t%s\t%s\t%s\t%s\n", job.getJobID(), job.getRunState(),
+ job.getStartTime(), job.getUsername(), job.getQueue(),
job.getJobPriority().name(), job.getSchedulingInfo());
}
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java Sun Apr 17 05:19:05 2011
@@ -156,11 +156,36 @@ public class JobStatus extends org.apach
float reduceProgress, float cleanupProgress,
int runState, JobPriority jp, String user, String jobName,
String jobFile, String trackingUrl) {
- super(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
- getEnum(runState), org.apache.hadoop.mapreduce.JobPriority.valueOf(jp.name()),
- user, jobName, jobFile, trackingUrl);
+ this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+ runState, jp,
+ user, jobName, "default", jobFile, trackingUrl);
}
+ /**
+ * Create a job status object for a given jobid.
+ * @param jobid The jobid of the job
+ * @param setupProgress The progress made on the setup
+ * @param mapProgress The progress made on the maps
+ * @param reduceProgress The progress made on the reduces
+ * @param cleanupProgress The progress made on the cleanup
+ * @param runState The current state of the job
+ * @param jp Priority of the job.
+ * @param user userid of the person who submitted the job.
+ * @param jobName user-specified job name.
+ * @param queue job queue name.
+ * @param jobFile job configuration file.
+ * @param trackingUrl link to the web-ui for details of the job.
+ */
+ public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+ float reduceProgress, float cleanupProgress,
+ int runState, JobPriority jp,
+ String user, String jobName, String queue,
+ String jobFile, String trackingUrl) {
+ super(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+ getEnum(runState), org.apache.hadoop.mapreduce.JobPriority.valueOf(jp.name()),
+ user, jobName, queue, jobFile, trackingUrl);
+ }
+
public static JobStatus downgrade(org.apache.hadoop.mapreduce.JobStatus stat){
JobStatus old = new JobStatus(JobID.downgrade(stat.getJobID()),
stat.getSetupProgress(), stat.getMapProgress(), stat.getReduceProgress(),
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java Sun Apr 17 05:19:05 2011
@@ -78,6 +78,7 @@ public class JobStatus implements Writab
private State runState;
private long startTime;
private String user;
+ private String queue;
private JobPriority priority;
private String schedulingInfo="NA";
@@ -115,22 +116,47 @@ public class JobStatus implements Writab
float reduceProgress, float cleanupProgress,
State runState, JobPriority jp, String user, String jobName,
String jobFile, String trackingUrl) {
- this.jobid = jobid;
- this.setupProgress = setupProgress;
- this.mapProgress = mapProgress;
- this.reduceProgress = reduceProgress;
- this.cleanupProgress = cleanupProgress;
- this.runState = runState;
- this.user = user;
- if (jp == null) {
- throw new IllegalArgumentException("Job Priority cannot be null.");
- }
- priority = jp;
- this.jobName = jobName;
- this.jobFile = jobFile;
- this.trackingUrl = trackingUrl;
+ this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+ runState, jp, user, jobName, "default", jobFile, trackingUrl);
}
+ /**
+ * Create a job status object for a given jobid.
+ * @param jobid The jobid of the job
+ * @param setupProgress The progress made on the setup
+ * @param mapProgress The progress made on the maps
+ * @param reduceProgress The progress made on the reduces
+ * @param cleanupProgress The progress made on the cleanup
+ * @param runState The current state of the job
+ * @param jp Priority of the job.
+ * @param user userid of the person who submitted the job.
+ * @param jobName user-specified job name.
+ * @param queue queue name
+ * @param jobFile job configuration file.
+ * @param trackingUrl link to the web-ui for details of the job.
+ */
+ public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+ float reduceProgress, float cleanupProgress,
+ State runState, JobPriority jp,
+ String user, String jobName, String queue,
+ String jobFile, String trackingUrl) {
+ this.jobid = jobid;
+ this.setupProgress = setupProgress;
+ this.mapProgress = mapProgress;
+ this.reduceProgress = reduceProgress;
+ this.cleanupProgress = cleanupProgress;
+ this.runState = runState;
+ this.user = user;
+ this.queue = queue;
+ if (jp == null) {
+ throw new IllegalArgumentException("Job Priority cannot be null.");
+ }
+ priority = jp;
+ this.jobName = jobName;
+ this.jobFile = jobFile;
+ this.trackingUrl = trackingUrl;
+ }
+
/**
* Sets the map progress of this job
* @param p The value of map progress to set to
@@ -243,6 +269,14 @@ public class JobStatus implements Writab
this.jobACLs = acls;
}
+ void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ public String getQueue() {
+ return queue;
+ }
+
/**
* @return Percentage of progress in maps
*/
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java Sun Apr 17 05:19:05 2011
@@ -527,11 +527,12 @@ public class CLI extends Configured impl
throws IOException, InterruptedException {
System.out.println("Total jobs:" + jobs.length);
System.out.println("JobId\tState\tStartTime\t" +
- "UserName\tPriority\tSchedulingInfo");
+ "UserName\tQueue\tPriority\tSchedulingInfo");
for (JobStatus job : jobs) {
- System.out.printf("%s\t%s\t%d\t%s\t%s\t%s\n", job.getJobID().toString(),
+ System.out.printf("%s\t%s\t%d\t%s\t%s\t%s\t%s\n", job.getJobID().toString(),
job.getState(), job.getStartTime(),
- job.getUsername(), job.getPriority().name(), job.getSchedulingInfo());
+ job.getUsername(), job.getQueue(),
+ job.getPriority().name(), job.getSchedulingInfo());
}
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Sun Apr 17 05:19:05 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -52,12 +53,12 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationMaster;
import org.apache.hadoop.yarn.api.records.ApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.NodeManagerInfo;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -126,12 +127,6 @@ public class ResourceMgrDelegate {
}
- public QueueInfo[] getChildQueues(String arg0) throws IOException,
- InterruptedException {
- throw new IOException("Not implemented");
- }
-
-
public ClusterMetrics getClusterMetrics() throws IOException,
InterruptedException {
GetClusterMetricsRequest request = recordFactory.newRecordInstance(GetClusterMetricsRequest.class);
@@ -160,10 +155,38 @@ public class ResourceMgrDelegate {
return TypeConverter.fromYarn(applicationId);
}
+ private static final String ROOT = "root";
+
+ private GetQueueInfoRequest getQueueInfoRequest(String queueName,
+ boolean includeApplications, boolean includeChildQueues, boolean recursive) {
+ GetQueueInfoRequest request =
+ recordFactory.newRecordInstance(GetQueueInfoRequest.class);
+ request.setQueueName(queueName);
+ request.setIncludeApplications(includeApplications);
+ request.setIncludeChildQueues(includeChildQueues);
+ request.setRecursive(recursive);
+ return request;
+
+ }
- public QueueInfo getQueue(String arg0) throws IOException,
- InterruptedException {
- throw new IOException("Not implemented");
+ public QueueInfo getQueue(String queueName) throws IOException,
+ InterruptedException {
+ GetQueueInfoRequest request =
+ getQueueInfoRequest(queueName, true, false, false);
+ recordFactory.newRecordInstance(GetQueueInfoRequest.class);
+ return TypeConverter.fromYarn(
+ applicationsManager.getQueueInfo(request).getQueueInfo());
+ }
+
+ private void getChildQueues(org.apache.hadoop.yarn.api.records.QueueInfo parent,
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> queues) {
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> childQueues =
+ parent.getChildQueues();
+
+ for (org.apache.hadoop.yarn.api.records.QueueInfo child : childQueues) {
+ queues.add(child);
+ getChildQueues(child, queues);
+ }
}
@@ -174,14 +197,42 @@ public class ResourceMgrDelegate {
public QueueInfo[] getQueues() throws IOException, InterruptedException {
- throw new IOException("Not implemented");
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> queues =
+ new ArrayList<org.apache.hadoop.yarn.api.records.QueueInfo>();
+
+ org.apache.hadoop.yarn.api.records.QueueInfo rootQueue =
+ applicationsManager.getQueueInfo(
+ getQueueInfoRequest(ROOT, false, true, true)).getQueueInfo();
+ getChildQueues(rootQueue, queues);
+
+ return TypeConverter.fromYarn(queues);
}
public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
- throw new IOException("Not Implemented");
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> queues =
+ new ArrayList<org.apache.hadoop.yarn.api.records.QueueInfo>();
+
+ org.apache.hadoop.yarn.api.records.QueueInfo rootQueue =
+ applicationsManager.getQueueInfo(
+ getQueueInfoRequest(ROOT, false, true, false)).getQueueInfo();
+ getChildQueues(rootQueue, queues);
+
+ return TypeConverter.fromYarn(queues);
}
+ public QueueInfo[] getChildQueues(String parent) throws IOException,
+ InterruptedException {
+ List<org.apache.hadoop.yarn.api.records.QueueInfo> queues =
+ new ArrayList<org.apache.hadoop.yarn.api.records.QueueInfo>();
+
+ org.apache.hadoop.yarn.api.records.QueueInfo parentQueue =
+ applicationsManager.getQueueInfo(
+ getQueueInfoRequest(parent, false, true, false)).getQueueInfo();
+ getChildQueues(parentQueue, queues);
+
+ return TypeConverter.fromYarn(queues);
+ }
public String getStagingAreaDir() throws IOException, InterruptedException {
// Path path = new Path(MRJobConstants.JOB_SUBMIT_DIR);
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Sun Apr 17 05:19:05 2011
@@ -136,12 +136,6 @@ public class YARNRunner implements Clien
}
@Override
- public QueueInfo[] getChildQueues(String arg0) throws IOException,
- InterruptedException {
- return resMgrDelegate.getChildQueues(arg0);
- }
-
- @Override
public ClusterMetrics getClusterMetrics() throws IOException,
InterruptedException {
return resMgrDelegate.getClusterMetrics();
@@ -164,9 +158,9 @@ public class YARNRunner implements Clien
}
@Override
- public QueueInfo getQueue(String arg0) throws IOException,
+ public QueueInfo getQueue(String queueName) throws IOException,
InterruptedException {
- return resMgrDelegate.getQueue(arg0);
+ return resMgrDelegate.getQueue(queueName);
}
@Override
@@ -186,6 +180,12 @@ public class YARNRunner implements Clien
}
@Override
+ public QueueInfo[] getChildQueues(String parent) throws IOException,
+ InterruptedException {
+ return resMgrDelegate.getChildQueues(parent);
+ }
+
+ @Override
public String getStagingAreaDir() throws IOException, InterruptedException {
return resMgrDelegate.getStagingAreaDir();
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java Sun Apr 17 05:19:05 2011
@@ -73,13 +73,15 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationMaster;
import org.apache.hadoop.yarn.api.records.ApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationStatus;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -231,6 +233,13 @@ public class TestClientRedirect {
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
+ throws YarnRemoteException {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
class HistoryService extends AMService {
Modified: hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/java/org/apache/hadoop/mapred/JobInProgress.java Sun Apr 17 05:19:05 2011
@@ -442,9 +442,10 @@ public class JobInProgress {
this.priority = conf.getJobPriority();
this.profile = new JobProfile(conf.getUser(), this.jobId, jobFile
.toString(), url, conf.getJobName(), conf.getQueueName());
- this.status = new JobStatus(this.jobId, 0.0f, 0.0f, JobStatus.PREP,
- profile.getUser(), profile.getJobName(), profile.getJobFile(),
- profile.getURL().toString());
+ this.status = new JobStatus(this.jobId, 0.0f, 0.0f, 0.0f, 0.0f,
+ JobStatus.PREP, JobPriority.NORMAL,
+ profile.getUser(), profile.getJobName(), profile.getQueueName(),
+ profile.getJobFile(), profile.getURL().toString());
this.jobtracker.getInstrumentation().addPrepJob(conf, this.jobId);
status.setStartTime(startTime);
this.status.setJobPriority(this.priority);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java Sun Apr 17 05:19:05 2011
@@ -12,6 +12,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -24,4 +26,5 @@ public interface ClientRMProtocol {
public GetClusterMetricsResponse getClusterMetrics(GetClusterMetricsRequest request) throws YarnRemoteException;
public GetAllApplicationsResponse getAllApplications(GetAllApplicationsRequest request) throws YarnRemoteException;
public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request) throws YarnRemoteException;
+ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) throws YarnRemoteException;
}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoRequest.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoRequest.java?rev=1094103&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoRequest.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoRequest.java Sun Apr 17 05:19:05 2011
@@ -0,0 +1,16 @@
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+public interface GetQueueInfoRequest {
+ String getQueueName();
+ void setQueueName(String queueName);
+
+ boolean getIncludeApplications();
+ void setIncludeApplications(boolean includeApplications);
+
+ boolean getIncludeChildQueues();
+ void setIncludeChildQueues(boolean includeChildQueues);
+
+ boolean getRecursive();
+ void setRecursive(boolean recursive);
+}
+
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoResponse.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoResponse.java?rev=1094103&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoResponse.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetQueueInfoResponse.java Sun Apr 17 05:19:05 2011
@@ -0,0 +1,8 @@
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+
+public interface GetQueueInfoResponse {
+ QueueInfo getQueueInfo();
+ void setQueueInfo(QueueInfo queueInfo);
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodesResponsePBImpl.java Sun Apr 17 05:19:05 2011
@@ -39,12 +39,10 @@ public class GetClusterNodesResponsePBIm
@Override
public void setNodeManagerList(List<NodeManagerInfo> nodeManagers) {
- maybeInitBuilder();
if (nodeManagers == null) {
builder.clearNodeManagers();
}
this.nodeManagerInfoList = nodeManagers;
-
}
@Override
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoRequestPBImpl.java?rev=1094103&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoRequestPBImpl.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoRequestPBImpl.java Sun Apr 17 05:19:05 2011
@@ -0,0 +1,91 @@
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
+import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProtoOrBuilder;
+
+public class GetQueueInfoRequestPBImpl extends
+ ProtoBase<GetQueueInfoRequestProto> implements GetQueueInfoRequest {
+
+ GetQueueInfoRequestProto proto =
+ GetQueueInfoRequestProto.getDefaultInstance();
+ GetQueueInfoRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public GetQueueInfoRequestPBImpl() {
+ builder = GetQueueInfoRequestProto.newBuilder();
+ }
+
+ public GetQueueInfoRequestPBImpl(GetQueueInfoRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ @Override
+ public boolean getIncludeApplications() {
+ GetQueueInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasIncludeApplications()) ? p.getIncludeApplications() : false;
+ }
+
+ @Override
+ public boolean getIncludeChildQueues() {
+ GetQueueInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasIncludeChildQueues()) ? p.getIncludeChildQueues() : false;
+ }
+
+ @Override
+ public String getQueueName() {
+ GetQueueInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasQueueName()) ? p.getQueueName() : null;
+ }
+
+ @Override
+ public boolean getRecursive() {
+ GetQueueInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasRecursive()) ? p.getRecursive() : false;
+ }
+
+ @Override
+ public void setIncludeApplications(boolean includeApplications) {
+ maybeInitBuilder();
+ builder.setIncludeApplications(includeApplications);
+ }
+
+ @Override
+ public void setIncludeChildQueues(boolean includeChildQueues) {
+ maybeInitBuilder();
+ builder.setIncludeChildQueues(includeChildQueues);
+ }
+
+ @Override
+ public void setQueueName(String queueName) {
+ maybeInitBuilder();
+ if (queueName == null) {
+ builder.clearQueueName();
+ return;
+ }
+ builder.setQueueName((queueName));
+ }
+
+ @Override
+ public void setRecursive(boolean recursive) {
+ maybeInitBuilder();
+ builder.setRecursive(recursive);
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetQueueInfoRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public GetQueueInfoRequestProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoResponsePBImpl.java?rev=1094103&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoResponsePBImpl.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetQueueInfoResponsePBImpl.java Sun Apr 17 05:19:05 2011
@@ -0,0 +1,90 @@
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
+import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.impl.pb.QueueInfoPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoResponseProtoOrBuilder;
+
+public class GetQueueInfoResponsePBImpl extends ProtoBase<GetQueueInfoResponseProto>
+implements GetQueueInfoResponse {
+
+ QueueInfo queueInfo;
+
+ GetQueueInfoResponseProto proto =
+ GetQueueInfoResponseProto.getDefaultInstance();
+ GetQueueInfoResponseProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public GetQueueInfoResponsePBImpl() {
+ builder = GetQueueInfoResponseProto.newBuilder();
+ }
+
+ public GetQueueInfoResponsePBImpl(GetQueueInfoResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ @Override
+ public GetQueueInfoResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public QueueInfo getQueueInfo() {
+ if (this.queueInfo != null) {
+ return this.queueInfo;
+ }
+
+ GetQueueInfoResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasQueueInfo()) {
+ return null;
+ }
+ this.queueInfo = convertFromProtoFormat(p.getQueueInfo());
+ return this.queueInfo;
+ }
+
+ @Override
+ public void setQueueInfo(QueueInfo queueInfo) {
+ maybeInitBuilder();
+ if(queueInfo == null) {
+ builder.clearQueueInfo();
+ }
+ this.queueInfo = queueInfo;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.queueInfo != null) {
+ builder.setQueueInfo(convertToProtoFormat(this.queueInfo));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetQueueInfoResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private QueueInfo convertFromProtoFormat(QueueInfoProto queueInfo) {
+ return new QueueInfoPBImpl(queueInfo);
+ }
+
+ private QueueInfoProto convertToProtoFormat(QueueInfo queueInfo) {
+ return ((QueueInfoPBImpl)queueInfo).getProto();
+ }
+
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationState.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationState.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationState.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationState.java Sun Apr 17 05:19:05 2011
@@ -1,5 +1,16 @@
package org.apache.hadoop.yarn.api.records;
public enum ApplicationState {
- PENDING, ALLOCATING, ALLOCATED, EXPIRED_PENDING, LAUNCHING, LAUNCHED, RUNNING, PAUSED, CLEANUP, COMPLETED, KILLED, FAILED
+ PENDING,
+ ALLOCATING,
+ ALLOCATED,
+ EXPIRED_PENDING,
+ LAUNCHING,
+ LAUNCHED,
+ RUNNING,
+ PAUSED,
+ CLEANUP,
+ COMPLETED,
+ KILLED,
+ FAILED
}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java?rev=1094103&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java Sun Apr 17 05:19:05 2011
@@ -0,0 +1,23 @@
+package org.apache.hadoop.yarn.api.records;
+
+import java.util.List;
+
+public interface QueueInfo {
+ String getQueueName();
+ void setQueueName(String queueName);
+
+ float getCapacity();
+ void setCapacity(float capacity);
+
+ float getMaximumCapacity();
+ void setMaximumCapacity(float maximumCapacity);
+
+ float getCurrentCapacity();
+ void setCurrentCapacity(float currentCapacity);
+
+ List<QueueInfo> getChildQueues();
+ void setChildQueues(List<QueueInfo> childQueues);
+
+ List<Application> getApplications();
+ void setApplications(List<Application> applications);
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java?rev=1094103&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java Sun Apr 17 05:19:05 2011
@@ -0,0 +1,255 @@
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.Application;
+import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProtoOrBuilder;
+
+public class QueueInfoPBImpl extends ProtoBase<QueueInfoProto> implements
+ QueueInfo {
+
+ QueueInfoProto proto = QueueInfoProto.getDefaultInstance();
+ QueueInfoProto.Builder builder = null;
+ boolean viaProto = false;
+
+ List<Application> applicationsList;
+ List<QueueInfo> childQueuesList;
+
+ public QueueInfoPBImpl() {
+ builder = QueueInfoProto.newBuilder();
+ }
+
+ public QueueInfoPBImpl(QueueInfoProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ @Override
+ public List<Application> getApplications() {
+ initLocalApplicationsList();
+ return this.applicationsList;
+ }
+
+ @Override
+ public float getCapacity() {
+ QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasCapacity()) ? p.getCapacity() : -1;
+ }
+
+ @Override
+ public List<QueueInfo> getChildQueues() {
+ initLocalChildQueuesList();
+ return this.childQueuesList;
+ }
+
+ @Override
+ public float getCurrentCapacity() {
+ QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasCurrentCapacity()) ? p.getCurrentCapacity() : 0;
+ }
+
+ @Override
+ public float getMaximumCapacity() {
+ QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasMaximumCapacity()) ? p.getMaximumCapacity() : -1;
+ }
+
+ @Override
+ public String getQueueName() {
+ QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasQueueName()) ? p.getQueueName() : null;
+ }
+
+ @Override
+ public void setApplications(List<Application> applications) {
+ if (applications == null) {
+ builder.clearApplications();
+ }
+ this.applicationsList = applications;
+ }
+
+ @Override
+ public void setCapacity(float capacity) {
+ maybeInitBuilder();
+ builder.setCapacity(capacity);
+ }
+
+ @Override
+ public void setChildQueues(List<QueueInfo> childQueues) {
+ if (childQueues == null) {
+ builder.clearChildQueues();
+ }
+ this.childQueuesList = childQueues;
+ }
+
+ @Override
+ public void setCurrentCapacity(float currentCapacity) {
+ maybeInitBuilder();
+ builder.setCurrentCapacity(currentCapacity);
+ }
+
+ @Override
+ public void setMaximumCapacity(float maximumCapacity) {
+ maybeInitBuilder();
+ builder.setMaximumCapacity(maximumCapacity);
+ }
+
+ @Override
+ public void setQueueName(String queueName) {
+ maybeInitBuilder();
+ if (queueName == null) {
+ builder.clearQueueName();
+ }
+ builder.setQueueName(queueName);
+ }
+
+ @Override
+ public QueueInfoProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void initLocalApplicationsList() {
+ if (this.applicationsList != null) {
+ return;
+ }
+ QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
+ List<ApplicationProto> list = p.getApplicationsList();
+ applicationsList = new ArrayList<Application>();
+
+ for (ApplicationProto a : list) {
+ applicationsList.add(convertFromProtoFormat(a));
+ }
+ }
+
+ private void addApplicationsToProto() {
+ maybeInitBuilder();
+ builder.clearApplications();
+ if (applicationsList == null)
+ return;
+ Iterable<ApplicationProto> iterable = new Iterable<ApplicationProto>() {
+ @Override
+ public Iterator<ApplicationProto> iterator() {
+ return new Iterator<ApplicationProto>() {
+
+ Iterator<Application> iter = applicationsList.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ApplicationProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
+ builder.addAllApplications(iterable);
+ }
+
+ private void initLocalChildQueuesList() {
+ if (this.childQueuesList != null) {
+ return;
+ }
+ QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
+ List<QueueInfoProto> list = p.getChildQueuesList();
+ childQueuesList = new ArrayList<QueueInfo>();
+
+ for (QueueInfoProto a : list) {
+ childQueuesList.add(convertFromProtoFormat(a));
+ }
+ }
+
+ private void addChildQueuesInfoToProto() {
+ maybeInitBuilder();
+ builder.clearChildQueues();
+ if (childQueuesList == null)
+ return;
+ Iterable<QueueInfoProto> iterable = new Iterable<QueueInfoProto>() {
+ @Override
+ public Iterator<QueueInfoProto> iterator() {
+ return new Iterator<QueueInfoProto>() {
+
+ Iterator<QueueInfo> iter = childQueuesList.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public QueueInfoProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
+ builder.addAllChildQueues(iterable);
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.childQueuesList != null) {
+ addChildQueuesInfoToProto();
+ }
+ if (this.applicationsList != null) {
+ addApplicationsToProto();
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = QueueInfoProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+
+ private ApplicationPBImpl convertFromProtoFormat(ApplicationProto a) {
+ return new ApplicationPBImpl(a);
+ }
+
+ private ApplicationProto convertToProtoFormat(Application t) {
+ return ((ApplicationPBImpl)t).getProto();
+ }
+
+ private QueueInfoPBImpl convertFromProtoFormat(QueueInfoProto a) {
+ return new QueueInfoPBImpl(a);
+ }
+
+ private QueueInfoProto convertToProtoFormat(QueueInfo q) {
+ return ((QueueInfoPBImpl)q).getProto();
+ }
+
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/client_RM_protocol.proto
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/client_RM_protocol.proto?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/client_RM_protocol.proto (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/client_RM_protocol.proto Sun Apr 17 05:19:05 2011
@@ -18,5 +18,6 @@ service ClientRMProtocolService {
rpc getClusterMetrics (GetClusterMetricsRequestProto) returns (GetClusterMetricsResponseProto);
rpc getAllApplications (GetAllApplicationsRequestProto) returns (GetAllApplicationsResponseProto);
rpc getClusterNodes (GetClusterNodesRequestProto) returns (GetClusterNodesResponseProto);
+ rpc getQueueInfo (GetQueueInfoRequestProto) returns (GetQueueInfoResponseProto);
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto Sun Apr 17 05:19:05 2011
@@ -162,6 +162,15 @@ message YarnClusterMetricsProto {
optional int32 num_node_managers = 1;
}
+message QueueInfoProto {
+ optional string queueName = 1;
+ optional float capacity = 2;
+ optional float maximumCapacity = 3;
+ optional float currentCapacity = 4;
+ repeated QueueInfoProto childQueues = 5;
+ repeated ApplicationProto applications = 6;
+}
+
////////////////////////////////////////////////////////////////////////
////// From container_manager //////////////////////////////////////////
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_service_protos.proto Sun Apr 17 05:19:05 2011
@@ -88,9 +88,19 @@ message GetClusterNodesResponseProto {
repeated NodeManagerInfoProto nodeManagers = 1;
}
+message GetQueueInfoRequestProto {
+ optional string queueName = 1;
+ optional bool includeApplications = 2;
+ optional bool includeChildQueues = 3;
+ optional bool recursive = 4;
+}
+
+message GetQueueInfoResponseProto {
+ optional QueueInfoProto queueInfo = 1;
+}
//////////////////////////////////////////////////////
-/////// client_RM_Protocol ///////////////////////////
+/////// client_NM_Protocol ///////////////////////////
//////////////////////////////////////////////////////
message StartContainerRequestProto {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java Sun Apr 17 05:19:05 2011
@@ -19,6 +19,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationRequestPBImpl;
@@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationIdRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationIdResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -45,6 +49,8 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationIdRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoResponseProto;
import com.google.protobuf.ServiceException;
@@ -181,4 +187,23 @@ public class ClientRMProtocolPBClientImp
}
}
+ @Override
+ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
+ throws YarnRemoteException {
+ GetQueueInfoRequestProto requestProto =
+ ((GetQueueInfoRequestPBImpl)request).getProto();
+ try {
+ return new GetQueueInfoResponsePBImpl(
+ proxy.getQueueInfo(null, requestProto));
+ } catch (ServiceException e) {
+ if (e.getCause() instanceof YarnRemoteException) {
+ throw (YarnRemoteException)e.getCause();
+ } else if (e.getCause() instanceof UndeclaredThrowableException) {
+ throw (UndeclaredThrowableException)e.getCause();
+ } else {
+ throw new UndeclaredThrowableException(e);
+ }
+ }
+ }
+
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java Sun Apr 17 05:19:05 2011
@@ -7,6 +7,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationResponsePBImpl;
@@ -20,6 +21,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationIdRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationIdResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -36,6 +39,8 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationIdRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationIdResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto;
@@ -139,4 +144,17 @@ public class ClientRMProtocolPBServiceIm
}
}
+ @Override
+ public GetQueueInfoResponseProto getQueueInfo(RpcController controller,
+ GetQueueInfoRequestProto proto) throws ServiceException {
+ GetQueueInfoRequestPBImpl request =
+ new GetQueueInfoRequestPBImpl(proto);
+ try {
+ GetQueueInfoResponse response = real.getQueueInfo(request);
+ return ((GetQueueInfoResponsePBImpl)response).getProto();
+ } catch (YarnRemoteException e) {
+ throw new ServiceException(e);
+ }
+ }
+
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Sun Apr 17 05:19:05 2011
@@ -43,11 +43,14 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.NodeManagerInfo;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -58,6 +61,7 @@ import org.apache.hadoop.yarn.security.c
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.ApplicationsManager;
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.ResourceContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.service.AbstractService;
@@ -67,18 +71,22 @@ import org.apache.hadoop.yarn.service.Ab
*/
public class ClientRMService extends AbstractService implements ClientRMProtocol {
private static final Log LOG = LogFactory.getLog(ClientRMService.class);
- private ResourceContext clusterInfo;
- private ApplicationsManager applicationsManager;
+
+ final private ResourceContext clusterInfo;
+ final private ApplicationsManager applicationsManager;
+ final private ResourceScheduler scheduler;
+
private String clientServiceBindAddress;
private Server server;
private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
InetSocketAddress clientBindAddress;
public ClientRMService(ApplicationsManager applicationsManager,
- ResourceContext clusterInfo) {
+ ResourceContext clusterInfo, ResourceScheduler scheduler) {
super(ClientRMService.class.getName());
this.clusterInfo = clusterInfo;
this.applicationsManager = applicationsManager;
+ this.scheduler = scheduler;
}
@Override
@@ -178,6 +186,26 @@ public class ClientRMService extends Abs
return response;
}
+ @Override
+ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
+ throws YarnRemoteException {
+ GetQueueInfoResponse response =
+ recordFactory.newRecordInstance(GetQueueInfoResponse.class);
+ try {
+ QueueInfo queueInfo =
+ scheduler.getQueueInfo(request.getQueueName(),
+ request.getIncludeApplications(),
+ request.getIncludeChildQueues(),
+ request.getRecursive());
+ response.setQueueInfo(queueInfo);
+ } catch (IOException ioe) {
+ LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe);
+ throw RPCUtil.getRemoteException(ioe);
+ }
+
+ return response;
+ }
+
private NodeManagerInfo createNodeManagerInfo(NodeInfo nodeInfo) {
NodeManagerInfo node =
recordFactory.newRecordInstance(NodeManagerInfo.class);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Sun Apr 17 05:19:05 2011
@@ -179,7 +179,7 @@ public class ResourceManager extends Com
}
protected ClientRMService createClientRMService() {
- return new ClientRMService(applicationsManager, rmResourceTracker);
+ return new ClientRMService(applicationsManager, rmResourceTracker, scheduler);
}
protected ApplicationMasterService createApplicationMasterService() {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java Sun Apr 17 05:19:05 2011
@@ -34,6 +34,8 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.ApplicationStatus;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -315,4 +317,25 @@ public class Application {
synchronized public List<NodeInfo> getAllNodesForApplication() {
return new ArrayList<NodeInfo>(applicationOnNodes);
}
+
+
+ synchronized public org.apache.hadoop.yarn.api.records.Application
+ getApplicationInfo() {
+ org.apache.hadoop.yarn.api.records.Application application =
+ recordFactory.newRecordInstance(
+ org.apache.hadoop.yarn.api.records.Application.class);
+ application.setApplicationId(applicationId);
+ application.setMasterHost("");
+ application.setName("");
+ application.setQueue(queue.getQueueName());
+ application.setState(ApplicationState.RUNNING);
+ application.setUser(user);
+
+ ApplicationStatus status =
+ recordFactory.newRecordInstance(ApplicationStatus.class);
+ status.setApplicationId(applicationId);
+ application.setStatus(status);
+
+ return application;
+ }
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java Sun Apr 17 05:19:05 2011
@@ -1,28 +1,44 @@
/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
@Evolving
@LimitedPrivate("yarn")
public interface Queue {
+ /**
+ * Get the queue name
+ * @return queue name
+ */
String getQueueName();
+
+
+ /**
+ * Get queue information
+ * @param includeApplications include applications?
+ * @param includeChildQueues include child queues?
+ * @param recursive recursively get child queue information?
+ * @return queue information
+ */
+ QueueInfo getQueueInfo(boolean includeApplications,
+ boolean includeChildQueues, boolean recursive);
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java Sun Apr 17 05:19:05 2011
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
/**
@@ -43,5 +44,38 @@ public interface YarnScheduler {
*/
List<Container> allocate(ApplicationId applicationId,
List<ResourceRequest> ask, List<Container> release)
- throws IOException;
+ throws IOException;
+
+ /**
+ * A new application has been submitted to the ResourceManager
+ * @param applicationId application which has been submitted
+ * @param user application user
+ * @param queue queue to which the applications is being submitted
+ * @param priority application priority
+ */
+ public void addApplication(ApplicationId applicationId, String user,
+ String queue, Priority priority)
+ throws IOException;
+
+ /**
+ * A submitted application has completed.
+ * @param applicationId completed application
+ */
+ public void removeApplication(ApplicationId applicationId)
+ throws IOException;
+
+
+ /**
+ * Get queue information
+ * @param queueName queue name
+ * @param includeApplications include applications?
+ * @param includeChildQueues include child queues?
+ * @param recursive get children queues?
+ * @return queue information
+ * @throws IOException
+ */
+ public QueueInfo getQueueInfo(String queueName, boolean includeApplications,
+ boolean includeChildQueues, boolean recursive)
+ throws IOException;
+
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Sun Apr 17 05:19:05 2011
@@ -38,6 +38,7 @@ import org.apache.hadoop.security.Access
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -136,6 +137,7 @@ implements ResourceScheduler, CapacitySc
private void initializeQueues(CapacitySchedulerConfiguration conf) {
root = parseQueue(conf, null, ROOT);
+ queues.put(ROOT, root);
LOG.info("Initialized root queue " + root);
}
@@ -170,15 +172,7 @@ implements ResourceScheduler, CapacitySc
return queue;
}
- /**
- * Add an application to the capacity scheduler. This application needs to be
- * tracked.
- * @param applicationId the application id of this application
- * @param user the user who owns the application
- * @param queueName the queue which the application belongs to
- * @param priority the priority of the application
- * @throws IOException
- */
+ @Override
public void addApplication(ApplicationId applicationId,
String user, String queueName, Priority priority)
throws IOException {
@@ -209,12 +203,7 @@ implements ResourceScheduler, CapacitySc
", currently active: " + applications.size());
}
- /**
- * Remove an application. Releases the resources of the application and
- * then makes sure its removed from data structures of the scheduler.
- * @param applicationId the applicationId of the application
- * @throws IOException
- */
+ @Override
public void removeApplication(ApplicationId applicationId)
throws IOException {
Application application = getApplication(applicationId);
@@ -278,6 +267,18 @@ implements ResourceScheduler, CapacitySc
return allContainers;
}
+ @Override
+ public synchronized QueueInfo getQueueInfo(String queueName,
+ boolean includeApplications, boolean includeChildQueues, boolean recursive)
+ throws IOException {
+ Queue queue = this.queues.get(queueName);
+
+ if (queue == null) {
+ throw new IOException("Unknown queue: " + queueName);
+ }
+ return queue.getQueueInfo(includeApplications, includeChildQueues, recursive);
+ }
+
private void normalizeRequests(List<ResourceRequest> asks) {
for (ResourceRequest ask : asks) {
normalizeRequest(ask);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Sun Apr 17 05:19:05 2011
@@ -33,11 +33,14 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
@@ -55,7 +58,7 @@ public class LeafQueue implements Queue
private final Queue parent;
private final float capacity;
private final float absoluteCapacity;
- private final float maxCapacity;
+ private final float maximumCapacity;
private final float absoluteMaxCapacity;
private final int userLimit;
private final float userLimitFactor;
@@ -77,6 +80,13 @@ public class LeafQueue implements Queue
private Map<String, User> users = new HashMap<String, User>();
+ private QueueInfo queueInfo;
+ private Map<ApplicationId, org.apache.hadoop.yarn.api.records.Application>
+ applicationInfos;
+
+ private final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+
public LeafQueue(CapacitySchedulerContext cs,
String queueName, Queue parent,
Comparator<Application> applicationComparator) {
@@ -90,10 +100,10 @@ public class LeafQueue implements Queue
(float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
this.absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
- this.maxCapacity = cs.getConfiguration().getMaximumCapacity(getQueuePath());
+ this.maximumCapacity = cs.getConfiguration().getMaximumCapacity(getQueuePath());
this.absoluteMaxCapacity =
- (maxCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
- Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maxCapacity) / 100;
+ (maximumCapacity == CapacitySchedulerConfiguration.UNDEFINED) ?
+ Float.MAX_VALUE : (parent.getAbsoluteCapacity() * maximumCapacity) / 100;
this.userLimit = cs.getConfiguration().getUserLimit(getQueuePath());
@@ -105,12 +115,22 @@ public class LeafQueue implements Queue
this.maxApplicationsPerUser =
(int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
+ this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
+ this.queueInfo.setCapacity(capacity);
+ this.queueInfo.setMaximumCapacity(maximumCapacity);
+ this.queueInfo.setQueueName(queueName);
+ this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
+
+ this.applicationInfos =
+ new HashMap<ApplicationId,
+ org.apache.hadoop.yarn.api.records.Application>();
+
LOG.info("DEBUG --- LeafQueue:" +
" name=" + queueName +
", fullname=" + getQueuePath() +
", capacity=" + capacity +
", asboluteCapacity=" + absoluteCapacity +
- ", maxCapacity=" + maxCapacity +
+ ", maxCapacity=" + maximumCapacity +
", asboluteMaxCapacity=" + absoluteMaxCapacity +
", userLimit=" + userLimit + ", userLimitFactor=" + userLimitFactor +
", maxApplications=" + maxApplications +
@@ -131,7 +151,7 @@ public class LeafQueue implements Queue
@Override
public float getMaximumCapacity() {
- return maxCapacity;
+ return maximumCapacity;
}
@Override
@@ -195,6 +215,23 @@ public class LeafQueue implements Queue
return numContainers;
}
+ @Override
+ public synchronized QueueInfo getQueueInfo(boolean includeApplications,
+ boolean includeChildQueues, boolean recursive) {
+ queueInfo.setCurrentCapacity(usedCapacity);
+
+ if (includeApplications) {
+ queueInfo.setApplications(
+ new ArrayList<org.apache.hadoop.yarn.api.records.Application>(
+ applicationInfos.values()));
+ } else {
+ queueInfo.setApplications(
+ new ArrayList<org.apache.hadoop.yarn.api.records.Application>());
+ }
+
+ return queueInfo;
+ }
+
public String toString() {
return queueName + ":" + capacity + ":" + absoluteCapacity + ":" +
getUsedCapacity() + ":" + getUtilization() + ":" +
@@ -238,7 +275,9 @@ public class LeafQueue implements Queue
// Accept
user.submitApplication();
applications.add(application);
-
+ applicationInfos.put(application.getApplicationId(),
+ application.getApplicationInfo());
+
LOG.info("Application submission -" +
" appId: " + application.getApplicationId() +
" user: " + user + "," + " leaf-queue: " + getQueueName() +
@@ -263,6 +302,8 @@ public class LeafQueue implements Queue
users.remove(application.getUser());
}
+ applicationInfos.remove(application.getApplicationId());
+
LOG.info("Application completion -" +
" appId: " + application.getApplicationId() +
" user: " + application.getUser() +