You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/10/06 03:16:57 UTC
svn commit: r1179484 [3/6] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-client-a...
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Thu Oct 6 01:16:48 2011
@@ -62,7 +62,6 @@ import org.apache.hadoop.yarn.api.Applic
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -70,6 +69,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -99,7 +99,7 @@ public class YARNRunner implements Clien
}
/**
- * Similar to {@link #YARNRunner(Configuration)} but allowing injecting
+ * Similar to {@link #YARNRunner(Configuration)} but allowing injecting
* {@link ResourceMgrDelegate}. Enables mocking and testing.
* @param conf the configuration object for the client
* @param resMgrDelegate the resourcemanager client handle.
@@ -107,12 +107,12 @@ public class YARNRunner implements Clien
public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate) {
this(conf, resMgrDelegate, new ClientCache(conf, resMgrDelegate));
}
-
+
/**
- * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)}
+ * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)}
* but allowing injecting {@link ClientCache}. Enable mocking and testing.
* @param conf the configuration object
- * @param resMgrDelegate the resource manager delegate
+ * @param resMgrDelegate the resource manager delegate
* @param clientCache the client cache object.
*/
public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate,
@@ -126,7 +126,7 @@ public class YARNRunner implements Clien
throw new RuntimeException("Error in instantiating YarnClient", ufe);
}
}
-
+
@Override
public void cancelDelegationToken(Token<DelegationTokenIdentifier> arg0)
throws IOException, InterruptedException {
@@ -152,7 +152,7 @@ public class YARNRunner implements Clien
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
- InterruptedException {
+ InterruptedException {
return resMgrDelegate.getClusterMetrics();
}
@@ -209,13 +209,13 @@ public class YARNRunner implements Clien
public String getSystemDir() throws IOException, InterruptedException {
return resMgrDelegate.getSystemDir();
}
-
+
@Override
public long getTaskTrackerExpiryInterval() throws IOException,
InterruptedException {
return resMgrDelegate.getTaskTrackerExpiryInterval();
}
-
+
@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
throws IOException, InterruptedException {
@@ -230,20 +230,20 @@ public class YARNRunner implements Clien
}
// Construct necessary information to start the MR AM
- ApplicationSubmissionContext appContext =
+ ApplicationSubmissionContext appContext =
createApplicationSubmissionContext(conf, jobSubmitDir, ts);
-
+
// Submit to ResourceManager
ApplicationId applicationId = resMgrDelegate.submitApplication(appContext);
-
+
ApplicationReport appMaster = resMgrDelegate
.getApplicationReport(applicationId);
- String diagnostics =
- (appMaster == null ?
+ String diagnostics =
+ (appMaster == null ?
"application report is null" : appMaster.getDiagnostics());
- if (appMaster == null || appMaster.getState() == ApplicationState.FAILED
- || appMaster.getState() == ApplicationState.KILLED) {
- throw new IOException("Failed to run job : " +
+ if (appMaster == null || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED
+ || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) {
+ throw new IOException("Failed to run job : " +
diagnostics);
}
return clientCache.getClient(jobId).getJobStatus(jobId);
@@ -266,7 +266,7 @@ public class YARNRunner implements Clien
Configuration jobConf,
String jobSubmitDir, Credentials ts) throws IOException {
ApplicationId applicationId = resMgrDelegate.getApplicationId();
-
+
// Setup resource requirements
Resource capability = recordFactory.newRecordInstance(Resource.class);
capability.setMemory(conf.getInt(MRJobConfig.MR_AM_VMEM_MB,
@@ -276,9 +276,9 @@ public class YARNRunner implements Clien
// Setup LocalResources
Map<String, LocalResource> localResources =
new HashMap<String, LocalResource>();
-
+
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE);
-
+
URL yarnUrlForJobSubmitDir = ConverterUtils
.getYarnUrlFromPath(defaultFileContext.getDefaultFileSystem()
.resolvePath(
@@ -299,18 +299,18 @@ public class YARNRunner implements Clien
LOG.info("Job jar is not present. "
+ "Not adding any jar to the list of resources.");
}
-
+
// TODO gross hack
- for (String s : new String[] {
- MRJobConfig.JOB_SPLIT,
+ for (String s : new String[] {
+ MRJobConfig.JOB_SPLIT,
MRJobConfig.JOB_SPLIT_METAINFO,
MRJobConfig.APPLICATION_TOKENS_FILE }) {
localResources.put(
MRJobConfig.JOB_SUBMIT_DIR + "/" + s,
- createApplicationResource(defaultFileContext,
+ createApplicationResource(defaultFileContext,
new Path(jobSubmitDir, s)));
}
-
+
// Setup security tokens
ByteBuffer securityTokens = null;
if (UserGroupInformation.isSecurityEnabled()) {
@@ -322,20 +322,20 @@ public class YARNRunner implements Clien
// Setup the command to run the AM
Vector<CharSequence> vargs = new Vector<CharSequence>(8);
vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
-
+
long logSize = TaskLog.getTaskLogLength(new JobConf(conf));
vargs.add("-Dlog4j.configuration=container-log4j.properties");
vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "="
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize);
-
+
vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));
vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
- vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
+ vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Path.SEPARATOR + ApplicationConstants.STDOUT);
- vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
+ vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Path.SEPARATOR + ApplicationConstants.STDERR);
@@ -349,12 +349,12 @@ public class YARNRunner implements Clien
LOG.info("Command to launch container for ApplicationMaster is : "
+ mergedCommand);
-
- // Setup the CLASSPATH in environment
+
+ // Setup the CLASSPATH in environment
// i.e. add { job jar, CWD, Hadoop jars} to classpath.
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment);
-
+
// Parse distributed cache
MRApps.setupDistributedCache(jobConf, localResources);
@@ -374,12 +374,12 @@ public class YARNRunner implements Clien
appContext.setUser( // User name
UserGroupInformation.getCurrentUser().getShortUserName());
appContext.setQueue( // Queue name
- jobConf.get(JobContext.QUEUE_NAME,
+ jobConf.get(JobContext.QUEUE_NAME,
YarnConfiguration.DEFAULT_QUEUE_NAME));
appContext.setApplicationName( // Job name
- jobConf.get(JobContext.JOB_NAME,
- YarnConfiguration.DEFAULT_APPLICATION_NAME));
- appContext.setAMContainerSpec(amContainer); // AM Container
+ jobConf.get(JobContext.JOB_NAME,
+ YarnConfiguration.DEFAULT_APPLICATION_NAME));
+ appContext.setAMContainerSpec(amContainer); // AM Container
return appContext;
}
@@ -394,14 +394,14 @@ public class YARNRunner implements Clien
public long getProtocolVersion(String arg0, long arg1) throws IOException {
return resMgrDelegate.getProtocolVersion(arg0, arg1);
}
-
+
@Override
public long renewDelegationToken(Token<DelegationTokenIdentifier> arg0)
throws IOException, InterruptedException {
return resMgrDelegate.renewDelegationToken(arg0);
}
-
+
@Override
public Counters getJobCounters(JobID arg0) throws IOException,
InterruptedException {
@@ -419,7 +419,7 @@ public class YARNRunner implements Clien
JobStatus status = clientCache.getClient(jobID).getJobStatus(jobID);
return status;
}
-
+
@Override
public TaskCompletionEvent[] getTaskCompletionEvents(JobID arg0, int arg1,
int arg2) throws IOException, InterruptedException {
@@ -446,8 +446,8 @@ public class YARNRunner implements Clien
if (status.getState() != JobStatus.State.RUNNING) {
resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
return;
- }
-
+ }
+
try {
/* send a kill to the AM */
clientCache.getClient(arg0).killJob(arg0);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java Thu Oct 6 01:16:48 2011
@@ -88,7 +88,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -107,17 +108,17 @@ public class TestClientRedirect {
private static final Log LOG = LogFactory.getLog(TestClientRedirect.class);
private static final String RMADDRESS = "0.0.0.0:8054";
private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-
+
private static final String AMHOSTADDRESS = "0.0.0.0:10020";
private static final String HSHOSTADDRESS = "0.0.0.0:10021";
- private volatile boolean amContact = false;
+ private volatile boolean amContact = false;
private volatile boolean hsContact = false;
private volatile boolean amRunning = false;
private volatile boolean amRestarting = false;
@Test
public void testRedirect() throws Exception {
-
+
Configuration conf = new YarnConfiguration();
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.set(YarnConfiguration.RM_ADDRESS, RMADDRESS);
@@ -125,7 +126,7 @@ public class TestClientRedirect {
RMService rmService = new RMService("test");
rmService.init(conf);
rmService.start();
-
+
AMService amService = new AMService();
amService.init(conf);
amService.start(conf);
@@ -134,16 +135,16 @@ public class TestClientRedirect {
HistoryService historyService = new HistoryService();
historyService.init(conf);
historyService.start(conf);
-
+
LOG.info("services started");
Cluster cluster = new Cluster(conf);
org.apache.hadoop.mapreduce.JobID jobID =
new org.apache.hadoop.mapred.JobID("201103121733", 1);
- org.apache.hadoop.mapreduce.Counters counters =
+ org.apache.hadoop.mapreduce.Counters counters =
cluster.getJob(jobID).getCounters();
validateCounters(counters);
Assert.assertTrue(amContact);
-
+
LOG.info("Sleeping for 5 seconds before stop for" +
" the client socket to not get EOF immediately..");
Thread.sleep(5000);
@@ -155,17 +156,17 @@ public class TestClientRedirect {
LOG.info("Sleeping for 5 seconds after stop for" +
" the server to exit cleanly..");
Thread.sleep(5000);
-
+
amRestarting = true;
// Same client
//results are returned from fake (not started job)
counters = cluster.getJob(jobID).getCounters();
Assert.assertEquals(0, counters.countCounters());
Job job = cluster.getJob(jobID);
- org.apache.hadoop.mapreduce.TaskID taskId =
+ org.apache.hadoop.mapreduce.TaskID taskId =
new org.apache.hadoop.mapreduce.TaskID(jobID, TaskType.MAP, 0);
TaskAttemptID tId = new TaskAttemptID(taskId, 0);
-
+
//invoke all methods to check that no exception is thrown
job.killJob();
job.killTask(tId);
@@ -175,25 +176,25 @@ public class TestClientRedirect {
job.getTaskDiagnostics(tId);
job.getTaskReports(TaskType.MAP);
job.getTrackingURL();
-
+
amRestarting = false;
amService = new AMService();
amService.init(conf);
amService.start(conf);
amRunning = true;
amContact = false; //reset
-
+
counters = cluster.getJob(jobID).getCounters();
validateCounters(counters);
Assert.assertTrue(amContact);
-
+
amRunning = false;
// Same client
counters = cluster.getJob(jobID).getCounters();
validateCounters(counters);
Assert.assertTrue(hsContact);
-
+
rmService.stop();
historyService.stop();
}
@@ -248,7 +249,7 @@ public class TestClientRedirect {
public GetNewApplicationResponse getNewApplication(GetNewApplicationRequest request) throws YarnRemoteException {
return null;
}
-
+
@Override
public GetApplicationReportResponse getApplicationReport(
GetApplicationReportRequest request) throws YarnRemoteException {
@@ -256,12 +257,14 @@ public class TestClientRedirect {
ApplicationReport application = recordFactory
.newRecordInstance(ApplicationReport.class);
application.setApplicationId(applicationId);
+ application.setFinalApplicationStatus(FinalApplicationStatus.UNDEFINED);
if (amRunning) {
- application.setState(ApplicationState.RUNNING);
+ application.setYarnApplicationState(YarnApplicationState.RUNNING);
} else if (amRestarting) {
- application.setState(ApplicationState.SUBMITTED);
+ application.setYarnApplicationState(YarnApplicationState.SUBMITTED);
} else {
- application.setState(ApplicationState.SUCCEEDED);
+ application.setYarnApplicationState(YarnApplicationState.FINISHED);
+ application.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
}
String[] split = AMHOSTADDRESS.split(":");
application.setHost(split[0]);
@@ -339,7 +342,7 @@ public class TestClientRedirect {
}
}
- class AMService extends AbstractService
+ class AMService extends AbstractService
implements MRClientProtocol {
private InetSocketAddress bindAddress;
private Server server;
@@ -347,7 +350,7 @@ public class TestClientRedirect {
public AMService() {
this(AMHOSTADDRESS);
}
-
+
public AMService(String hostAddress) {
super("AMService");
this.hostAddress = hostAddress;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java Thu Oct 6 01:16:48 2011
@@ -32,8 +32,9 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.ipc.RPCUtil;
@@ -124,6 +125,26 @@ public class TestClientServiceDelegate {
Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
}
+
+ @Test
+ public void testJobReportFromHistoryServer() throws Exception {
+ MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
+ when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(
+ getJobReportResponseFromHistoryServer());
+ ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
+ when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
+ .thenReturn(null);
+ ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
+ historyServerProxy, rm);
+
+ JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
+ Assert.assertNotNull(jobStatus);
+ Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());
+ Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());
+ Assert.assertEquals(1.0f, jobStatus.getMapProgress());
+ Assert.assertEquals(1.0f, jobStatus.getReduceProgress());
+ }
+
private GetJobReportRequest getJobReportRequest() {
GetJobReportRequest request = Records.newRecord(GetJobReportRequest.class);
request.setJobId(jobId);
@@ -143,7 +164,7 @@ public class TestClientServiceDelegate {
private ApplicationReport getApplicationReport() {
ApplicationReport applicationReport = Records
.newRecord(ApplicationReport.class);
- applicationReport.setState(ApplicationState.SUCCEEDED);
+ applicationReport.setYarnApplicationState(YarnApplicationState.FINISHED);
applicationReport.setUser("root");
applicationReport.setHost("N/A");
applicationReport.setName("N/A");
@@ -152,6 +173,7 @@ public class TestClientServiceDelegate {
applicationReport.setFinishTime(0);
applicationReport.setTrackingUrl("N/A");
applicationReport.setDiagnostics("N/A");
+ applicationReport.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
return applicationReport;
}
@@ -170,4 +192,17 @@ public class TestClientServiceDelegate {
return clientServiceDelegate;
}
+ private GetJobReportResponse getJobReportResponseFromHistoryServer() {
+ GetJobReportResponse jobReportResponse = Records
+ .newRecord(GetJobReportResponse.class);
+ JobReport jobReport = Records.newRecord(JobReport.class);
+ jobReport.setJobId(jobId);
+ jobReport.setJobState(JobState.SUCCEEDED);
+ jobReport.setMapProgress(1.0f);
+ jobReport.setReduceProgress(1.0f);
+ jobReport.setJobFile("TestJobFilePath");
+ jobReport.setTrackingUrl("TestTrackingUrl");
+ jobReportResponse.setJobReport(jobReport);
+ return jobReportResponse;
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Thu Oct 6 01:16:48 2011
@@ -273,9 +273,9 @@ public class TestMRJobs {
TaskCompletionEvent[] events = job.getTaskCompletionEvents(0, 2);
Assert.assertEquals(TaskCompletionEvent.Status.FAILED,
- events[0].getStatus().FAILED);
- Assert.assertEquals(TaskCompletionEvent.Status.FAILED,
- events[1].getStatus().FAILED);
+ events[0].getStatus());
+ Assert.assertEquals(TaskCompletionEvent.Status.TIPFAILED,
+ events[1].getStatus());
Assert.assertEquals(JobStatus.State.FAILED, job.getJobState());
verifyFailingMapperCounters(job);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java Thu Oct 6 01:16:48 2011
@@ -36,7 +36,6 @@ import org.apache.hadoop.mapreduce.Count
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.junit.Before;
import org.junit.After;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java Thu Oct 6 01:16:48 2011
@@ -64,10 +64,10 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -77,25 +77,25 @@ import org.mockito.invocation.Invocation
import org.mockito.stubbing.Answer;
/**
- * Test YarnRunner and make sure the client side plugin works
+ * Test YarnRunner and make sure the client side plugin works
* fine
*/
public class TestYARNRunner extends TestCase {
private static final Log LOG = LogFactory.getLog(TestYARNRunner.class);
private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-
+
private YARNRunner yarnRunner;
private ResourceMgrDelegate resourceMgrDelegate;
private YarnConfiguration conf;
private ClientCache clientCache;
private ApplicationId appId;
private JobID jobId;
- private File testWorkDir =
+ private File testWorkDir =
new File("target", TestYARNRunner.class.getName());
private ApplicationSubmissionContext submissionContext;
private ClientServiceDelegate clientDelegate;
private static final String failString = "Rejected job";
-
+
@Before
public void setUp() throws Exception {
resourceMgrDelegate = mock(ResourceMgrDelegate.class);
@@ -115,7 +115,7 @@ public class TestYARNRunner extends Test
}
).when(yarnRunner).createApplicationSubmissionContext(any(Configuration.class),
any(String.class), any(Credentials.class));
-
+
appId = recordFactory.newRecordInstance(ApplicationId.class);
appId.setClusterTimestamp(System.currentTimeMillis());
appId.setId(1);
@@ -125,13 +125,13 @@ public class TestYARNRunner extends Test
}
testWorkDir.mkdirs();
}
-
-
+
+
@Test
public void testJobKill() throws Exception {
clientDelegate = mock(ClientServiceDelegate.class);
- when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new
- org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f,
+ when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new
+ org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f,
State.PREP, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp"));
when(clientDelegate.killJob(any(JobID.class))).thenReturn(true);
doAnswer(
@@ -145,13 +145,13 @@ public class TestYARNRunner extends Test
).when(clientCache).getClient(any(JobID.class));
yarnRunner.killJob(jobId);
verify(resourceMgrDelegate).killApplication(appId);
- when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new
- org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f,
+ when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new
+ org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f,
State.RUNNING, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp"));
yarnRunner.killJob(jobId);
verify(clientDelegate).killJob(jobId);
}
-
+
@Test
public void testJobSubmissionFailure() throws Exception {
when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class))).
@@ -159,7 +159,7 @@ public class TestYARNRunner extends Test
ApplicationReport report = mock(ApplicationReport.class);
when(report.getApplicationId()).thenReturn(appId);
when(report.getDiagnostics()).thenReturn(failString);
- when(report.getState()).thenReturn(ApplicationState.FAILED);
+ when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED);
when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report);
Credentials credentials = new Credentials();
File jobxml = new File(testWorkDir, "job.xml");
@@ -167,13 +167,13 @@ public class TestYARNRunner extends Test
conf.writeXml(out);
out.close();
try {
- yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
+ yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
} catch(IOException io) {
LOG.info("Logging exception:", io);
assertTrue(io.getLocalizedMessage().contains(failString));
}
}
-
+
@Test
public void testResourceMgrDelegate() throws Exception {
/* we not want a mock of resourcemgr deleagte */
@@ -184,19 +184,19 @@ public class TestYARNRunner extends Test
.thenReturn(null);
delegate.killApplication(appId);
verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class));
-
+
/* make sure getalljobs calls get all applications */
when(clientRMProtocol.getAllApplications(any(GetAllApplicationsRequest.class))).
thenReturn(recordFactory.newRecordInstance(GetAllApplicationsResponse.class));
delegate.getAllJobs();
verify(clientRMProtocol).getAllApplications(any(GetAllApplicationsRequest.class));
-
+
/* make sure getapplication report is called */
when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class)))
.thenReturn(recordFactory.newRecordInstance(GetApplicationReportResponse.class));
delegate.getApplicationReport(appId);
verify(clientRMProtocol).getApplicationReport(any(GetApplicationReportRequest.class));
-
+
/* make sure metrics is called */
GetClusterMetricsResponse clusterMetricsResponse = recordFactory.newRecordInstance
(GetClusterMetricsResponse.class);
@@ -206,7 +206,7 @@ public class TestYARNRunner extends Test
.thenReturn(clusterMetricsResponse);
delegate.getClusterMetrics();
verify(clientRMProtocol).getClusterMetrics(any(GetClusterMetricsRequest.class));
-
+
when(clientRMProtocol.getClusterNodes(any(GetClusterNodesRequest.class))).
thenReturn(recordFactory.newRecordInstance(GetClusterNodesResponse.class));
delegate.getActiveTrackers();
@@ -227,7 +227,7 @@ public class TestYARNRunner extends Test
thenReturn(queueInfoResponse);
delegate.getQueues();
verify(clientRMProtocol).getQueueInfo(any(GetQueueInfoRequest.class));
-
+
GetQueueUserAclsInfoResponse aclResponse = recordFactory.newRecordInstance(
GetQueueUserAclsInfoResponse.class);
when(clientRMProtocol.getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class)))
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn Thu Oct 6 01:16:48 2011
@@ -47,7 +47,7 @@
# YARN_ROOT_LOGGER The root appender. Default is INFO,console
#
-bin=`dirname "$0"`
+bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
. "$bin"/yarn-config.sh
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh Thu Oct 6 01:16:48 2011
@@ -36,7 +36,7 @@ if [ $# -le 1 ]; then
exit 1
fi
-bin=`dirname "$0"`
+bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
. "$bin"/yarn-config.sh
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java Thu Oct 6 01:16:48 2011
@@ -22,15 +22,16 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.AMRMProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
/**
- * <p>The finalization request sent by the <code>ApplicationMaster</code> to
+ * <p>The finalization request sent by the <code>ApplicationMaster</code> to
* inform the <code>ResourceManager</code> about its completion.</p>
- *
+ *
* <p>The final request includes details such:
* <ul>
* <li>
- * {@link ApplicationAttemptId} being managed by the
+ * {@link ApplicationAttemptId} being managed by the
* <code>ApplicationMaster</code>
* </li>
* <li>Final state of the <code>ApplicationMaster</code></li>
@@ -47,19 +48,19 @@ import org.apache.hadoop.yarn.api.record
public interface FinishApplicationMasterRequest {
/**
- * Get the <code>ApplicationAttemptId</code> being managed by the
+ * Get the <code>ApplicationAttemptId</code> being managed by the
* <code>ApplicationMaster</code>.
- * @return <code>ApplicationAttemptId</code> being managed by the
+ * @return <code>ApplicationAttemptId</code> being managed by the
* <code>ApplicationMaster</code>
*/
@Public
@Stable
ApplicationAttemptId getApplicationAttemptId();
-
+
/**
- * Set the <code>ApplicationAttemptId</code> being managed by the
+ * Set the <code>ApplicationAttemptId</code> being managed by the
* <code>ApplicationMaster</code>.
- * @param applicationAttemptId <code>ApplicationAttemptId</code> being managed
+ * @param applicationAttemptId <code>ApplicationAttemptId</code> being managed
* by the <code>ApplicationMaster</code>
*/
@Public
@@ -72,15 +73,15 @@ public interface FinishApplicationMaster
*/
@Public
@Stable
- String getFinalState();
-
+ FinalApplicationStatus getFinalApplicationStatus();
+
/**
- * Set <em>final state</em> of the <code>ApplicationMaster</code>
- * @param finalState <em>final state</em> of the <code>ApplicationMaster</code>
+ * Set the <em>finish state</em> of the <code>ApplicationMaster</code>
+ * @param finishState <em>finish state</em> of the <code>ApplicationMaster</code>
*/
@Public
@Stable
- void setFinalState(String finalState);
+ void setFinishApplicationStatus(FinalApplicationStatus finishState);
/**
* Get <em>diagnostic information</em> on application failure.
@@ -89,7 +90,7 @@ public interface FinishApplicationMaster
@Public
@Stable
String getDiagnostics();
-
+
/**
* Set <em>diagnostic information</em> on application failure.
* @param diagnostics <em>diagnostic information</em> on application failure
@@ -105,10 +106,10 @@ public interface FinishApplicationMaster
@Public
@Stable
String getTrackingUrl();
-
+
/**
* Set the <em>tracking URL</em>for the <code>ApplicationMaster</code>
- * @param url <em>tracking URL</em>for the
+ * @param url <em>tracking URL</em>for the
* <code>ApplicationMaster</code>
*/
@Public
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java Thu Oct 6 01:16:48 2011
@@ -21,23 +21,24 @@ package org.apache.hadoop.yarn.api.proto
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.util.ProtoUtils;
-
public class FinishApplicationMasterRequestPBImpl extends ProtoBase<FinishApplicationMasterRequestProto> implements FinishApplicationMasterRequest {
FinishApplicationMasterRequestProto proto = FinishApplicationMasterRequestProto.getDefaultInstance();
FinishApplicationMasterRequestProto.Builder builder = null;
boolean viaProto = false;
-
+
private ApplicationAttemptId appAttemptId = null;
-
-
+
+
public FinishApplicationMasterRequestPBImpl() {
builder = FinishApplicationMasterRequestProto.newBuilder();
}
@@ -46,7 +47,7 @@ public class FinishApplicationMasterRequ
this.proto = proto;
viaProto = true;
}
-
+
public FinishApplicationMasterRequestProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
@@ -61,7 +62,7 @@ public class FinishApplicationMasterRequ
}
private void mergeLocalToProto() {
- if (viaProto)
+ if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
@@ -74,8 +75,7 @@ public class FinishApplicationMasterRequ
}
viaProto = false;
}
-
-
+
@Override
public ApplicationAttemptId getApplicationAttemptId() {
FinishApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder;
@@ -92,7 +92,7 @@ public class FinishApplicationMasterRequ
@Override
public void setAppAttemptId(ApplicationAttemptId applicationAttemptId) {
maybeInitBuilder();
- if (applicationAttemptId == null)
+ if (applicationAttemptId == null)
builder.clearApplicationAttemptId();
this.appAttemptId = applicationAttemptId;
}
@@ -122,15 +122,22 @@ public class FinishApplicationMasterRequ
}
@Override
- public String getFinalState() {
+ public FinalApplicationStatus getFinalApplicationStatus() {
FinishApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder;
- return p.getFinalState();
+ if (!p.hasFinalApplicationStatus()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getFinalApplicationStatus());
}
@Override
- public void setFinalState(String state) {
+ public void setFinishApplicationStatus(FinalApplicationStatus finishState) {
maybeInitBuilder();
- builder.setFinalState(state);
+ if (finishState == null) {
+ builder.clearFinalApplicationStatus();
+ return;
+ }
+ builder.setFinalApplicationStatus(convertToProtoFormat(finishState));
}
private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) {
@@ -141,6 +148,13 @@ public class FinishApplicationMasterRequ
return ((ApplicationAttemptIdPBImpl)t).getProto();
}
+ private FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto s) {
+ return ProtoUtils.convertFromProtoFormat(s);
+ }
+
+ private FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus s) {
+ return ProtoUtils.convertToProtoFormat(s);
+ }
-}
+}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java Thu Oct 6 01:16:48 2011
@@ -23,24 +23,22 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder;
-
-public class RegisterApplicationMasterResponsePBImpl
-extends ProtoBase<RegisterApplicationMasterResponseProto>
+public class RegisterApplicationMasterResponsePBImpl
+extends ProtoBase<RegisterApplicationMasterResponseProto>
implements RegisterApplicationMasterResponse {
- RegisterApplicationMasterResponseProto proto =
+ RegisterApplicationMasterResponseProto proto =
RegisterApplicationMasterResponseProto.getDefaultInstance();
RegisterApplicationMasterResponseProto.Builder builder = null;
boolean viaProto = false;
-
+
private Resource minimumResourceCapability;
private Resource maximumResourceCapability;
-
+
public RegisterApplicationMasterResponsePBImpl() {
builder = RegisterApplicationMasterResponseProto.newBuilder();
}
@@ -49,16 +47,16 @@ implements RegisterApplicationMasterResp
this.proto = proto;
viaProto = true;
}
-
+
public RegisterApplicationMasterResponseProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
-
+
private void mergeLocalToProto() {
- if (viaProto)
+ if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
@@ -94,7 +92,7 @@ implements RegisterApplicationMasterResp
if (!p.hasMaximumCapability()) {
return null;
}
-
+
this.maximumResourceCapability = convertFromProtoFormat(p.getMaximumCapability());
return this.maximumResourceCapability;
}
@@ -109,7 +107,7 @@ implements RegisterApplicationMasterResp
if (!p.hasMinimumCapability()) {
return null;
}
-
+
this.minimumResourceCapability = convertFromProtoFormat(p.getMinimumCapability());
return this.minimumResourceCapability;
}
@@ -140,4 +138,4 @@ implements RegisterApplicationMasterResp
return ((ResourcePBImpl)resource).getProto();
}
-}
+}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java Thu Oct 6 01:16:48 2011
@@ -22,38 +22,38 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
- * <em>For internal use only...</em>
+ * <em>For internal use only...</em>
*/
@Private
@Unstable
public interface ApplicationMaster {
ApplicationId getApplicationId();
void setApplicationId(ApplicationId appId);
-
+
String getHost();
void setHost(String host);
-
+
int getRpcPort();
void setRpcPort(int rpcPort);
-
+
String getTrackingUrl();
void setTrackingUrl(String url);
-
+
ApplicationStatus getStatus();
void setStatus(ApplicationStatus status);
-
- ApplicationState getState();
- void setState(ApplicationState state);
-
+
+ YarnApplicationState getState();
+ void setState(YarnApplicationState state);
+
String getClientToken();
void setClientToken(String clientToken);
-
+
int getAMFailCount();
void setAMFailCount(int amFailCount);
-
+
int getContainerCount();
void setContainerCount(int containerCount);
-
+
String getDiagnostics();
void setDiagnostics(String diagnostics);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java Thu Oct 6 01:16:48 2011
@@ -36,13 +36,13 @@ import org.apache.hadoop.yarn.api.Client
* <li>Host on which the <code>ApplicationMaster</code>is running.</li>
* <li>RPC port of the <code>ApplicationMaster</code>.</li>
* <li>Tracking URL.</li>
- * <li>{@link ApplicationState} of the application.</li>
+ * <li>{@link YarnApplicationState} of the application.</li>
* <li>Diagnostic information in case of errors.</li>
* <li>Start time of the application.</li>
* <li>Client token of the application (if security is enabled).</li>
* </ul>
* </p>
- *
+ *
* @see ClientRMProtocol#getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest)
*/
@Public
@@ -56,7 +56,7 @@ public interface ApplicationReport {
@Public
@Stable
ApplicationId getApplicationId();
-
+
@Private
@Unstable
void setApplicationId(ApplicationId applicationId);
@@ -68,7 +68,7 @@ public interface ApplicationReport {
@Public
@Stable
String getUser();
-
+
@Private
@Unstable
void setUser(String user);
@@ -80,7 +80,7 @@ public interface ApplicationReport {
@Public
@Stable
String getQueue();
-
+
@Private
@Unstable
void setQueue(String queue);
@@ -92,21 +92,21 @@ public interface ApplicationReport {
@Public
@Stable
String getName();
-
+
@Private
@Unstable
void setName(String name);
/**
- * Get the <em>host</em> on which the <code>ApplicationMaster</code>
+ * Get the <em>host</em> on which the <code>ApplicationMaster</code>
* is running.
- * @return <em>host</em> on which the <code>ApplicationMaster</code>
+ * @return <em>host</em> on which the <code>ApplicationMaster</code>
* is running
*/
@Public
@Stable
String getHost();
-
+
@Private
@Unstable
void setHost(String host);
@@ -118,47 +118,47 @@ public interface ApplicationReport {
@Public
@Stable
int getRpcPort();
-
+
@Private
@Unstable
void setRpcPort(int rpcPort);
/**
- * Get the <em>client token</em> for communicating with the
+ * Get the <em>client token</em> for communicating with the
* <code>ApplicationMaster</code>.
- * @return <em>client token</em> for communicating with the
+ * @return <em>client token</em> for communicating with the
* <code>ApplicationMaster</code>
*/
@Public
@Stable
String getClientToken();
-
+
@Private
@Unstable
void setClientToken(String clientToken);
/**
- * Get the <code>ApplicationState</code> of the application.
- * @return <code>ApplicationState</code> of the application
+ * Get the <code>YarnApplicationState</code> of the application.
+ * @return <code>YarnApplicationState</code> of the application
*/
@Public
@Stable
- ApplicationState getState();
-
+ YarnApplicationState getYarnApplicationState();
+
@Private
@Unstable
- void setState(ApplicationState state);
+ void setYarnApplicationState(YarnApplicationState state);
/**
- * Get the <em>diagnositic information</em> of the application in case of
+ * Get the <em>diagnositic information</em> of the application in case of
* errors.
- * @return <em>diagnositic information</em> of the application in case
+ * @return <em>diagnositic information</em> of the application in case
* of errors
*/
@Public
@Stable
String getDiagnostics();
-
+
@Private
@Unstable
void setDiagnostics(String diagnostics);
@@ -170,11 +170,11 @@ public interface ApplicationReport {
@Public
@Stable
String getTrackingUrl();
-
+
@Private
@Unstable
void setTrackingUrl(String url);
-
+
/**
* Get the <em>start time</em> of the application.
* @return <em>start time</em> of the application
@@ -182,7 +182,7 @@ public interface ApplicationReport {
@Public
@Stable
long getStartTime();
-
+
@Private
@Unstable
void setStartTime(long startTime);
@@ -194,8 +194,21 @@ public interface ApplicationReport {
@Public
@Stable
long getFinishTime();
-
+
@Private
@Unstable
void setFinishTime(long finishTime);
+
+
+ /**
+ * Get the <em>final finish status</em> of the application.
+ */
+ @Public
+ @Stable
+ FinalApplicationStatus getFinalApplicationStatus();
+
+ @Private
+ @Unstable
+ void setFinalApplicationStatus(FinalApplicationStatus finishState);
+
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMasterPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMasterPBImpl.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMasterPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMasterPBImpl.java Thu Oct 6 01:16:48 2011
@@ -19,31 +19,28 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
-import java.util.List;
-
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationMaster;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationStatus;
import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMasterProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMasterProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationStatusProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import org.apache.hadoop.yarn.util.ProtoUtils;
-
public class ApplicationMasterPBImpl extends ProtoBase<ApplicationMasterProto> implements ApplicationMaster {
ApplicationMasterProto proto = ApplicationMasterProto.getDefaultInstance();
ApplicationMasterProto.Builder builder = null;
boolean viaProto = false;
-
+
private ApplicationId applicationId = null;
private ApplicationStatus applicationStatus = null;
-
-
+
+
public ApplicationMasterPBImpl() {
builder = ApplicationMasterProto.newBuilder();
}
@@ -52,9 +49,9 @@ public class ApplicationMasterPBImpl ext
this.proto = proto;
viaProto = true;
}
-
+
public ApplicationMasterProto getProto() {
-
+
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
@@ -72,24 +69,24 @@ public class ApplicationMasterPBImpl ext
}
private void mergeLocalToProto() {
- if (viaProto)
+ if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
-
+
viaProto = true;
}
-
+
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ApplicationMasterProto.newBuilder(proto);
}
viaProto = false;
}
-
-
+
+
@Override
- public ApplicationState getState() {
+ public YarnApplicationState getState() {
ApplicationMasterProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasState()) {
return null;
@@ -98,7 +95,7 @@ public class ApplicationMasterPBImpl ext
}
@Override
- public void setState(ApplicationState state) {
+ public void setState(YarnApplicationState state) {
maybeInitBuilder();
if (state == null) {
builder.clearState();
@@ -124,7 +121,7 @@ public class ApplicationMasterPBImpl ext
}
builder.setHost((host));
}
-
+
@Override
public ApplicationId getApplicationId() {
ApplicationMasterProtoOrBuilder p = viaProto ? proto : builder;
@@ -135,7 +132,7 @@ public class ApplicationMasterPBImpl ext
return null;
}
applicationId = convertFromProtoFormat(p.getApplicationId());
-
+
return applicationId;
}
@@ -145,7 +142,7 @@ public class ApplicationMasterPBImpl ext
if (applicationId == null)
builder.clearApplicationId();
this.applicationId = applicationId;
-
+
}
@Override
public int getRpcPort() {
@@ -179,7 +176,7 @@ public class ApplicationMasterPBImpl ext
return null;
}
this.applicationStatus = convertFromProtoFormat(p.getStatus());
-
+
return this.applicationStatus;
}
@@ -189,7 +186,7 @@ public class ApplicationMasterPBImpl ext
if (status == null)
builder.clearStatus();
this.applicationStatus = status;
-
+
}
@Override
public String getClientToken() {
@@ -209,7 +206,7 @@ public class ApplicationMasterPBImpl ext
}
builder.setClientToken((clientToken));
}
-
+
@Override
public int getAMFailCount() {
ApplicationMasterProtoOrBuilder p = viaProto ? proto : builder;
@@ -250,11 +247,11 @@ public class ApplicationMasterPBImpl ext
builder.setDiagnostics(diagnostics);
}
- private ApplicationStateProto convertToProtoFormat(ApplicationState e) {
+ private YarnApplicationStateProto convertToProtoFormat(YarnApplicationState e) {
return ProtoUtils.convertToProtoFormat(e);
}
- private ApplicationState convertFromProtoFormat(ApplicationStateProto e) {
+ private YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto e) {
return ProtoUtils.convertFromProtoFormat(e);
}
@@ -274,4 +271,4 @@ public class ApplicationMasterPBImpl ext
return ((ApplicationStatusPBImpl)t).getProto();
}
-}
+}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Thu Oct 6 01:16:48 2011
@@ -18,17 +18,19 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import org.apache.hadoop.yarn.util.ProtoUtils;
-public class ApplicationReportPBImpl extends ProtoBase<ApplicationReportProto>
+public class ApplicationReportPBImpl extends ProtoBase<ApplicationReportProto>
implements ApplicationReport {
ApplicationReportProto proto = ApplicationReportProto.getDefaultInstance();
ApplicationReportProto.Builder builder = null;
@@ -39,7 +41,7 @@ implements ApplicationReport {
public ApplicationReportPBImpl() {
builder = ApplicationReportProto.newBuilder();
}
-
+
public ApplicationReportPBImpl(ApplicationReportProto proto) {
this.proto = proto;
viaProto = true;
@@ -87,12 +89,12 @@ implements ApplicationReport {
}
@Override
- public ApplicationState getState() {
+ public YarnApplicationState getYarnApplicationState() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasState()) {
+ if (!p.hasYarnApplicationState()) {
return null;
}
- return convertFromProtoFormat(p.getState());
+ return convertFromProtoFormat(p.getYarnApplicationState());
}
@Override
@@ -139,6 +141,27 @@ implements ApplicationReport {
}
@Override
+ public long getStartTime() {
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getStartTime();
+ }
+
+ @Override
+ public long getFinishTime() {
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getFinishTime();
+ }
+
+ @Override
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasFinalApplicationStatus()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getFinalApplicationStatus());
+ }
+
+ @Override
public void setApplicationId(ApplicationId applicationId) {
maybeInitBuilder();
if (applicationId == null)
@@ -177,13 +200,13 @@ implements ApplicationReport {
}
@Override
- public void setState(ApplicationState state) {
+ public void setYarnApplicationState(YarnApplicationState state) {
maybeInitBuilder();
if (state == null) {
- builder.clearState();
+ builder.clearYarnApplicationState();
return;
}
- builder.setState(convertToProtoFormat(state));
+ builder.setYarnApplicationState(convertToProtoFormat(state));
}
@Override
@@ -233,35 +256,33 @@ implements ApplicationReport {
}
@Override
- public ApplicationReportProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public long getStartTime() {
- ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
- return p.getStartTime();
- }
-
- @Override
public void setStartTime(long startTime) {
maybeInitBuilder();
builder.setStartTime(startTime);
}
@Override
- public long getFinishTime() {
- ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
- return p.getFinishTime();
+ public void setFinishTime(long finishTime) {
+ maybeInitBuilder();
+ builder.setFinishTime(finishTime);
}
@Override
- public void setFinishTime(long finishTime) {
+ public void setFinalApplicationStatus(FinalApplicationStatus finishState) {
maybeInitBuilder();
- builder.setFinishTime(finishTime);
+ if (finishState == null) {
+ builder.clearFinalApplicationStatus();
+ return;
+ }
+ builder.setFinalApplicationStatus(convertToProtoFormat(finishState));
+ }
+
+ @Override
+ public ApplicationReportProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
}
private void mergeLocalToBuilder() {
@@ -291,16 +312,25 @@ implements ApplicationReport {
return ((ApplicationIdPBImpl) t).getProto();
}
- private ApplicationState convertFromProtoFormat(ApplicationStateProto s) {
+ private ApplicationIdPBImpl convertFromProtoFormat(
+ ApplicationIdProto applicationId) {
+ return new ApplicationIdPBImpl(applicationId);
+ }
+
+ private YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto s) {
return ProtoUtils.convertFromProtoFormat(s);
}
- private ApplicationStateProto convertToProtoFormat(ApplicationState s) {
+ private YarnApplicationStateProto convertToProtoFormat(YarnApplicationState s) {
return ProtoUtils.convertToProtoFormat(s);
}
- private ApplicationIdPBImpl convertFromProtoFormat(
- ApplicationIdProto applicationId) {
- return new ApplicationIdPBImpl(applicationId);
+ private FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto s) {
+ return ProtoUtils.convertFromProtoFormat(s);
}
+
+ private FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus s) {
+ return ProtoUtils.convertToProtoFormat(s);
+ }
+
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java Thu Oct 6 01:16:48 2011
@@ -20,24 +20,26 @@ package org.apache.hadoop.yarn.util;
import java.nio.ByteBuffer;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationStateProto;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.ByteString;
public class ProtoUtils {
-
-
+
+
/*
* ContainerState
*/
@@ -48,18 +50,29 @@ public class ProtoUtils {
public static ContainerState convertFromProtoFormat(ContainerStateProto e) {
return ContainerState.valueOf(e.name().replace(CONTAINER_STATE_PREFIX, ""));
}
-
+
/*
- * ApplicationState
+ * YarnApplicationState
*/
- public static ApplicationStateProto convertToProtoFormat(ApplicationState e) {
- return ApplicationStateProto.valueOf(e.name());
+ public static YarnApplicationStateProto convertToProtoFormat(YarnApplicationState e) {
+ return YarnApplicationStateProto.valueOf(e.name());
}
- public static ApplicationState convertFromProtoFormat(ApplicationStateProto e) {
- return ApplicationState.valueOf(e.name());
+ public static YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto e) {
+ return YarnApplicationState.valueOf(e.name());
}
-
+
+ /*
+ * FinalApplicationStatus
+ */
+ private static String FINAL_APPLICATION_STATUS_PREFIX = "APP_";
+ public static FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus e) {
+ return FinalApplicationStatusProto.valueOf(FINAL_APPLICATION_STATUS_PREFIX + e.name());
+ }
+ public static FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto e) {
+ return FinalApplicationStatus.valueOf(e.name().replace(FINAL_APPLICATION_STATUS_PREFIX, ""));
+ }
+
/*
* LocalResourceType
*/
@@ -69,7 +82,7 @@ public class ProtoUtils {
public static LocalResourceType convertFromProtoFormat(LocalResourceTypeProto e) {
return LocalResourceType.valueOf(e.name());
}
-
+
/*
* LocalResourceVisibility
*/
@@ -79,7 +92,7 @@ public class ProtoUtils {
public static LocalResourceVisibility convertFromProtoFormat(LocalResourceVisibilityProto e) {
return LocalResourceVisibility.valueOf(e.name());
}
-
+
/*
* ByteBuffer
*/
@@ -98,7 +111,7 @@ public class ProtoUtils {
byteBuffer.position(oldPos);
return bs;
}
-
+
/*
* QueueState
*/
@@ -109,7 +122,7 @@ public class ProtoUtils {
public static QueueState convertFromProtoFormat(QueueStateProto e) {
return QueueState.valueOf(e.name().replace(QUEUE_STATE_PREFIX, ""));
}
-
+
/*
* QueueACL
*/
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Thu Oct 6 01:16:48 2011
@@ -76,14 +76,20 @@ message ContainerProto {
optional ContainerStatusProto container_status = 8;
}
-enum ApplicationStateProto {
+enum YarnApplicationStateProto {
NEW = 1;
SUBMITTED = 2;
RUNNING = 3;
- RESTARTING = 4;
- SUCCEEDED = 5;
- FAILED = 6;
- KILLED = 7;
+ FINISHED = 4;
+ FAILED = 5;
+ KILLED = 6;
+}
+
+enum FinalApplicationStatusProto {
+ APP_UNDEFINED = 0;
+ APP_SUCCEEDED = 1;
+ APP_FAILED = 2;
+ APP_KILLED = 3;
}
message ApplicationStatusProto {
@@ -98,7 +104,7 @@ message ApplicationMasterProto {
optional int32 rpc_port = 3;
optional string trackingUrl = 4;
optional ApplicationStatusProto status = 5;
- optional ApplicationStateProto state = 6;
+ optional YarnApplicationStateProto state = 6;
optional string client_token = 7;
optional int32 containerCount = 8;
optional int32 amFailCount = 9;
@@ -107,7 +113,7 @@ message ApplicationMasterProto {
message URLProto {
optional string scheme = 1;
- optional string host = 2;
+ optional string host = 2;
optional int32 port = 3;
optional string file = 4;
}
@@ -140,12 +146,13 @@ message ApplicationReportProto {
optional int32 rpc_port = 6;
optional string client_token = 7;
optional ApplicationStatusProto status = 8;
- optional ApplicationStateProto state = 9;
+ optional YarnApplicationStateProto yarn_application_state = 9;
optional ContainerProto masterContainer = 10;
optional string trackingUrl = 11;
optional string diagnostics = 12 [default = "N/A"];
optional int64 startTime = 13;
optional int64 finishTime = 14;
+ optional FinalApplicationStatusProto final_application_status = 15;
}
message NodeIdProto {
@@ -195,7 +202,7 @@ message AMResponseProto {
message ApplicationSubmissionContextProto {
optional ApplicationIdProto application_id = 1;
optional string application_name = 2 [default = "N/A"];
- optional string user = 3;
+ optional string user = 3;
optional string queue = 4 [default = "default"];
optional PriorityProto priority = 5;
optional ContainerLaunchContextProto am_container_spec = 6;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Thu Oct 6 01:16:48 2011
@@ -42,7 +42,7 @@ message FinishApplicationMasterRequestPr
optional ApplicationAttemptIdProto application_attempt_id = 1;
optional string diagnostics = 2;
optional string tracking_url = 3;
- optional string final_state = 4;
+ optional FinalApplicationStatusProto final_application_status = 4;
}
message FinishApplicationMasterResponseProto {
@@ -115,7 +115,7 @@ message GetClusterNodesRequestProto {
}
message GetClusterNodesResponseProto {
- repeated NodeReportProto nodeReports = 1;
+ repeated NodeReportProto nodeReports = 1;
}
message GetQueueInfoRequestProto {
@@ -133,7 +133,7 @@ message GetQueueUserAclsInfoRequestProto
}
message GetQueueUserAclsInfoResponseProto {
- repeated QueueUserACLInfoProto queueUserAcls = 1;
+ repeated QueueUserACLInfoProto queueUserAcls = 1;
}
//////////////////////////////////////////////////////
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Thu Oct 6 01:16:48 2011
@@ -44,14 +44,7 @@ public class YarnConfiguration extends C
/** ACL of who can modify this application.*/
public static final String APPLICATION_ACL_MODIFY_APP =
"yarn.app.acl.modify-job";
-
- /**
- * Security info class This is an internal config set and
- * read by YARN itself.
- */
- public static final String YARN_SECURITY_INFO =
- "yarn.security.info.class";
-
+
/** Delay before deleting resource to ease debugging of NM issues */
public static final String DEBUG_NM_DELETE_DELAY_SEC =
YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec";
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java Thu Oct 6 01:16:48 2011
@@ -45,12 +45,9 @@ public class HadoopYarnProtoRPC extends
@Override
public Object getProxy(Class protocol, InetSocketAddress addr,
Configuration conf) {
- Configuration myConf = new Configuration(conf);
LOG.info("Creating a HadoopYarnProtoRpc proxy for protocol " + protocol);
- LOG.debug("Configured SecurityInfo class name is "
- + myConf.get(YarnConfiguration.YARN_SECURITY_INFO));
-
- return RpcFactoryProvider.getClientFactory(myConf).getClient(protocol, 1, addr, myConf);
+ return RpcFactoryProvider.getClientFactory(conf).getClient(protocol, 1,
+ addr, conf);
}
@Override
@@ -60,8 +57,6 @@ public class HadoopYarnProtoRPC extends
int numHandlers) {
LOG.info("Creating a HadoopYarnProtoRpc server for protocol " + protocol +
" with " + numHandlers + " handlers");
- LOG.info("Configured SecurityInfo class name is "
- + conf.get(YarnConfiguration.YARN_SECURITY_INFO));
final RPC.Server hadoopServer;
hadoopServer =
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java Thu Oct 6 01:16:48 2011
@@ -46,13 +46,10 @@ public class HadoopYarnRPC extends YarnR
@Override
public Object getProxy(Class protocol, InetSocketAddress addr,
Configuration conf) {
- Configuration myConf = new Configuration(conf);
LOG.info("Creating a HadoopYarnRpc proxy for protocol " + protocol);
- LOG.debug("Configured SecurityInfo class name is "
- + myConf.get(YarnConfiguration.YARN_SECURITY_INFO));
- RPC.setProtocolEngine(myConf, protocol, AvroSpecificRpcEngine.class);
+ RPC.setProtocolEngine(conf, protocol, AvroSpecificRpcEngine.class);
try {
- return RPC.getProxy(protocol, 1, addr, myConf);
+ return RPC.getProxy(protocol, 1, addr, conf);
} catch (IOException e) {
throw new YarnException(e);
}
@@ -65,8 +62,6 @@ public class HadoopYarnRPC extends YarnR
int numHandlers) {
LOG.info("Creating a HadoopYarnRpc server for protocol " + protocol +
" with " + numHandlers + " handlers");
- LOG.info("Configured SecurityInfo class name is "
- + conf.get(YarnConfiguration.YARN_SECURITY_INFO));
RPC.setProtocolEngine(conf, protocol, AvroSpecificRpcEngine.class);
final RPC.Server hadoopServer;
try {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java Thu Oct 6 01:16:48 2011
@@ -44,7 +44,8 @@ public class ContainerTokenIdentifier ex
private String nmHostName;
private Resource resource;
- public ContainerTokenIdentifier(ContainerId containerID, String hostName, Resource r) {
+ public ContainerTokenIdentifier(ContainerId containerID, String hostName,
+ Resource r) {
this.containerId = containerID;
this.nmHostName = hostName;
this.resource = r;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Thu Oct 6 01:16:48 2011
@@ -24,9 +24,10 @@ import java.util.List;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -69,12 +70,12 @@ public class BuilderUtils {
}
}
- public static class ResourceRequestComparator
+ public static class ResourceRequestComparator
implements java.util.Comparator<org.apache.hadoop.yarn.api.records.ResourceRequest> {
@Override
public int compare(org.apache.hadoop.yarn.api.records.ResourceRequest r1,
org.apache.hadoop.yarn.api.records.ResourceRequest r2) {
-
+
// Compare priority, host and capability
int ret = r1.getPriority().compareTo(r2.getPriority());
if (ret == 0) {
@@ -198,12 +199,12 @@ public class BuilderUtils {
String nodeHttpAddress, Resource resource, Priority priority) {
ContainerId containerID =
newContainerId(recordFactory, appAttemptId, containerId);
- return newContainer(containerID, nodeId, nodeHttpAddress,
+ return newContainer(containerID, nodeId, nodeHttpAddress,
resource, priority);
}
public static Container newContainer(ContainerId containerId,
- NodeId nodeId, String nodeHttpAddress,
+ NodeId nodeId, String nodeHttpAddress,
Resource resource, Priority priority) {
Container container = recordFactory.newRecordInstance(Container.class);
container.setId(containerId);
@@ -242,8 +243,9 @@ public class BuilderUtils {
public static ApplicationReport newApplicationReport(
ApplicationId applicationId, String user, String queue, String name,
- String host, int rpcPort, String clientToken, ApplicationState state,
- String diagnostics, String url, long startTime, long finishTime) {
+ String host, int rpcPort, String clientToken, YarnApplicationState state,
+ String diagnostics, String url, long startTime, long finishTime,
+ FinalApplicationStatus finalStatus) {
ApplicationReport report = recordFactory
.newRecordInstance(ApplicationReport.class);
report.setApplicationId(applicationId);
@@ -253,20 +255,21 @@ public class BuilderUtils {
report.setHost(host);
report.setRpcPort(rpcPort);
report.setClientToken(clientToken);
- report.setState(state);
+ report.setYarnApplicationState(state);
report.setDiagnostics(diagnostics);
report.setTrackingUrl(url);
report.setStartTime(startTime);
report.setFinishTime(finishTime);
+ report.setFinalApplicationStatus(finalStatus);
return report;
}
-
+
public static Resource newResource(int memory) {
Resource resource = recordFactory.newRecordInstance(Resource.class);
resource.setMemory(memory);
return resource;
}
-
+
public static URL newURL(String scheme, String host, int port, String file) {
URL url = recordFactory.newRecordInstance(URL.class);
url.setScheme(scheme);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Thu Oct 6 01:16:48 2011
@@ -33,7 +33,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -181,4 +183,5 @@ public class ConverterUtils {
+ applicationAttmeptIdStr, n);
}
}
+
}