You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/03/21 21:14:27 UTC
[14/50] [abbrv] hadoop git commit: MAPREDUCE-6579.
JobStatus#getFailureInfo should not output diagnostic information when the
job is running. (Akira AJISAKA via wangda)
MAPREDUCE-6579. JobStatus#getFailureInfo should not output diagnostic information when the job is running. (Akira AJISAKA via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6529c875
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6529c875
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6529c875
Branch: refs/heads/YARN-3368
Commit: 6529c87551e07e9da1e1b273eff4c6bad6b3e838
Parents: 5176a69
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Mar 15 17:28:40 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Mar 15 17:28:40 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/mapred/ClusterStatus.java | 35 +++++++-
.../org/apache/hadoop/mapred/NotRunningJob.java | 7 +-
.../apache/hadoop/mapred/TestNetworkedJob.java | 89 ++++++++++----------
3 files changed, 84 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6529c875/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java
index 904897b..c87b583 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java
@@ -171,7 +171,40 @@ public class ClusterStatus implements Writable {
sb.append(blackListReport.replace("\n", ":"));
return sb.toString();
}
-
+
+ @Override
+ public int hashCode() {
+ int result = trackerName != null ? trackerName.hashCode() : 0;
+ result = 31 * result + (reasonForBlackListing != null ?
+ reasonForBlackListing.hashCode() : 0);
+ result = 31 * result + (blackListReport != null ?
+ blackListReport.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final BlackListInfo that = (BlackListInfo) obj;
+ if (trackerName == null ? that.trackerName != null :
+ !trackerName.equals(that.trackerName)) {
+ return false;
+ }
+ if (reasonForBlackListing == null ? that.reasonForBlackListing != null :
+ !reasonForBlackListing.equals(that.reasonForBlackListing)) {
+ return false;
+ }
+ if (blackListReport == null ? that.blackListReport != null :
+ !blackListReport.equals(that.blackListReport)) {
+ return false;
+ }
+ return true;
+ }
}
public static final long UNINITIALIZED_MEMORY_VALUE = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6529c875/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
index 03552e4..90635a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
@@ -137,7 +137,12 @@ public class NotRunningJob implements MRClientProtocol {
jobReport.setJobState(jobState);
jobReport.setUser(applicationReport.getUser());
jobReport.setStartTime(applicationReport.getStartTime());
- jobReport.setDiagnostics(applicationReport.getDiagnostics());
+ YarnApplicationState state = applicationReport.getYarnApplicationState();
+ if (state == YarnApplicationState.KILLED
+ || state == YarnApplicationState.FAILED
+ || state == YarnApplicationState.FINISHED) {
+ jobReport.setDiagnostics(applicationReport.getDiagnostics());
+ }
jobReport.setJobName(applicationReport.getName());
jobReport.setTrackingUrl(applicationReport.getTrackingUrl());
jobReport.setFinishTime(applicationReport.getFinishTime());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6529c875/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
index 2e0887e..45c7d1f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
@@ -155,9 +155,9 @@ public class TestNetworkedJob {
// test getters
assertTrue(runningJob.getConfiguration().toString()
.endsWith("0001/job.xml"));
- assertEquals(runningJob.getID(), jobId);
- assertEquals(runningJob.getJobID(), jobId.toString());
- assertEquals(runningJob.getJobName(), "N/A");
+ assertEquals(jobId, runningJob.getID());
+ assertEquals(jobId.toString(), runningJob.getJobID());
+ assertEquals("N/A", runningJob.getJobName());
assertTrue(runningJob.getJobFile().endsWith(
".staging/" + runningJob.getJobID() + "/job.xml"));
assertTrue(runningJob.getTrackingURL().length() > 0);
@@ -169,40 +169,40 @@ public class TestNetworkedJob {
TaskCompletionEvent[] tce = runningJob.getTaskCompletionEvents(0);
assertEquals(tce.length, 0);
- assertEquals(runningJob.getHistoryUrl(),"");
+ assertEquals("", runningJob.getHistoryUrl());
assertFalse(runningJob.isRetired());
- assertEquals( runningJob.getFailureInfo(),"");
- assertEquals(runningJob.getJobStatus().getJobName(), "N/A");
- assertEquals(client.getMapTaskReports(jobId).length, 0);
+ assertEquals("", runningJob.getFailureInfo());
+ assertEquals("N/A", runningJob.getJobStatus().getJobName());
+ assertEquals(0, client.getMapTaskReports(jobId).length);
try {
client.getSetupTaskReports(jobId);
} catch (YarnRuntimeException e) {
- assertEquals(e.getMessage(), "Unrecognized task type: JOB_SETUP");
+ assertEquals("Unrecognized task type: JOB_SETUP", e.getMessage());
}
try {
client.getCleanupTaskReports(jobId);
} catch (YarnRuntimeException e) {
- assertEquals(e.getMessage(), "Unrecognized task type: JOB_CLEANUP");
+ assertEquals("Unrecognized task type: JOB_CLEANUP", e.getMessage());
}
- assertEquals(client.getReduceTaskReports(jobId).length, 0);
+ assertEquals(0, client.getReduceTaskReports(jobId).length);
// test ClusterStatus
ClusterStatus status = client.getClusterStatus(true);
- assertEquals(status.getActiveTrackerNames().size(), 2);
+ assertEquals(2, status.getActiveTrackerNames().size());
// it method does not implemented and always return empty array or null;
- assertEquals(status.getBlacklistedTrackers(), 0);
- assertEquals(status.getBlacklistedTrackerNames().size(), 0);
- assertEquals(status.getBlackListedTrackersInfo().size(), 0);
- assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING);
- assertEquals(status.getMapTasks(), 1);
- assertEquals(status.getMaxMapTasks(), 20);
- assertEquals(status.getMaxReduceTasks(), 4);
- assertEquals(status.getNumExcludedNodes(), 0);
- assertEquals(status.getReduceTasks(), 1);
- assertEquals(status.getTaskTrackers(), 2);
- assertEquals(status.getTTExpiryInterval(), 0);
- assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING);
- assertEquals(status.getGraylistedTrackers(), 0);
+ assertEquals(0, status.getBlacklistedTrackers());
+ assertEquals(0, status.getBlacklistedTrackerNames().size());
+ assertEquals(0, status.getBlackListedTrackersInfo().size());
+ assertEquals(JobTrackerStatus.RUNNING, status.getJobTrackerStatus());
+ assertEquals(1, status.getMapTasks());
+ assertEquals(20, status.getMaxMapTasks());
+ assertEquals(4, status.getMaxReduceTasks());
+ assertEquals(0, status.getNumExcludedNodes());
+ assertEquals(1, status.getReduceTasks());
+ assertEquals(2, status.getTaskTrackers());
+ assertEquals(0, status.getTTExpiryInterval());
+ assertEquals(JobTrackerStatus.RUNNING, status.getJobTrackerStatus());
+ assertEquals(0, status.getGraylistedTrackers());
// test read and write
ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
@@ -219,34 +219,34 @@ public class TestNetworkedJob {
// test taskStatusfilter
JobClient.setTaskOutputFilter(job, TaskStatusFilter.ALL);
- assertEquals(JobClient.getTaskOutputFilter(job), TaskStatusFilter.ALL);
+ assertEquals(TaskStatusFilter.ALL, JobClient.getTaskOutputFilter(job));
// runningJob.setJobPriority(JobPriority.HIGH.name());
// test default map
- assertEquals(client.getDefaultMaps(), 20);
- assertEquals(client.getDefaultReduces(), 4);
- assertEquals(client.getSystemDir().getName(), "jobSubmitDir");
+ assertEquals(20, client.getDefaultMaps());
+ assertEquals(4, client.getDefaultReduces());
+ assertEquals("jobSubmitDir", client.getSystemDir().getName());
// test queue information
JobQueueInfo[] rootQueueInfo = client.getRootQueues();
- assertEquals(rootQueueInfo.length, 1);
- assertEquals(rootQueueInfo[0].getQueueName(), "default");
+ assertEquals(1, rootQueueInfo.length);
+ assertEquals("default", rootQueueInfo[0].getQueueName());
JobQueueInfo[] qinfo = client.getQueues();
- assertEquals(qinfo.length, 1);
- assertEquals(qinfo[0].getQueueName(), "default");
- assertEquals(client.getChildQueues("default").length, 0);
- assertEquals(client.getJobsFromQueue("default").length, 1);
+ assertEquals(1, qinfo.length);
+ assertEquals("default", qinfo[0].getQueueName());
+ assertEquals(0, client.getChildQueues("default").length);
+ assertEquals(1, client.getJobsFromQueue("default").length);
assertTrue(client.getJobsFromQueue("default")[0].getJobFile().endsWith(
"/job.xml"));
JobQueueInfo qi = client.getQueueInfo("default");
- assertEquals(qi.getQueueName(), "default");
- assertEquals(qi.getQueueState(), "running");
+ assertEquals("default", qi.getQueueName());
+ assertEquals("running", qi.getQueueState());
QueueAclsInfo[] aai = client.getQueueAclsForCurrentUser();
- assertEquals(aai.length, 2);
- assertEquals(aai[0].getQueueName(), "root");
- assertEquals(aai[1].getQueueName(), "default");
+ assertEquals(2, aai.length);
+ assertEquals("root", aai[0].getQueueName());
+ assertEquals("default", aai[1].getQueueName());
// test JobClient
// The following asserts read JobStatus twice and ensure the returned
@@ -282,12 +282,11 @@ public class TestNetworkedJob {
BlackListInfo info2 = new BlackListInfo();
info2.readFields(new DataInputStream(new ByteArrayInputStream(byteOut
.toByteArray())));
- assertEquals(info, info);
- assertEquals(info.toString(), info.toString());
- assertEquals(info.getTrackerName(), "trackerName");
- assertEquals(info.getReasonForBlackListing(), "reasonForBlackListing");
- assertEquals(info.getBlackListReport(), "blackListInfo");
-
+ assertEquals(info, info2);
+ assertEquals(info.toString(), info2.toString());
+ assertEquals("trackerName", info2.getTrackerName());
+ assertEquals("reasonForBlackListing", info2.getReasonForBlackListing());
+ assertEquals("blackListInfo", info2.getBlackListReport());
}
/**
* test run from command line JobQueueClient