You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2017/06/13 22:21:13 UTC
hadoop git commit: MAPREDUCE-6895. Job end notification not send due
to YarnRuntimeException. Contributed by yunjiong zhao.
Repository: hadoop
Updated Branches:
refs/heads/trunk 036a24bab -> 6ed54f343
MAPREDUCE-6895. Job end notification not send due to YarnRuntimeException. Contributed by yunjiong zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ed54f34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ed54f34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ed54f34
Branch: refs/heads/trunk
Commit: 6ed54f3439ea9c7af6bf129ebe1938380febb5e2
Parents: 036a24b
Author: Junping Du <ju...@apache.org>
Authored: Tue Jun 13 15:21:04 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Tue Jun 13 15:21:04 2017 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/app/JobEndNotifier.java | 5 --
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 51 ++++++++++++--------
2 files changed, 31 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed54f34/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
index 836fad5..3bf0542 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
@@ -153,11 +153,6 @@ public class JobEndNotifier implements Configurable {
*/
public void notify(JobReport jobReport)
throws InterruptedException {
- // Do we need job-end notification?
- if (userUrl == null) {
- Log.getLog().info("Job end notification URL not set, skipping.");
- return;
- }
//Do string replacements for jobId and jobStatus
if (userUrl.contains(JOB_ID)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed54f34/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index ba7f929..1445481 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -646,6 +646,12 @@ public class MRAppMaster extends CompositeService {
// note in a workflow scenario, this may lead to creation of a new
// job (FIXME?)
+ JobEndNotifier notifier = null;
+ if (getConfig().get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL) != null) {
+ notifier = new JobEndNotifier();
+ notifier.setConf(getConfig());
+ }
+
try {
//if isLastAMRetry comes as true, should never set it to false
if ( !isLastAMRetry){
@@ -660,28 +666,11 @@ public class MRAppMaster extends CompositeService {
LOG.info("Calling stop for all the services");
MRAppMaster.this.stop();
- if (isLastAMRetry) {
+ if (isLastAMRetry && notifier != null) {
// Send job-end notification when it is safe to report termination to
// users and it is the last AM retry
- if (getConfig().get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL) != null) {
- try {
- LOG.info("Job end notification started for jobID : "
- + job.getReport().getJobId());
- JobEndNotifier notifier = new JobEndNotifier();
- notifier.setConf(getConfig());
- JobReport report = job.getReport();
- // If unregistration fails, the final state is unavailable. However,
- // at the last AM Retry, the client will finally be notified FAILED
- // from RM, so we should let users know FAILED via notifier as well
- if (!context.hasSuccessfullyUnregistered()) {
- report.setJobState(JobState.FAILED);
- }
- notifier.notify(report);
- } catch (InterruptedException ie) {
- LOG.warn("Job end notification interrupted for jobID : "
- + job.getReport().getJobId(), ie);
- }
- }
+ sendJobEndNotify(notifier);
+ notifier = null;
}
try {
@@ -693,10 +682,32 @@ public class MRAppMaster extends CompositeService {
} catch (Throwable t) {
LOG.warn("Graceful stop failed. Exiting.. ", t);
exitMRAppMaster(1, t);
+ } finally {
+ if (isLastAMRetry && notifier != null) {
+ sendJobEndNotify(notifier);
+ }
}
exitMRAppMaster(0, null);
}
+ private void sendJobEndNotify(JobEndNotifier notifier) {
+ try {
+ LOG.info("Job end notification started for jobID : "
+ + job.getReport().getJobId());
+ // If unregistration fails, the final state is unavailable. However,
+ // at the last AM Retry, the client will finally be notified FAILED
+ // from RM, so we should let users know FAILED via notifier as well
+ JobReport report = job.getReport();
+ if (!context.hasSuccessfullyUnregistered()) {
+ report.setJobState(JobState.FAILED);
+ }
+ notifier.notify(report);
+ } catch (InterruptedException ie) {
+ LOG.warn("Job end notification interrupted for jobID : "
+ + job.getReport().getJobId(), ie);
+ }
+ }
+
/** MRAppMaster exit method which has been instrumented for both runtime and
* unit testing.
* If the main thread has not been started, this method was called from a
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org